diff --git a/autosubmit/auto-transfer_provenance.sh b/autosubmit/auto-transfer_provenance.sh new file mode 100644 index 0000000000000000000000000000000000000000..c8dd7cb94b0cc66a37c9c6f05b2ddcf12c445d9d --- /dev/null +++ b/autosubmit/auto-transfer_provenance.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +############ AUTOSUBMIT INPUTS ############ +outdir=%common.OUTDIR% +expid=%config.EXPID% +############################### + +#JSON directory +dir1=/esarchive/autosubmit/${expid}/tmp/$(basename ${outdir})/outputs/provenance + +destdir1=/gpfs/archive/bsc32/${dir1} + +#Recipes directory +dir2=/esarchive/autosubmit/${expid}/tmp/$(basename ${outdir})/logs/recipes + +destdir2=/gpfs/archive/bsc32/${dir2} + +#Create directories and copy files +mkdir -p ${destdir1} + +cp -r /gpfs/archive/bsc32/${outdir}/outputs/provenance/* ${destdir1} + +mkdir -p ${destdir2} + +cp -r /gpfs/archive/bsc32/${outdir}/logs/recipes/* ${destdir2} diff --git a/autosubmit/conf_esarchive/autosubmit.yml b/autosubmit/conf_esarchive/autosubmit.yml index 0fd5d5c6aaf61945d131da77cda08d8d1fdd86cd..00469806fc8aa1ac679d7d8a12297b2d5dfc304f 100644 --- a/autosubmit/conf_esarchive/autosubmit.yml +++ b/autosubmit/conf_esarchive/autosubmit.yml @@ -1,6 +1,6 @@ config: EXPID: - AUTOSUBMIT_VERSION: 4.0.0b0 + AUTOSUBMIT_VERSION: 4.1.11 MAXWAITINGJOBS: 16 # Default maximum number of jobs to be running at the same time at any platform # Default: 6 diff --git a/autosubmit/conf_esarchive/jobs.yml b/autosubmit/conf_esarchive/jobs.yml index 8196dcc10ba0dd5fab8fd2a5e738ea852fa4a555..73dbddbcba6b245cf45926996c423cd0f7bfc6b6 100644 --- a/autosubmit/conf_esarchive/jobs.yml +++ b/autosubmit/conf_esarchive/jobs.yml @@ -6,6 +6,7 @@ JOBS: NOTIFY_ON: PLATFORM: nord3v2 PROCESSORS: + TASKS: 1 ## TODO: Uncomment (see #162) # NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes @@ -16,6 +17,7 @@ JOBS: NOTIFY_ON: PLATFORM: nord3v2 PROCESSORS: + TASKS: 1 ## TODO: Uncomment # NODES: 1 DEPENDENCIES: @@ -28,5 +30,14 @@ JOBS: PLATFORM: nord3v2 NOTIFY_ON: PROCESSORS: 1 + TASKS: 1 DEPENDENCIES: - + transfer_provenance: + FILE: autosubmit/auto-transfer_provenance.sh + RUNNING: once + WALLCLOCK: 00:10 + PLATFORM: transfer + NOTIFY_ON: + PROCESSORS: 1 + TASKS: 1 + DEPENDENCIES: diff --git a/autosubmit/conf_esarchive/platforms.yml b/autosubmit/conf_esarchive/platforms.yml index 78056d62973552432a9e7b55194b8c5e0ecac09a..7199f0e083f064e8254811dd71f6809543574159 100644 --- a/autosubmit/conf_esarchive/platforms.yml +++ b/autosubmit/conf_esarchive/platforms.yml @@ -9,3 +9,9 @@ Platforms: PROCESSORS_PER_NODE: 16 SERIAL_QUEUE: debug QUEUE: bsc_es + transfer: + TYPE: slurm + HOST: transfer2.bsc.es + PROJECT: bsc32 + SCRATCH_DIR: /gpfs/scratch/ + USER: diff --git a/autosubmit/conf_gpfs/autosubmit.yml b/autosubmit/conf_gpfs/autosubmit.yml index 0fd5d5c6aaf61945d131da77cda08d8d1fdd86cd..00469806fc8aa1ac679d7d8a12297b2d5dfc304f 100644 --- a/autosubmit/conf_gpfs/autosubmit.yml +++ b/autosubmit/conf_gpfs/autosubmit.yml @@ -1,6 +1,6 @@ config: EXPID: - AUTOSUBMIT_VERSION: 4.0.0b0 + AUTOSUBMIT_VERSION: 4.1.11 MAXWAITINGJOBS: 16 # Default maximum number of jobs to be running at the same time at any platform # Default: 6 diff --git a/autosubmit/conf_gpfs/jobs.yml b/autosubmit/conf_gpfs/jobs.yml index e6806ca7192dfc4d74b22cfd3dc5ca1c5d5d1cf7..556920a217cfd63354d0a7a8a8d4f06361cc13ad 100644 --- a/autosubmit/conf_gpfs/jobs.yml +++ b/autosubmit/conf_gpfs/jobs.yml @@ -6,6 +6,7 @@ JOBS: PLATFORM: transfer NOTIFY_ON: PROCESSORS: 1 + TASKS: 1 verification: FILE: autosubmit/auto-verification.sh RUNNING: chunk @@ -13,6 +14,7 @@ JOBS: NOTIFY_ON: PLATFORM: PROCESSORS: + TASKS: 1 ## TODO: Uncomment # NODES: 1 # SPLITS: # n_atomic_recipes, number of atomic recipes @@ -23,6 +25,7 @@ JOBS: NOTIFY_ON: PLATFORM: PROCESSORS: + TASKS: 1 DEPENDENCIES: ## TODO: Uncomment # NODES: 1 @@ -35,6 +38,7 @@ JOBS: PLATFORM: NOTIFY_ON: PROCESSORS: 1 + TASKS: 1 DEPENDENCIES: transfer_results: FILE: autosubmit/auto-transfer_results.sh @@ -43,3 +47,13 @@ JOBS: PLATFORM: transfer NOTIFY_ON: PROCESSORS: 1 + TASKS: 1 + transfer_provenance: + FILE: autosubmit/auto-transfer_provenance.sh + RUNNING: once + WALLCLOCK: 00:10 + PLATFORM: transfer + NOTIFY_ON: + PROCESSORS: 1 + TASKS: 1 + DEPENDENCIES: diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index c656d0de422d1808246112885239c9148f8c6dae..ded4734b64dc4b437ba8327cee9bd5c0c3f68a49 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -5,6 +5,7 @@ gpfs: EC-Earth3-i4: name: "EC-Earth3-i4" institution: "EC-Earth-Consortium" + modelling_center: "EC-Earth-Consortium" src: hcst: "exp/CMIP6/dcppA-hindcast/EC-Earth3-i4/DCPP/EC-Earth-Consortium/EC-Earth3-i4/dcppA-hindcast/" fcst: @@ -43,6 +44,7 @@ esarchive: EC-Earth3-i1: name: "EC-Earth3-i1" institution: "EC-Earth-Consortium" + modelling_center: "EC-Earth-Consortium" src: hcst: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" fcst: @@ -72,6 +74,7 @@ esarchive: EC-Earth3-i2: name: "EC-Earth3-i2" institution: "EC-Earth-Consortium" + modelling_center: "EC-Earth-Consortium" src: hcst: "exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" fcst: @@ -94,6 +97,7 @@ esarchive: EC-Earth3-i4: name: "EC-Earth3-i4" institution: "EC-Earth-Consortium" + modelling_center: "EC-Earth-Consortium" src: hcst: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" fcst: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppB-forecast/" @@ -225,6 +229,7 @@ esarchive: CMCC-CM2-SR5: name: "CMCC-CM2-SR5" institution: "Euro-Mediterranean Center on Climate Change" + modelling_center: "CMCC" src: hcst: "exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/" fcst: "exp/CMIP6/dcppB-forecast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppB-forecast/" diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml index 5236698cafdbd0b963680900c92d5459a30424c8..66ebb7be671466684a123021d279a6fb84529b24 100644 --- a/conf/archive_reference.yml +++ b/conf/archive_reference.yml @@ -4,6 +4,7 @@ gpfs: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ERA5" src: "recon/ecmwf/era5/" monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", "psl":"monthly_mean/psl_f1h-r1440x721cds/", @@ -20,6 +21,7 @@ esarchive: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ERA5" src: "recon/ecmwf/era5/" weekly_mean: {"tas":"weekly_mean/tas_f1h-r1440x721cds/", "prlr":"weekly_mean/prlr_f1h-r1440x721cds/"} @@ -58,6 +60,7 @@ esarchive: ERA5-Land: name: "ERA5-Land" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ERA5" src: "recon/ecmwf/era5land/" daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", @@ -71,6 +74,7 @@ esarchive: UERRA: name: "ECMWF UERRA" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "recon/ecmwf/uerra_mescan/" daily_mean: {"tas":"daily_mean/tas_f6h/"} monthly_mean: {"tas":"monthly_mean/tas_f6h/"} @@ -79,6 +83,7 @@ esarchive: CERRA: name: "ECMWF CERRA" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "recon/ecmwf/cerra/" daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", @@ -94,6 +99,7 @@ esarchive: CERRA-Land: name: "ECMWF CERRA-Land" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "recon/ecmwf/cerraland/" daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} @@ -102,6 +108,7 @@ esarchive: HadCRUT5: name: "HadCRUT5" institution: "Met Office" + modelling_center: "Met Office" src: "obs/ukmo/hadcrut_v5.0_analysis/" monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} calendar: "proleptic_gregorian" @@ -109,6 +116,7 @@ esarchive: BEST: name: "BEST" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "obs/berkeleyearth/berkeleyearth/" daily_mean: {"tas":"daily_mean/tas/"} monthly_mean: {"tas":"monthly_mean/tas/"} @@ -177,6 +185,7 @@ mars: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ERA5" src: "GRIB_era5_tas/" monthly_mean: {"tas":""} calendar: "gregorian" @@ -189,6 +198,7 @@ sample: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ERA5" src: "GRIB_era5_tas/" monthly_mean: {"tas":"", "prlr":""} calendar: "gregorian" @@ -201,6 +211,7 @@ IFCA: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ERA5" src: "ERA5/" daily_mean: {"psl":"psl_f1h-r1440x721cds/"} calendar: "gregorian" diff --git a/conf/archive_seasonal.yml b/conf/archive_seasonal.yml index 7b5ecda2b915af9340adf200b3c8af0a4582e977..fed0db59c335940eb703476f44041d0dfbde2347 100644 --- a/conf/archive_seasonal.yml +++ b/conf/archive_seasonal.yml @@ -23,6 +23,7 @@ gpfs: ECMWF-SEAS5.1: name: "ECMWF SEAS5 (v5.1)" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "exp/ecmwf/system51c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", @@ -55,6 +56,7 @@ gpfs: CMCC-SPS3.5: name: "CMCC-SPS3.5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "CMCC" src: "exp/cmcc/system35c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", @@ -70,6 +72,7 @@ gpfs: Meteo-France-System8: name: "Meteo-France System 8" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "Meteo-France" src: "exp/meteofrance/system8c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_s0-24h/", @@ -85,6 +88,7 @@ gpfs: UK-MetOffice-Glosea601: name: "UK MetOffice GloSea 6 (v6.01)" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "UK MetOffice" src: "exp/ukmo/glosea6_system601-c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", @@ -115,6 +119,7 @@ gpfs: NCEP-CFSv2: name: "NCEP CFSv2" institution: "NOAA NCEP" #? + modelling_center: "NOAA" src: "exp/ncep/cfs-v2/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", @@ -147,6 +152,7 @@ gpfs: ECCC-CanCM4i: name: "ECCC CanCM4i (v3)" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECCC" src: "exp/eccc/eccc3/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_s0-24h/", @@ -163,6 +169,7 @@ gpfs: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ERA5" src: "recon/ecmwf/era5/" monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", "psl":"monthly_mean/psl_f1h-r1440x721cds/", @@ -180,6 +187,7 @@ esarchive: ECMWF-SEAS5: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "exp/ecmwf/system5c3s/" daily_mean: {"tas":"daily_mean/tas_f6h/", "rsds":"daily/rsds_s0-24h/", "prlr":"daily/prlr_s0-24h/", "tasmin":"daily/tasmin/", @@ -209,6 +217,7 @@ esarchive: ECMWF-SEAS5.1: name: "ECMWF SEAS5 (v5.1)" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "exp/ecmwf/system51c3s/" daily_mean: {"tas":"daily_mean/tas_f6h/", "prlr":"daily/prlr_s0-24h/", "sfcWind":"daily_mean/sfcWind_f6h/", @@ -229,6 +238,7 @@ esarchive: Meteo-France-System7: name: "Meteo-France System 7" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "Meteo-France" src: "exp/meteofrance/system7c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "g500":"monthly_mean/g500_f12h/", "prlr":"monthly_mean/prlr_f24h/", "sfcWind": "monthly_mean/sfcWind_f6h/", @@ -272,6 +282,7 @@ esarchive: CMCC-SPS3.5: name: "CMCC-SPS3.5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "CMCC" src: "exp/cmcc/system35c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", "g500":"monthly_mean/g500_f12h/", "sfcWind":"monthly_mean/sfcWind_f6h/", @@ -286,6 +297,7 @@ esarchive: JMA-CPS2: name: "JMA System 2" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "JMA" src: "exp/jma/system2c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} @@ -298,6 +310,7 @@ esarchive: ECCC-CanCM4i: name: "ECCC CanCM4i" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECCC" src: "exp/eccc/eccc1/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} @@ -311,6 +324,7 @@ esarchive: UK-MetOffice-Glosea600: name: "UK MetOffice GloSea 6 (v6.0)" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "UK MetOffice" src: "exp/ukmo/glosea6_system600-c3s/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "tasmin":"monthly_mean/tasmin_f24h/", "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f24h/"} @@ -351,6 +365,7 @@ esarchive: NCEP-CFSv2: name: "NCEP CFSv2" institution: "NOAA NCEP" #? + modelling_center: "NOAA" src: "exp/ncep/cfs-v2/" monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} @@ -367,6 +382,7 @@ mars: ECMWF-SEAS5: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "GRIB_system5_tas_CORRECTED/" monthly_mean: {"tas":""} nmember: @@ -382,6 +398,7 @@ sample: ECMWF-SEAS5.1: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: monthly_mean: {"tas":"", "prlr":""} nmember: @@ -397,6 +414,7 @@ IFCA: ECMWF-SEAS5: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" + modelling_center: "ECMWF" src: "ECMWF-SEAS5/" daily_mean: {"tas":"daily_mean/tas_f6h/", "psl":"daily_mean/psl_f6h/", "prlr":"daily_mean/prlr_s0-24h/", "sfcWind":"daily_mean/sfcWind_f6h/", diff --git a/conf/archive_subseasonal.yml b/conf/archive_subseasonal.yml index 300884dcfc9482fe656ca1e41a3974a1cabf498c..ab0f386382418c2a6fcb149176bb625034d6f778 100644 --- a/conf/archive_subseasonal.yml +++ b/conf/archive_subseasonal.yml @@ -4,6 +4,7 @@ esarchive: NCEP-CFSv2: name: "NCEP CFSv2" institution: "NOAA NCEP" #? + modelling_center: "NOAA" src: "exp/ncep/cfs-v2/" weekly_mean: {"tas":"weekly_mean/s2s/tas_f24h/", "prlr":"weekly_mean/s2s/prlr_f24h/", diff --git a/conf/autosubmit.yml b/conf/autosubmit.yml index 99b29f3c8022de791317ce78d2312143ba916c5b..a0b55ca941913bad6574c5ee7856c7907049e6ed 100644 --- a/conf/autosubmit.yml +++ b/conf/autosubmit.yml @@ -1,14 +1,14 @@ esarchive: platform: nord3v2 - module_version: autosubmit/4.0.98-foss-2015a-Python-3.7.3 - auto_version: 4.0.98 + module_version: autosubmit/4.1.11-foss-2021b-Python-3.9.6 + auto_version: 4.1.11 conf_format: yaml experiment_dir: /esarchive/autosubmit/ userID: bsc032 tmp_dir: mars: platform: NORD3 ## TO BE CHANGED - module_version: autosubmit/4.0.0b-foss-2015a-Python-3.7.3 ## TO BE CHANGED + module_version: autosubmit/4.1.11-foss-2021b-Python-3.9.6 ## TO BE CHANGED auto_version: 4.0.0 conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED @@ -16,8 +16,8 @@ mars: tmp_dir: gpfs: platform: mn5 - module_version: autosubmit/4.0.98-foss-2015a-Python-3.7.3 - auto_version: 4.0.98 + module_version: autosubmit/4.1.11-foss-2021b-Python-3.9.6 + auto_version: 4.1.11 conf_format: yaml experiment_dir: /esarchive/autosubmit/ ## TO BE CHANGED userID: bsc032 diff --git a/conf/variable-dictionary.yml b/conf/variable-dictionary.yml index 78b19b1a3fbd08002cd59e18da6786cd7dc3f33b..8390446bf044ce70834291cc4c6b813e19cc357c 100644 --- a/conf/variable-dictionary.yml +++ b/conf/variable-dictionary.yml @@ -238,65 +238,267 @@ coords: metrics: enscorr: long_name: "Ensemble Mean Correlation Coefficient" + representation: "SkillScore" + definition: "Calculate correlation between forecasts and observations for an ensemble forecast, including an adjustment for finite ensemble sizes" enscorr_specs: long_name: "Ensemble Mean Correlation Coefficient" + representation: "SkillScore" + definition: "Calculate correlation between forecasts and observations for an ensemble forecast, including an adjustment for finite ensemble sizes" enscorr_p.value: long_name: "Ensemble Mean Correlation p-value" + representation: + definition: enscorr_conf.low: long_name: "Ensemble Mean Correlation Lower Confidence Interval" + representation: + definition: enscorr_conf.up: long_name: "Ensemble Mean Correlation Upper Confidence Interval" + representation: + definition: enscorr_significance: long_name: "Ensemble Mean Correlation Statistical Significance" + representation: + definition: corr: long_name: "Ensemble Correlation Coefficient" + representation: + definition: corr_specs: long_name: "Ensemble Correlation Coefficient" + representation: + definition: corr_p.value: long_name: "Ensemble Correlation p-value" + representation: + definition: corr_conf.low: long_name: "Ensemble Correlation Lower Confidence Interval" + representation: + definition: corr_conf.up: long_name: "Ensemble Correlation Upper Confidence Interval" + representation: + definition: corr_significance: long_name: "Ensemble Correlation Statistical Significance" + representation: + definition: rps: long_name: "Ranked Probability Score" + representation: "SkillScore" + definition: "The Ranked Probability Score (RPS; Wilks, 2011) is defined as the sum of the squared differences between the cumulative forecast probabilities (computed from the ensemble members) and the observations + (defined as 0 did not happen and 100 of multi-categorical probabilistic forecasts. The RPS ranges between 0 (perfect forecast) and n-1 (worst possible forecast), where n is the number of categories. + In the case of a forecast divided into two categories (the lowest number of categories that a probabilistic forecast can have), the RPS corresponds to the Brier Score (BS; Wilks, 2011), therefore ranging between 0 and 1. + The function first calculates the probabilities for forecasts and observations, then use them to calculate RPS. Or, the probabilities of exp and obs can be provided directly to compute the score. + If there is more than one dataset, RPS will be computed for each pair of exp and obs data. The fraction of acceptable NAs can be adjusted." frps: long_name: "Fair Ranked Probability Score" + representation: "MulticategoryProbability" + definition: "Calculate the difference (mean score of the reference forecast) minus (mean score of the forecast). Uncertainty is assessed by the Diebold-Mariano test for equality of predictive accuracy." rpss: long_name: "Ranked Probability Skill Score" + representation: "SkillScore" + definition: "The Ranked Probability Skill Score (RPSS; Wilks, 2011) is the skill score based on the Ranked Probability Score (RPS; Wilks, 2011). It can be used to assess whether a forecast presents an improvement or worsening + with respect to a reference forecast. The RPSS ranges between minus infinite and 1. If the RPSS is positive, it indicates that the forecast has higher skill than the reference forecast, while a negative value means that it has a lower skill. + Examples of reference forecasts are the climatological forecast (same probabilities for all categories for all time steps), persistence, a previous model version, and another model. It is computed as RPSS = 1 - RPS_exp / RPS_ref. + The statistical significance is obtained based on a Random Walk test at the specified confidence level (DelSole and Tippett, 2016). The function accepts either the ensemble members or the probabilities of each data as inputs. + If there is more than one dataset, RPSS will be computed for each pair of exp and obs data. The NA ratio of data will be examined before the calculation. If the ratio is higher than the threshold (assigned by parameter na.rm), + NA will be returned directly. NAs are counted by per-pair method, which means that only the time steps that all the datasets have values count as non-NA values." rpss_significance: long_name: "Ranked Probability Skill Score Statistical Significance" rpss_specs: long_name: "Ranked Probability Skill Score" + representation: "SkillScore" + definition: "The Ranked Probability Skill Score (RPSS; Wilks, 2011) is the skill score based on the Ranked Probability Score (RPS; Wilks, 2011). It can be used to assess whether a forecast presents an improvement or worsening + with respect to a reference forecast. The RPSS ranges between minus infinite and 1. If the RPSS is positive, it indicates that the forecast has higher skill than the reference forecast, while a negative value means that it has a lower skill. + Examples of reference forecasts are the climatological forecast (same probabilities for all categories for all time steps), persistence, a previous model version, and another model. It is computed as RPSS = 1 - RPS_exp / RPS_ref. + The statistical significance is obtained based on a Random Walk test at the specified confidence level (DelSole and Tippett, 2016). The function accepts either the ensemble members or the probabilities of each data as inputs. + If there is more than one dataset, RPSS will be computed for each pair of exp and obs data. The NA ratio of data will be examined before the calculation. If the ratio is higher than the threshold (assigned by parameter na.rm), + NA will be returned directly. NAs are counted by per-pair method, which means that only the time steps that all the datasets have values count as non-NA values. " frpss: long_name: "Fair Ranked Probability Skill Score" + representation: "SkillScore" + definition: "Calculate the difference (mean skill score of the reference forecast) minus (mean skill score of the forecast). Uncertainty is assessed by the Diebold-Mariano test for equality of predictive accuracy." frpss_significance: long_name: "Fair Ranked Probability Skill Score Statistical Significance" frpss_specs: long_name: "Fair Ranked Probability Skill Score" + representation: "SkillScore" + definition: "Calculate the difference (mean skill score of the reference forecast) minus (mean skill score of the forecast). Uncertainty is assessed by the Diebold-Mariano test for equality of predictive accuracy." bss10: long_name: "Brier Skill Score Lower Extreme" + representation: "SkillScore" + definition: "Compute the lower extreme Brier score (BS) and the components of its standard decompostion with the two withinbin components described in Stephenson et al., (2008). + It also returns the bias-corrected decomposition of the BS (Ferro and Fricker, 2012). It has the climatology as the reference forecast." bss10_specs: long_name: "Brier Skill Score Lower Extreme" + representation: "SkillScore" + definition: "Compute the lower extreme Brier score (BS) and the components of its standard decompostion with the two withinbin components described in Stephenson et al., (2008). + It also returns the bias-corrected decomposition of the BS (Ferro and Fricker, 2012). It has the climatology as the reference forecast." bss10_significance: long_name: "Brier Score Lower Extreme Statistical Significance" + representation: + definition: bss90: long_name: "Brier Skill Score Upper Extreme" + representation: "SkillScore" + definition: "Compute the upper extreme Brier score (BS) and the components of its standard decompostion with the two withinbin components described in Stephenson et al., (2008). + It also returns the bias-corrected decomposition of the BS (Ferro and Fricker, 2012). It has the climatology as the reference forecast." bss90_significance: long_name: "Brier Skill Score Upper Extreme Statistical Significance" + representation: + definition: crps: long_name: "Continuous Ranked Probability Score" + representation: "EnsembleForecast" + definition: "The Continuous Ranked Probability Score (CRPS; Wilks, 2011) is the continuous version of the Ranked Probability Score (RPS; Wilks, 2011). + It is a skill metric to evaluate the full distribution of probabilistic forecasts. It has a negative orientation (i.e., the higher-quality forecast the smaller CRPS) and it rewards + the forecast that has probability concentration around the observed value. In case of a deterministic forecast, the CRPS is reduced to the mean absolute error. + It has the same units as the data. The function is based on enscrps_cpp from SpecsVerification. If there is more than one dataset, CRPS will be computed for each pair of exp and obs data." crpss: long_name: "Continuous Ranked Probability Skill Score" + representation: "SkillScore" + definition: "The Continuous Ranked Probability Skill Score (CRPSS; Wilks, 2011) is the skill score based on the Continuous Ranked Probability Score (CRPS; Wilks, 2011). + It can be used to assess whether a forecast presents an improvement or worsening with respect to a reference forecast. The CRPSS ranges between minus infinite and 1. + If the CRPSS is positive, it indicates that the forecast has higher skill than the reference forecast, while a negative value means that it has a lower skill. + Examples of reference forecasts are the climatological forecast, persistence, a previous model version, or another model. It is computed as CRPSS = 1 - CRPS_exp / CRPS_ref. + The statistical significance is obtained based on a Random Walk test at the specified confidence level (DelSole and Tippett, 2016)." mean_bias: long_name: "Mean Bias" + representation: "DeterministicContinuous" + definition: "The Mean Bias or Mean Error (Wilks, 2011) is defined as the mean difference between the ensemble mean forecast and the observations. + It is a deterministic metric. Positive values indicate that the forecasts are on average too high and negative values indicate that the forecasts are on average too low. + It also allows to compute the Absolute Mean Bias or bias without temporal mean. + If there is more than one dataset, the result will be computed for each pair of exp and obs data" mean_bias_ss: long_name: "Mean Bias Skill Score" + representation: "SkillScore" + definition: "The Absolute Mean Bias Skill Score is based on the Absolute Mean Error (Wilks, 2011) between the ensemble mean forecast and the observations. + It measures the accuracy of the forecast in comparison with a reference forecast to assess whether the forecast presents an improvement or a worsening with respect to that reference. The Mean Bias Skill Score ranges between minus infinite and 1. + Positive values indicate that the forecast has higher skill than the reference forecast, while negative values indicate that it has a lower skill. Examples of reference forecasts are the climatological forecast (average of the observations), + a previous model version, or another model. It is computed as AbsBiasSS = 1 - AbsBias_exp / AbsBias_ref. The statistical significance is obtained based on a Random Walk test at the confidence level specified (DelSole and Tippett, 2016). + If there is more than one dataset, the result will be computed for each pair of exp and obs data." mean_bias_ss_significance: long_name: "Mean Bias Skill Score Statistical Significance" + enssprerr: long_name: "Ensemble Spread-To-Error Ratio" + representation: "Metric" + definition: "The Ensemble Spread-To-Error Ratio (ENS-SPRERR) evaluates the reliability of an ensemble forecasting system by comparing + the ensemble spread (a measure of forecast uncertainty) to the actual forecast error. Ideally, the spread should match the + forecast error, indicating a well-calibrated system. The ratio is computed as ENS-SPRERR = Spread / Error, where Spread is + the standard deviation of the ensemble forecasts, and Error is the mean absolute error of the ensemble mean. A value close + to 1 indicates a reliable system, values less than 1 suggest under-dispersive ensembles (spread too narrow), and values greater + than 1 indicate over-dispersive ensembles (spread too wide). This metric helps assess whether the ensemble system is + appropriately representing uncertainty in the forecasts." + rmsss: long_name: "Root Mean Square Skill Score" + representation: "Skill Score" + definition: "The Root Mean Square Skill Score (RMSSS) evaluates the relative skill of a forecasting system compared to a reference + forecast, such as climatology. It is derived from the Root Mean Square Error (RMSE) of the forecast and the reference. + The formula is RMSSS = 1 - (RMSE_forecast / RMSE_reference). Positive RMSSS values indicate that the forecast is more + skillful than the reference, while negative values imply it is less skillful. This score is widely used in assessing + improvements over baseline methods." + + rps_clim_syear: + long_name: "Ranked Probability Score for Climatology" + representation: "Metric" + definition: "The Ranked Probability Score for Climatology evaluates the accuracy of a forecast relative to the + climatological probabilities for a specific seasonal year. It measures the cumulative squared differences + between the probabilistic forecast and the observed outcomes, weighted across all possible categories." + + rps_syear: + long_name: "Ranked Probability Score" + representation: "Metric" + definition: "The Ranked Probability Score (Seasonal Year) assesses the accuracy of probabilistic forecasts for a + specific seasonal year. It quantifies the squared differences between forecasted cumulative probabilities and the + observed outcome. This score is a measure of probabilistic accuracy over all forecast categories, with lower values + indicating better forecasts." + + crps_clim_syear: + long_name: "Continuous Ranked Probability Score for Climatology" + representation: "Metric" + definition: "The Continuous Ranked Probability Score for Climatology measures the quality of probabilistic + forecasts relative to climatological probabilities for a specific seasonal year. It calculates the difference + between the forecast cumulative distribution and the observed distribution, taking into account all possible + outcomes." + + crps_syear: + long_name: "Continuous Ranked Probability Score" + representation: "Metric" + definition: "The Continuous Ranked Probability Score evaluates the accuracy of probabilistic forecasts + for continuous variables over a specific seasonal year. It measures the difference between the forecasted and observed cumulative + distributions across all possible outcomes. Lower CRPS values signify better forecast performance and reliability, making this a key + metric in ensemble forecasting systems." + + rms: + long_name: "Root Mean Square Error" + representation: "Metric" + definition: "The Root Mean Square Error (RMS) measures the average magnitude of errors between forecasted values and observed values. + It is calculated as the square root of the mean of the squared differences between forecasts and observations. The formula + is RMS = sqrt((1/n) * sum((Forecast_i - Observed_i)^2)), where n is the number of forecasts. RMS provides a comprehensive + measure of accuracy, penalizing larger errors more heavily than smaller ones. Lower RMS values indicate more accurate forecasts." + +calibration: + qmap: + long_name: "Quantile Mapping" + representation: "BiasCorrection" + definition: + bias: + long_name: "Bias Correction (Only)" + representation: "BiasCorrection" + definition: "This method corrects the bias in the system. It does not address variance discrepancies between the system and observation" + + evmos: + long_name: "Error Variance Minimization Objective Score" + representation: "BiasCorrection" + definition: "This method not only corrects the bias but also applies a variance inflation technique to ensure that the corrected + system maintains correspondence with the variance of the observation. It aims to adjust both bias and variance." + mse_min: + long_name: "Mean Squared Error (Minimizing)" + representation: "BiasCorrection" + definition: "This ensemble calibration method aims to correct bias, overall system variance, and ensemble spread. + It minimizes a constrained mean-squared error using three parameters.The goal is to adjust the system to match the observed variance and spread." + crps_min: + long_name: "Continuous Ranked Probability Score (Minimizing)" + representation: "BiasCorrection" + definition: "This ensemble calibration method corrects bias, overall system variance, and ensemble spread. + It does so by minimizing the Continuous Ranked Probability Score (CRPS) using four parameters. + CRPS is a scoring rule that evaluates the probabilistic systems." + rpc-based: + long_name: "Rank Probability Score based" + representation: "BiasCorrection" + definition: "This method adjusts the system variance to ensure that the ratio of predictable components (RPC) is equal to one. + It aims to balance the system variance in relation to the predictability of the system." + +#Downscaling Methods +downscaling: + int: + long_name: Interpolation" + representation: "ESD" + definition: "Regrid of a coarse-scale grid into a finescale grid, or interpolate model data into a point location. Different interpolation methods, + based on different mathematical approaches, can be applied: conservative, nearest neighbor, bilinear or bicubic. Does not rely on any data for training." + intbc: + long_name: "Interpolation with Bias Correction" + representation: "ESD" + definition: "Interpolate model data into a fine-scale grid or point location. Later, a bias adjustment of the interpolated values is performed. + Bias adjustment techniques include simple bias correction, calibration or quantile mapping" + intlr: + long_name: "Interpolation with Linear Regression" + representation: "ESD" + definition: "Interpolate model data into a fine-scale grid or point location. Later, a linear-regression with the interpolated + values is fitted using high-res observations as predictands, and then applied with model data to correct the interpolated values." + logreg: + long_name: "Logistic Regression" + representation: "ESD" + definition: "Relate ensemble mean anomalies of the large-scale forecasts directly to probabilities of observing above normal/normal/below normal conditions at the local scale using a sigmoid function. + It does not produce an ensemble of forecasts but rather their associated probabilities. It is a statistical method with few parameters to train, and only benefits from local information, but it has shown good performance." + + analogs: + long_name: "Analogs" + representation: "ESD" + definition: "Analog interpolation, or Analog Ensemble (AnEn) interpolation, utilizes historical situations resembling current conditions to estimate values at unsampled locations. + It selects a subset of similar historical events (analog situations), assigns weights based on similarity, and computes a weighted average of analog values to interpolate values at the target location, + offering an alternative approach to traditional interpolation methods. This method captures spatial variability and provides uncertainty estimates, particularly beneficial in areas with complex terrain or sparse observational data." \ No newline at end of file diff --git a/example_scripts/test_provenance_1.R b/example_scripts/test_provenance_1.R new file mode 100644 index 0000000000000000000000000000000000000000..4119a278097eefc192da536f9ca88575cf085599 --- /dev/null +++ b/example_scripts/test_provenance_1.R @@ -0,0 +1,35 @@ +source("modules/Loading/Loading.R") +source("modules/Calibration/Calibration.R") +source("modules/Anomalies/Anomalies.R") +source("modules/Skill/Skill.R") +source("modules/Saving/Saving.R") +source("modules/Visualization/Visualization.R") +source("modules/Downscaling/Downscaling.R") +source("tools/prepare_outputs.R") +source("modules/Units/Units.R") +source("modules/Aggregation/Aggregation.R") + +recipe_file <- "recipes/examples/test_provenance_1.yml" + +recipe <- prepare_outputs(recipe_file) + +# Load datasets +data <- Loading(recipe = recipe) + +# Units transformation +data <- Units(recipe = recipe, data = data) + +# Temporal aggregation +data_agg <- Aggregation(recipe = recipe, data = data) + +#Calibration +data_cal <- Calibration(recipe=recipe, data=data_agg) + +#Anomalies +data_ano <- Anomalies(recipe, data_cal) + +#Metrics +skill_metrics <- Skill(recipe, data_ano) + +#Visualization +Visualization(recipe, data, skill_metrics, significance = T) diff --git a/modules/Aggregation/Aggregation.R b/modules/Aggregation/Aggregation.R index 15e4ec39a1e5984fafa9e90d785ea3384195b8ef..7fed4269ed6ebb8fc8da19c3bf72904808bccf1a 100644 --- a/modules/Aggregation/Aggregation.R +++ b/modules/Aggregation/Aggregation.R @@ -10,6 +10,10 @@ source("modules/Aggregation/R/agg_ini_end.R") Aggregation <- function(recipe, data) { + #Provenance graph + graph.prov <- data$graph.prov + data$graph.prov <- NULL + ## TODO: Move checks to recipe checker ncores <- recipe$Analysis$ncores # is it already checked? NULL or number na.rm <- ifelse(is.null(recipe$Analysis$remove_NAs), @@ -58,12 +62,21 @@ Aggregation <- function(recipe, data) { } info(recipe$Run$logger, "##### TIME AGGREGATION COMPLETE #####") + # Generate provenance + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Analysis$Provenance) { + source("provenance/prov_Aggregation.R") + res$graph.prov <- graph.prov + res$graph.prov <- prov_Aggregation(recipe, res) + } + } return(res) } else { warn(recipe$Run$logger, paste("The Aggregation module has been called but parameter", "Time_aggregation:execute is set to 'no' in the recipe.", "The data is returned unchanged.")) + data$graph.prov <- graph.prov return(data) } } diff --git a/modules/Anomalies/Anomalies.R b/modules/Anomalies/Anomalies.R index 1fef3bda28c64533c8b48cee6c8bae365e2fa7a9..b015fadb40eb6fc6e264d7ae17309017383eb010 100644 --- a/modules/Anomalies/Anomalies.R +++ b/modules/Anomalies/Anomalies.R @@ -3,6 +3,10 @@ Anomalies <- function(recipe, data) { + #Provenance graph + graph.prov <- data$graph.prov + data$graph.prov <- NULL + if (is.null(recipe$Analysis$Workflow$Anomalies$compute)) { error(recipe$Run$logger, paste("The anomaly module has been called, but the element", @@ -138,8 +142,19 @@ Anomalies <- function(recipe, data) { ## TODO: Return fcst full value? .log_memory_usage(recipe$Run$logger, "After computing anomalies") - - return(list(hcst = data$hcst, obs = data$obs, fcst = data$fcst, - hcst.full_val = hcst_fullvalue, obs.full_val = obs_fullvalue)) + + #Provenance + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Analysis$Provenance) { + source("provenance/prov_Anomalies.R") + graph.prov <- prov_Anomalies(recipe, graph.prov) + } + return(list(hcst = data$hcst, obs = data$obs, fcst = data$fcst, + hcst.full_val = hcst_fullvalue, obs.full_val = obs_fullvalue, + graph.prov = graph.prov)) + }else { + return(list(hcst = data$hcst, obs = data$obs, fcst = data$fcst, + hcst.full_val = hcst_fullvalue, obs.full_val = obs_fullvalue)) + } } diff --git a/modules/Calibration/Calibration.R b/modules/Calibration/Calibration.R index f1362a22c7608ca95322351d5ba1ea5422792670..18b24c3a7b484f1140dacb1466de88fe23957690 100644 --- a/modules/Calibration/Calibration.R +++ b/modules/Calibration/Calibration.R @@ -6,6 +6,10 @@ Calibration <- function(recipe, data) { # Optionally, it may also have hcst.full_val and obs.full_val. # recipe: object obtained when passing the .yml recipe file to read_yaml() + #Provenance graph + graph.prov <- data$graph.prov + data$graph.prov <- NULL + method <- tolower(recipe$Analysis$Workflow$Calibration$method) if (method == "raw") { @@ -184,10 +188,24 @@ Calibration <- function(recipe, data) { } } - ## TODO: Sort out returns - return_list <- list(hcst = hcst_calibrated, - obs = data$obs, - fcst = fcst_calibrated) + #Provenance + Return outputs + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Analysis$Provenance) { + source("provenance/prov_Calibration.R") + graph.prov <- prov_Calibration(recipe, graph.prov) + return_list <- list(hcst = hcst_calibrated, + obs = data$obs, + fcst = fcst_calibrated, + graph.prov = graph.prov) + } + } else{ + ## TODO: Sort out returns + return_list <- list(hcst = hcst_calibrated, + obs = data$obs, + fcst = fcst_calibrated) + + } + if (!is.null(hcst_full_calibrated)) { return_list <- append(return_list, list(hcst.full_val = hcst_full_calibrated, diff --git a/modules/Downscaling/Downscaling.R b/modules/Downscaling/Downscaling.R index 07c228388c30fd6cab90b9cb8bfb59aa444eb7f2..9681ec8b8f1d0816491b13c271411065192b4c1a 100644 --- a/modules/Downscaling/Downscaling.R +++ b/modules/Downscaling/Downscaling.R @@ -13,6 +13,10 @@ Downscaling <- function(recipe, data) { # data: list of s2dv_cube objects containing the hcst, obs and fcst. # recipe: object obtained when passing the .yml recipe file to read_yaml() + #Provenance graph + graph.prov <- data$graph.prov + data$graph.prov <- NULL + type <- tolower(recipe$Analysis$Workflow$Downscaling$type) fcst_only <- recipe$Analysis$Workflow$Downscaling$fcst_only @@ -451,6 +455,15 @@ Downscaling <- function(recipe, data) { if (recipe$Analysis$Workflow$Downscaling$save == 'all') { save_observations(recipe = recipe, data_cube = data$obs) } - - return(list(hcst = data$hcst, obs = data$obs, fcst = data$fcst)) + + # Generate provenance + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Analysis$Provenance) { + source("provenance/prov_Downscaling.R") + graph.prov <- prov_Downscaling(recipe, graph.prov) + return(list(hcst = data$hcst, obs = data$obs, fcst = data$fcst, graph.prov = data$graph.prov)) + } + }else { + return(list(hcst = data$hcst, obs = data$obs, fcst = data$fcst)) + } } diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R index 8e322ab13280f20161a26642af5f620e1eb3c87d..b37a346adbf0c961f32a1e30ddd0967df7da8be6 100644 --- a/modules/Loading/Loading.R +++ b/modules/Loading/Loading.R @@ -38,5 +38,12 @@ Loading <- function(recipe) { data_summary(data$fcst, recipe) } } + #Provenance METACLIP + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Analysis$Provenance) { + source("provenance/prov_Loading.R") + data$graph.prov <- prov_Loading(recipe, data) + } + } return(data) } diff --git a/modules/Loading/R/subseas_file_dates.R b/modules/Loading/R/subseas_file_dates.R index eca66c5572807abef6cc5496f5154accca380055..d2d6ff776e0ca57a5e3f0888fe5f66aea459fba2 100644 --- a/modules/Loading/R/subseas_file_dates.R +++ b/modules/Loading/R/subseas_file_dates.R @@ -42,7 +42,7 @@ subseas_file_dates <- function(startdate, n.skill.weeks, n.days, ftime_min <- as.numeric(substr(as.character(startdate), 1, 4)) - hcst.end ftime_max <- as.numeric(substr(as.character(startdate), 1, 4)) - hcst.start - startdate <- as.Date(toString(startdate), "%Y%m%d") + startdate <- as.Date(toString(startdate), "%Y%m%d") sdates <- numeric(0) while (length(sdates) < n.skill.weeks){ if (format(startdate, "%a") == "Thu" || format(startdate, "%a") == "Mon") { diff --git a/modules/Skill/Skill.R b/modules/Skill/Skill.R index 7c829c253512e11437bf7f983b99cb04d842af48..369d11f4090e5e1abe30198bd777a26083616de3 100644 --- a/modules/Skill/Skill.R +++ b/modules/Skill/Skill.R @@ -45,6 +45,11 @@ Skill <- function(recipe, data, agg = 'global') { # recipe. Be aware that some metrics, such as the CRPSS may not be # correct.") # } + + #Extract provenance graph + graph.prov <- data$graph.prov + data$graph.prov <- NULL + time_dim <- 'syear' memb_dim <- 'ensemble' metrics <- tolower(recipe$Analysis$Workflow$Skill$metric) @@ -383,6 +388,16 @@ Skill <- function(recipe, data, agg = 'global') { } } } + + #Provenance + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Analysis$Provenance) { + source("provenance/prov_Skill.R") + skill_metrics$graph.prov <- + prov_Skill(recipe, graph.prov) + } + } + # Return results return(skill_metrics) } diff --git a/modules/Units/Units.R b/modules/Units/Units.R index a143c0a2037684594cd7642149fa039240e4784c..c8beb85c0692ec838cf7212d0c87417a786e6d8c 100644 --- a/modules/Units/Units.R +++ b/modules/Units/Units.R @@ -14,6 +14,11 @@ Units <- function(recipe, data) { # from data the original units # deaccumulate option for CDS accumulated variables? ncores <- recipe$Analysis$ncores + + #Provenance graph + graph.prov <- data$graph.prov + data$graph.prov <- NULL + ## Do we need to convert other than ECVs? var_names <- lapply(data, function(x) {x$attrs$Variable$varName}) freq <- recipe$Analysis$Variable$freq @@ -61,6 +66,7 @@ Units <- function(recipe, data) { if (length(unique(c(unlist(orig_units), user_units))) == length(user_units)) { info(recipe$Run$logger, "##### NO UNIT CONVERSION NEEDED #####") res <- data + res$graph.prov <- graph.prov } else { if (recipe$Run$filesystem == 'esarchive' && (sum(!sapply(unique(orig_units), is.null)) != 1)) { @@ -87,6 +93,15 @@ Units <- function(recipe, data) { return(result) }, simplify = TRUE) # instead of lapply to get the named list directly info(recipe$Run$logger, "##### UNIT CONVERSION COMPLETE #####") + + #Provenance + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Analysis$Provenance) { + source("provenance/prov_Units.R") + res$graph.prov <- graph.prov + res$graph.prov <- prov_Units(recipe, res, orig_units) + } + } } return(res) } diff --git a/modules/Visualization/Visualization.R b/modules/Visualization/Visualization.R index 749fe3b64169465fb61738404c8253b0acea01cf..5dcaac2df7ec406cb0630ce5edfa04afc3e22521 100644 --- a/modules/Visualization/Visualization.R +++ b/modules/Visualization/Visualization.R @@ -26,6 +26,9 @@ Visualization <- function(recipe, # s2dv_cube objects # skill_metrics: list of arrays containing the computed skill metrics # significance: Bool. Whether to include significance dots where applicable + + graph.prov <- skill_metrics$graph.prov + skill_metrics$graph.prov <- NULL # Try to set default configuration if not specified by user if (is.null(output_conf) && !is.null(recipe$Analysis$Region$name)) { @@ -272,4 +275,13 @@ Visualization <- function(recipe, info(recipe$Run$logger, paste0("##### PLOT FILES CONVERTED TO ", toupper(extension), " #####")) } + + if (!is.null(recipe$Analysis$Provenance)){ + if (recipe$Analysis$Provenance) { + source("provenance/prov_Visualization.R") + # List all .png files in the directory + files <- list.files(outdir, pattern = "\\.png$", full.names = TRUE) + graph <- prov_Visualization(recipe, graph.prov, files) + } + } } diff --git a/provenance/R/prov_aggregation.R b/provenance/R/prov_aggregation.R new file mode 100644 index 0000000000000000000000000000000000000000..ef293fe632600d363ea94e49e4c887da258112cc --- /dev/null +++ b/provenance/R/prov_aggregation.R @@ -0,0 +1,164 @@ +#' @title Directed metadata graph construction for SUNSET's Aggregation module. +#' @description Generate provenance for the Aggregation operation within a +#' SUNSET workflow. +#' @param recipe Configuration file specifying parameters. +#' @param data s2dv_cube object generated by SUNSET's Loading.R function. +#' @param type Type of data: "fcst" for forecast, "hcst" for hindcast, +#' or "obs" for observational data. +#' @details Function to perform provenance aggregation for a specific dataset type. +# This function updates a provenance graph by defining and adding temporal aggregation steps +# based on a SUNSET recipe and the data aggregated in the Aggregation module. The graph is enriched +# with nodes and edges representing the temporal aggregation process, including metadata for each +# aggregation period and its temporal extent. +#' @references This function is based on the semantics outlined +#' in the Datasource ontology within the Metaclip Framework +#' (available at \url{http://www.metaclip.org/}). +#' @references https://earth.bsc.es/gitlab/es/sunset/-/tree/Dev-Provenance?ref_type=heads +#' Documentation for SUNSET's provenance generation. +#' @import igraph + + +prov_aggregation <- function(recipe, data, graph, type){ + #Define key terms for the graph + i <- switch(type, + "fcst" = "parentnodename_fcst", + "hcst" = "parentnodename_hcst", + "obs" = "parentnodename_obs") + parentnodename <- graph$parentnodename[[i]] + graph <- graph$graph + method <- recipe$Analysis$Workflow$Time_aggregation$method + ini <- recipe$Analysis$Workflow$Time_aggregation$ini + end <- recipe$Analysis$Workflow$Time_aggregation$end + + #Aggregation step + aggregation.nodename <- paste0("Aggregation_", type) + + aggregation_descriptions <- paste0("from month/week ", ini, " to month/week ", end, collapse = ", ") + description <- paste0("Temporal aggregation. \nMethod: ", method, + ". \nPeriods: ", aggregation_descriptions) + + graph <- add_vertices_sunset(graph, + nv = 1, + name = aggregation.nodename, + label = "Aggregation", + className = "ds:Aggregation", + attr = list("dc:description" = description) + ) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parentnodename), + getNodeIndexbyName(graph, aggregation.nodename)), + label = "ds:hadAggregation") + + var <- recipe$Analysis$Variables$name + var.freq <- recipe$Analysis$Variables$freq + time <- data[[type]]$attr$Dates + j <- length(data[[type]]$coords$syear) + + for (i in 1:length(ini)) { + + time_period <- time[(1 + j*(i-1)) : (j*i)] + time_period <- paste(format(time_period, format = "%Y-%m-%d"), collapse = " ") + timeper.nodename <- paste0("TemporalPeriod_", type, "_Aggregation_", i) + description <- paste0("Date values for aggregation of week/mont ", + ini[i], " with month/week ", end[i], ". The aggregation is stored + in this date values") + graph <- add_vertices(graph, + nv = 1, + name = timeper.nodename, + label = "TemporalPeriod", + className = "ds:TemporalPeriod", + description = "Temporal period for each aggregated section", + attr = list("prov:atTime" = time_period, + "dc:description"= description)) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, aggregation.nodename), + getNodeIndexbyName(graph, timeper.nodename)), + label = "ds:hasValidTemporalExtent") + } + + #start.time <- format(data[[type]]$attr$time_bounds$start, format = "%Y-%m-%d") + #end.time <- format(data[[type]]$attr$time_bounds$end, format = "%Y-%m-%d") + #start.time <- paste(start.time, collapse = " ") + #end.time <- paste(end.time, collapse = " ") + + + # if (var.freq == "monthly_mean" || type == "fcst") { + # for (i in 1:length(data[[type]]$coords$syear)){ + # start.time <- append(start.time, format(data[[type]]$attrs$Dates, + # format = "%Y-%m-%d %Z")[[i]]) + # j <- + # ((length(data[[type]]$coords$time) - 1) + # * length(data[[type]]$coords$syear) + i) + # end.time <- append(end.time, format(data[[type]]$attrs$Dates, + # format = "%Y-%m-%d %Z")[[j]]) + # } + # + # freq <- switch(var.freq, + # "monthly_mean" = "Monthly Mean", + # "weekly_mean" = "Weekly Mean") + # + # start.time <- paste(start.time, collapse = " ") + # end.time <- paste(end.time, collapse = " ") + # graph <- add_vertices(graph, + # nv = 1, + # name = timeper.nodename, + # label = "TemporalPeriod", + # className = "ds:TemporalPeriod", + # description = "Encode metadata of Temporal Periods", + # attr = list("prov:startedAtTime" = start.time, + # "prov:endedAtTime" = end.time, + # "ds:hasTimeFrequency" = freq)) + # + # graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, aggregation.nodename), + # getNodeIndexbyName(graph, timeper.nodename)), + # label = "ds:hasValidTemporalExtent") + # + # } else if (var.freq == "weekly_mean" && (type == "hcst" || type == "obs")) { + # sdates <- dates2load(recipe, recipe$Run$logger) + # years <- as.numeric(recipe$Analysis$Time$hcst_end) - as.numeric(recipe$Analysis$Time$hcst_start) + 1 + # time_period <- list() + # for (i in 1:years){ + # year_week_windows <- sdates$hcst[,,i] + # for (j in 1:recipe$Analysis$Time$sweek_window) { + # week_window <- year_week_windows[j,] + # formatted_dates <- format(as.POSIXct(week_window, format="%Y%m%d"), "%Y-%m-%d 09:00:00") + # week_window_dates <- unlist(paste0("(", paste(formatted_dates, collapse = " - "), ")")) + # time_period[[length(time_period) + 1]] <- week_window_dates + # } + # + # } + # + # timeper.nodename <- paste0("TemporalPeriod_", type, "_Aggregation") + # graph <- add_vertices(graph, + # nv = 1, + # name = timeper.nodename, + # label = "TemporalPeriod", + # className = "ds:TemporalPeriod", + # description = "Encode metadata of Temporal Periods", + # attr = list("prov:value" = paste(unlist(time_period), collapse = " "), + # "ds:hasTimeFrequency" = var.freq)) + # + # graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, aggregation.nodename), + # getNodeIndexbyName(graph, timeper.nodename)), + # label = "ds:hasValidTemporalExtent") + # } + + + + #New horizontal extent (Now we only have temporal aggregation) + #var <- unlist(strsplit(recipe$Analysis$Variables$name, + # ",\\s*"))[[1]] + #refgraph <- get_spatial_extent(data, type, var, Aggr = TRUE) + #graph <- my_union_graph(graph, refgraph$graph) + #graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, aggregation.nodename), + # getNodeIndexbyName(graph, refgraph$parentnodename)), + # label = "ds:hasHorizontalExtent") + + return(list("graph" = graph, "parentnodename" = aggregation.nodename)) + +} \ No newline at end of file diff --git a/provenance/R/prov_anomalies.R b/provenance/R/prov_anomalies.R new file mode 100644 index 0000000000000000000000000000000000000000..0bc7bf716c5ccdeb3562fcfa983bba66aab1ac0e --- /dev/null +++ b/provenance/R/prov_anomalies.R @@ -0,0 +1,91 @@ +prov_anomalies <- function(recipe, data, graph, type) { + #Define parentnodename and Anomalies provenance metadata + i <- switch(type, + "fcst" = "parentnodename_fcst", + "hcst" = "parentnodename_hcst", + "obs" = "parentnodename_obs") + parentnodename <- graph$parentnodename[[i]] + graph <- graph$graph + dc.description <- "Anomaly calculation in climatology involves subtracting + a climatological baseline from observed or modeled data + to isolate deviations from the average conditions. + This process, often augmented by cross-validation techniques + , allows for the identification and analysis of unusual or + abnormal climate patterns.For more details, see the CSTools + documentation for CST_Anomaly()" + + referenceURL <- "https://cran.r-project.org/web/packages/CSTools/CSTools.pdf" + + if (recipe$Analysis$Workflow$Anomalies$compute) { + #Add climatology node("obs" and "hcst"), for "fcst" the "hcst" + #climatology node is computed to use it as reference. + if (type == "obs" || type == "hcst") { + clim.nodename <- paste0("Climatology_", type) + + graph <- add_vertices(graph, + nv = 1, + name = clim.nodename, + label = "Climatology", + className = "ds:Climatology") + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parentnodename), + getNodeIndexbyName(graph, clim.nodename)), + label = "ds:hadClimatology") + }else { + clim.nodename <- "Climatology_hcst" + graph <- add_vertices(graph, + nv = 1, + name = clim.nodename, + label = "Climatology", + className = "ds:Climatology" + ) + } + + #Add Anomaly calculation provenance + anom.nodename <- paste0("Anomalies_", type) + + if (recipe$Analysis$Workflow$Anomalies$cross_validation) { + comment <- "Cross-validation computed. Cross-validation enhances + the robustness of anomaly calculations by validating the + chosen baseline against independent datasets, ensuring + greater reliability in assessing climate variability and + change. Anomalies thus derived are valuable for assessing + trends, detecting extremes, and informing decision-making in + various fields such as agriculture, water resource management + , and disaster preparedness." + + graph <- add_vertices(graph, + nv = 1, + name = anom.nodename, + label = "Anomaly", + className = "ds:Anomaly", + attr = list("dc:description" = dc.description, + "dc:comment" = comment, + "ds:referenceURL" = referenceURL)) + + }else{ + graph <- add_vertices(graph, + nv = 1, + name = anom.nodename, + label = "Anomaly", + className = "ds:Anomaly", + attr = list("dc:description" = dc.description, + "ds:referenceURL" = referenceURL + )) + } + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parentnodename), + getNodeIndexbyName(graph, anom.nodename)), + label = "ds:hadAnomalyCalculation") + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, anom.nodename), + getNodeIndexbyName(graph, clim.nodename)), + label = "ds:withReference") + + return(list("graph" = graph, "parentnodename" = anom.nodename)) + }else { + return(list("graph" = graph$graph, "parentnodename" = parentnodename)) + } +} \ No newline at end of file diff --git a/provenance/R/prov_calibration.R b/provenance/R/prov_calibration.R new file mode 100644 index 0000000000000000000000000000000000000000..d596f67af47581f29457c3b5883215163c8b80e3 --- /dev/null +++ b/provenance/R/prov_calibration.R @@ -0,0 +1,72 @@ +prov_calibration <- function(recipe, data, graph, type){ + + #Define Calibration parameters + cal <- tolower(recipe$Analysis$Workflow$Calibration$method) + archive <- read_yaml("conf/variable-dictionary.yml") + + if (cal %in% names(archive$calibration)) { + cal.method <- archive$calibration[[cal]]$long_name + cal.class <- paste0("cal:", archive$calibration[[cal]]$representation) + cal.description <- archive$calibration[[cal]]$definition + }else { + stop("Calibration method not registered or not correctlyy defined") + } + + if (!(cal == "qmap")) { + comment <- "For more details, see the CSTools documentation + for CST_Calibration()" + }else{ + comment <- "For more details, see the CSTools documentation + for CST_QuantileMapping()" + } + referenceURL <- "https://cran.r-project.org/web/packages/CSTools/CSTools.pdf" + reference.nodename <- graph$parentnodename$parentnodename_obs + + #Define last node to which Calibration is applied + if (type == "fcst" || type == "hcst") { + i <- switch(type, + "fcst" = "parentnodename_fcst", + "hcst" = "parentnodename_hcst") + parentnodename <- graph$parentnodename[[i]] + graph <- graph$graph + + #Add Calibration node and vertex + cal.nodename <- paste0("Calibration_", type) + graph <- add_vertices(graph, + nv = 1, + name = cal.nodename, + label = "Calibration", + className = "cal:Calibration") + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parentnodename), + getNodeIndexbyName(graph, cal.nodename)), + label = "cal:hadCalibration") + + #Add Calibration Method + calmethod.nodename <- paste0("CalibrationMethod_", "all") + graph <- add_vertices(graph, + nv = 1, + name = calmethod.nodename, + label = cal.method, + className = cal.class, + attr = list("dc:description" = cal.description, + "dc:comment" = comment, + "ds:referenceURL" = referenceURL)) + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, cal.nodename), + getNodeIndexbyName(graph, calmethod.nodename)), + label = "cal:withCalibrationMethod") + + #Add Reference node + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, cal.nodename), + getNodeIndexbyName(graph, reference.nodename)), + label = "cal:withReferenceData") + + return(list("graph" = graph, "parentnodename" = cal.nodename)) + }else { + return(list("graph" = graph$graph, + "parentnodename" = graph$parentnodename$parentnodename_obs)) + } +} \ No newline at end of file diff --git a/provenance/R/prov_dataset.R b/provenance/R/prov_dataset.R new file mode 100644 index 0000000000000000000000000000000000000000..5eb636851a90730b936fa41703693b92d1e0d9bd --- /dev/null +++ b/provenance/R/prov_dataset.R @@ -0,0 +1,194 @@ +#' @title Directed metadata graph construction for SUNSET's loaded Dataset. +#' @description Generate provenance for the loaded Dataset within a +#' SUNSET workflow. +#' @param recipe Configuration file specifying parameters. +#' @param data s2dv_cube object generated by SUNSET's Loading.R function. +#' @param type Type of data: "fcst" for forecast, "hcst" for hindcast, +#' or "obs" for observational data. +#' @details This function constructs provenance information for the +#' loaded Dataset within a SUNSET workflow.It creates nodes and +#' edges representing various aspects of the Dataset, including +#' its subclass, data provider, modeling center, and project +#' information. +#' @references This function is based on the semantics outlined +#' in the Datasource ontology within the Metaclip Framework +#' (available at \url{http://www.metaclip.org/}). +#' @references https://earth.bsc.es/gitlab/es/sunset/-/tree/Dev-Provenance?ref_type=heads +#' Documentation for SUNSET's provenance generation. # nolint: line_length_linter. +#' @import igraph + +prov_dataset <- function(recipe, data, type) { + #First metadata definitions + horizon <- tolower(recipe$Analysis$Horizon) #'subseasonal','seasonal','decadal' + exp.name <- recipe$Analysis$Datasets$System$name #System name + ref.name <- recipe$Analysis$Datasets$Reference$name #Reference name + + #---Dataset setting (extract metadata from conf/ file) + #Returns a list with the system archive and the reference archive + archive <- get_archive(recipe) + + #Select the system within the conf file + exp_descrip <- archive$System[[exp.name]] + + #Select the reference within the conf file + reference_descrip <- archive$Reference[[ref.name]] + + #Set vocabulary Classes + args.subclass <- NULL + modelling.center <- NULL + dataset.subclass <- 'Dataset' + if (type == "fcst" || type == "hcst") { + if (type == "fcst" && horizon == "seasonal") { + dataset.subclass <- "SeasonalOperationalForecast" + args.subclass <- "Seasonal Forecast" + }else if (type == "hcst" && horizon == "seasonal") { + dataset.subclass <- "SeasonalHindcast" + args.subclass <- "Seasonal Hindcast" + }else if (horizon == "decadal" && type == "fcst") { + args.subclass <- "Forecast decadal simulation, focused on predicting + long-term climate patterns and variability over a 10-year period." + }else if (horizon == "decadal" && type == "hcst") { + args.subclass <- "Hindcast decadal simulation, aimed at reconstructing + historical climate data for the past 10 years to validate + model accuracy." + }else if (horizon == "subseasonal" && type == "fcst") { + args.subclass <- "Forecast subseasonal simulation, targeting short-term + climate predictions over a few weeks to a few months." + }else if (horizon == "subseasonal" && type == "hcst") { + args.subclass <- "Hindcast subseasonal simulation, reconstructing past + weather patterns on a weeks-to-months timescale to improve + model forecasts." + } + modelling.center <- exp_descrip$modelling_center + fullname <- exp_descrip$institution + + if (!is.null(modelling.center)) { + if (modelling.center == "ECMWF") { + modelling.center.nodename <- "ds:ECMWF" + modelling.center <- "ECMWF" + }else { + modelling.center.nodename <- "ModellingCenter" + } + } + }else if (type == "obs") { + dataset.subclass <- "ObservationalDataset" + modelling.center <- reference_descrip$modelling_center + fullname <- reference_descrip$institution + + if (!is.null(modelling.center)) { + #These are individuals defined in the METACLIP datasource ontology + if (modelling.center == "ERA5" || modelling.center == "ERA5-Land") { + dataset.subclass <- "Reanalysis" + modelling.center.nodename <- "ds:ERA5" + modelling.center <- "ERA5" + }else if (modelling.center == "ECMWF") { + modelling.center.nodename <- "ds:ECMWF" + modelling.center <- "ECMWF" + }else { + modelling.center.nodename <- paste0("ModellingCenter_", type) + } + } + } + + # Terms to be included in METACLIP + #-DecadalForecast + #-DecadalHindcast + #-SubseasonalForecast + #-SubseasonalHindcast + + #Create the main graph + graph <- make_empty_graph(0) + members <- length(data[[type]]$coords$ensemble) + + #Add Dataset node + dataset.nodename <- paste0("Dataset_", type) + graph <- add_vertices_sunset(graph, + nv = 1, + name = dataset.nodename, + label = "Dataset", + className = paste0("ds:", dataset.subclass), + attr = list("dc:description" = args.subclass, + "ds:hadMember" = members)) + + #Add Data Provider + #data.provider <- "CDS" + #data.provider.nodename <- "ds:CDS" + #graph <- add_vertices(graph, + # nv = 1, + # name = data.provider.nodename, + # label = data.provider, + # className = "ds:DataProvider") + #graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, dataset.nodename), + # getNodeIndexbyName(graph, dataProvider.nodename)), + # label = "ds:hadDataProvider") + + #Add Modelling Center + if (!is.null(modelling.center)) { + graph <- add_vertices_sunset(graph, + nv = 1, + name = modelling.center.nodename, + label = modelling.center, + className = "ds:ModellingCenter", + attr = list("dc:title" = fullname)) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, dataset.nodename), + getNodeIndexbyName(graph, modelling.center.nodename)), + label = "ds:hadModellingCenter") + } + + #Add Project + #Project.nodename <- paste0("Project_", type) + #graph <- add_vertices(graph, + # nv = 1, + # name = "ds:Copernicus", + # label = "Copernicus", + # className = "ds:Project") + + #graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, dataset.nodename), + # getNodeIndexbyName(graph, "ds:Copernicus")), + # label = "ds:hadProject") + + var_names <- unlist(strsplit(recipe$Analysis$Variables$name, + ",\\s*")) + directory <- list() + i <- 0 + + for (var in var_names) { + #Add main directory + i <- i + 1 + directory_nodename <- paste0("Location_", "Dataset_", type, "_", i) + if (type == "fcst" || type == "hcst") { + directory <- paste0(archive$src, exp_descrip$src, + exp_descrip[[recipe$Analysis$Variables$freq]] + [[var]]) + }else if (type == "obs") { + directory <- paste0(archive$src, reference_descrip$src_ref, + reference_descrip[[recipe$Analysis$Variables$freq]] + [[var]]) + } + + graph <- add_vertices(graph, + nv = 1, + name = directory_nodename, + label = directory, + className = "prov:Location", + attr = list("prov:value" = directory, + "dc:description" = "Barcelona + Supercommputing Center Earth + Science Department + local repository. More information + about the datasets can be found + in SUNSETs + configuration files.", + "prov:atTime" = as.character(Sys.time()))) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, dataset.nodename), + getNodeIndexbyName(graph, directory_nodename)), + label = "prov:atLocation") + } + return(list("graph" = graph, "parentnodename" = dataset.nodename)) +} diff --git a/provenance/R/prov_datasetsubset.R b/provenance/R/prov_datasetsubset.R new file mode 100644 index 0000000000000000000000000000000000000000..893d6ca4604176edeb3270833fb11cf93ea51b7d --- /dev/null +++ b/provenance/R/prov_datasetsubset.R @@ -0,0 +1,208 @@ +#' @title Provenance Construction for DatasetSubset +#' @description Generate provenance for a DatasetSubset within the SUNSET framework. +#' @param recipe Configuration file specifying parameters. +#' @param data s2dv_cube object generated by SUNSET's Loading.R function. +#' @param graph igraph object representing the provenance graph. +#' @return A list containing the updated graph and the name of the parent node. +#' @details This function constructs provenance information for a DatasetSubset within SUNSET. +#' It adds nodes and edges to the provided graph to represent various aspects of the DatasetSubset, +#' including variables, spatial extents, realizations, and temporal periods. +#' @references https://earth.bsc.es/gitlab/es/sunset/-/tree/Dev-Provenance?ref_type=heads +#' Documentation for SUNSET's provenance generation. +#' @references This function is based on the semantics outlined in the Datasource ontology within +#' the Metaclip Framework (available at \url{http://www.metaclip.org/}). +#' @import igraph + +prov_datasetsubset <- function(recipe, data, graph) { + + #First metadata definitions + ref.name <- recipe$Analysis$Datasets$Reference$name #Reference name + exp.name <- recipe$Analysis$Datasets$System$name #System name + archive <- get_archive(recipe) + reference_descrip <- archive$Reference[[ref.name]] + exp_descrip <- archive$System[[exp.name]] + + #Extract the type from previous nodename + parent.node <- graph$parentnodename + type <- strsplit(parent.node, "_")[[1]][[2]] + + #Add DatasetSubset node + graph <- graph$graph + datasetsubset.nodename <- paste0("DatasetSubset_", type) + graph <- add_vertices(graph, + nv = 1, + name = datasetsubset.nodename, + label = "DatasetSubset", + className = "ds:DatasetSubset") + + # Link with existing parent node + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parent.node), + getNodeIndexbyName(graph, datasetsubset.nodename)), + label = paste0("ds:hadDatasetSubset")) + + #Add files directory + directory.nodename <- paste0("Location_", "Datasubset_", type) + + #Extract all directories + source_files <- data[[type]]$attr$source_files + directory <- paste(source_files, collapse = " ") + + graph <- add_vertices(graph, + nv = 1, + name = directory.nodename, + label = paste0(type, " ", "directories"), + className = "prov:Location", + attr = list("prov:value" = directory, + "dc:description" = "Barcelona + Supercommputing Center Earth + Science Department + local repository. More information + about the datasets can be found + at SUNSETs + configuration files.", + "prov:atTime" = as.character(Sys.time()))) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, datasetsubset.nodename), + getNodeIndexbyName(graph, directory.nodename)), + label = "prov:atLocation") + + # Add variable metadata + vars <- data[[type]]$attr$Variable$varName + var.freq <- recipe$Analysis$Variables$freq + + for (var in vars) { + units <- data[[type]]$attr$Variable[[var]]$units + long.name <- data[[type]]$attr$Variable[[var]]$long_name + var.nodename <- paste0("Variable_", var, "_", type) + graph <- add_vertices(graph, + nv = 1, + name = var.nodename, + label = var, + className = "ds:Variable", + description = "Encode metadata of a Variable", + attr = list("ds:withUnits" = units, + "ds:hasLongName" = long.name, + "ds:hasTimeFrequency" = var.freq)) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, datasetsubset.nodename), + getNodeIndexbyName(graph, var.nodename)), + label = "ds:hasVariable") + } + + #Add SpatialExtent (Horizontal) node + var <- unlist(strsplit(recipe$Analysis$Variables$name, + ",\\s*"))[[1]] + refgraph <- get_spatial_extent(data, type, var) + graph <- my_union_graph(graph, refgraph$graph) + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, datasetsubset.nodename), + getNodeIndexbyName(graph, refgraph$parentnodename)), + label = "ds:hasHorizontalExtent") + + #TODO: + #Add SpatialExtent (Vertical) node + #vextent.nodename <- paste0("VerticalExtent_", type) + #graph <- add_vertices(graph, + # nv = 1, + # name = vextent.nodename, + # label = "VerticalExtent", + # className = "ds:VerticalExtent") + + #graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, datasetsubset.nodename), + # getNodeIndexbyName(graph, vextent.nodename)), + # label = "ds:hasVerticalExtent") + + # Add members node + #if (type == "fcst" || type == "hcst") { + # browser() + # for(i in 1:length(as.numeric(data[[type]]$coords$ensemble))){ + # mem <- paste0("Member_", type, "_", i) + # mem.nodename <- paste0("Realization_", type, "_", i) + # mem <- exp_descrip$member[[i]] + # graph <- add_vertices(graph, + # nv = 1, + # name = mem.nodename, + # label = , + # className = "ds:Realization", + # description = "Encode metadata of Realizations", + # attr = list("ds:hasRun" = )) + + # graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, datasetsubset.nodename), + # getNodeIndexbyName(graph, mem.nodename)), + # label = "ds:hasRealization") + # } + # } + + #Add TemporalPeriod and Temporal Resolution + start.time <- list() + end.time <- list() + time.step <- list() + + if (var.freq == "monthly_mean" || type == "fcst") { + for (i in 1:length(data[[type]]$coords$syear)){ + start.time <- append(start.time, format(data[[type]]$attrs$Dates, + format = "%Y-%m-%d %Z")[[i]]) + j <- + ((length(data[[type]]$coords$time) - 1) + * length(data[[type]]$coords$syear) + i) + end.time <- append(end.time, format(data[[type]]$attrs$Dates, + format = "%Y-%m-%d %Z")[[j]]) + } + + freq <- switch(var.freq, + "monthly_mean" = "Monthly Mean", + "weekly_mean" = "Weekly Mean") #ADD "daily_mean" and "daiy", what happened with "weekly_mean"? + + timeper.nodename <- paste0("TemporalPeriod_", type) + start.time <- paste(start.time, collapse = " ") + end.time <- paste(end.time, collapse = " ") + graph <- add_vertices(graph, + nv = 1, + name = timeper.nodename, + label = "TemporalPeriod", + className = "ds:TemporalPeriod", + description = "Encode metadata of Temporal Periods", + attr = list("prov:startedAtTime" = start.time, + "prov:endedAtTime" = end.time, + "ds:hasTimeFrequency" = freq)) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, datasetsubset.nodename), + getNodeIndexbyName(graph, timeper.nodename)), + label = "ds:hasValidTemporalExtent") + + } else if (var.freq == "weekly_mean" && type == "hcst" || type == "obs") { + sdates <- dates2load(recipe, recipe$Run$logger) + years <- as.numeric(recipe$Analysis$Time$hcst_end) - as.numeric(recipe$Analysis$Time$hcst_start) + 1 + time_period <- list() + for (i in 1:years){ + year_week_windows <- sdates$hcst[,,i] + for (j in 1:recipe$Analysis$Time$sweek_window) { + week_window <- year_week_windows[j,] + formatted_dates <- format(as.POSIXct(week_window, format="%Y%m%d"), "%Y-%m-%d 09:00:00") + week_window_dates <- unlist(paste0("(", paste(formatted_dates, collapse = " - "), ")")) + time_period[[length(time_period) + 1]] <- week_window_dates + } + } + timeper.nodename <- paste0("TemporalPeriod_", type) + graph <- add_vertices(graph, + nv = 1, + name = timeper.nodename, + label = "TemporalPeriod", + className = "ds:TemporalPeriod", + description = "Encode metadata of Temporal Periods", + attr = list("prov:value" = paste(unlist(time_period), collapse = " "), + "ds:hasTimeFrequency" = var.freq)) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, datasetsubset.nodename), + getNodeIndexbyName(graph, timeper.nodename)), + label = "ds:hasValidTemporalExtent") + } + return(list("graph" = graph, "parentnodename" = datasetsubset.nodename)) +} diff --git a/provenance/R/prov_helpers.R b/provenance/R/prov_helpers.R new file mode 100644 index 0000000000000000000000000000000000000000..97786824f4769df5855c55d69b096d705550186a --- /dev/null +++ b/provenance/R/prov_helpers.R @@ -0,0 +1,689 @@ +#' @title Add vertices to a node in the igraph object +#' @description Add vertices to a node in the igraph object +#' @return Completed graph + +add_vertices_sunset <- function(graph, + name = NULL, + nv = 1, + label = NULL, + className = NULL, + attr = NULL) { + if (class(graph) != "igraph") stop("The input graph has not a valid format") + if (is.null(name)) stop("The 'name' attribute is required", call. = FALSE) + if (!name %in% vertex_attr(graph, name = "name")) { + if (is.null(className)) + stop("The 'className' attribute is required", call. = FALSE) + if (is.null(label)) stop("The 'label' attribute is required", call. = FALSE) + if (is.null(attr)) { + graph <- add_vertices(graph, + nv = nv, + name = name, + label = label, + className = className) + } else { + graph <- add_vertices(graph, + nv = nv, + name = name, + label = label, + className = className, + attr = attr) + } + } + return(graph) +} + +embedJSON <- function(png.in, json.file, png.out) { + txt <- scan(json.file, + what = character(), strip.white = TRUE, + quote = "'", blank.lines.skip = TRUE) + metadata <- paste(txt, collapse = " ") %>% charToRaw() %>% memCompress() %>% as.character() %>% paste(collapse = "") + metadata <- c(metaclip = metadata) + img <- png::readPNG(source = png.in) + # Embed compressed metadata + png::writePNG(image = img, + text = metadata, + target = png.out) +} + +#' @title Generate provenance for Spatial Extent definition +#' @description Generate provenance information of the spatial coordinates of a climate data object in s2dv_cube format. Returns a graph containing a single SpatialExtent node. +#' @param data The dataset containing spatial coordinates. +#' @param type The type of dataset (fcst, hcst, obs) +#' @param Reg Logical indicating if the operation is for regriddinging. +#' @param Cal Logical indicating if the operation is for calibration. +#' @param Downsc Logical indicating if the operation is for downscaling. +#' @return A list containing the graph and the parent node name. +#' @import igraph +#' @examples +#' # Example usage: +#' getSpatialExtent(data, type, Reg = TRUE) +#' getSpatialExtent(data, type, Cal = TRUE) +#' getSpatialExtent(data, type, Downsc = TRUE) +#' getSpatialExtent(data, type) + +get_spatial_extent <- function(data, type, var, Reg = FALSE, Cal = FALSE, Downsc = FALSE, Aggr = FALSE) { + + graph <- make_empty_graph(0) + + + + # Get maximum and minimum Latitude values + ymax <- max(as.numeric(data[[type]]$coords$lat)) + ymin <- min(as.numeric(data[[type]]$coords$lat)) + + # Get maximum and minimum Longitude values + xmax <- max(as.numeric(data[[type]]$coords$lon)) + xmin <- min(as.numeric(data[[type]]$coords$lon)) + + # Calculate Latitude resolution and get units + resY <- abs(as.numeric(data[[type]]$coords$lon)[2] - as.numeric(data[[type]]$coords$lon)[1]) + yunits <- data[[type]]$attrs$Variable$metadata[[var]]$dim[[1]]$units + + # Calculate Longitude resolution and get units + resX <- abs(as.numeric(data[[type]]$coords$lat)[2] - as.numeric(data[[type]]$coords$lat)[1]) + xunits <- data[[type]]$attrs$Variable$metadata[[var]]$dim[[2]]$units + + + #Time Resolution + #tunits <- data[[type]]$attrs$Variable$metadata[[var]]$dim[[3]]$units + + if(Reg){ + spatextent.nodename <- paste0("SpatialExtent_", "Reggriding_", type, "_", var) + }else if(Cal){ + spatextent.nodename <- paste0("SpatialExtent_", "Calibration_", type, "_", var) + }else if(Downsc){ + spatextent.nodename <- paste0("SpatialExtent_", "Downscaling_", type, "_", var) + }else if(Aggr){ + spatextent.nodename <- paste0("SpatialExtent_", "Aggregation_", type, "_", var) + }else{ + spatextent.nodename <- paste0("SpatialExtent_", "DatasetSubset_", type, "_", var) + } + graph <- + add_vertices_sunset( + graph, + nv = 1, + name = spatextent.nodename, + label = "SpatialExtent", + className = "ds:HorizontalExtent", + attr = + list( + "ds:xmin" = xmin, + "ds:xmax" = xmax, + "ds:ymin" = ymin, + "ds:ymax" = ymax, + "ds:hasHorizontalResX" = resX, + "ds:hasHorizontalResY" = resY, + "ds:hasProjection:" = NULL, + "ds:withUnits" = + paste0( + paste0("X:", xunits, sep = " "), " ", + paste0("Y:", yunits, sep = " "), " "))) + return(list("graph" = graph, "parentnodename" = spatextent.nodename)) +} + +my_union_graph <- function(...) { + graph.list <- list(...) + graph <- graph.list[[1]] + for (i in 2:length(graph.list)) { + int <- intersect(vertex_attr(graph, name = "name"), vertex_attr(graph.list[[i]], name = "name")) %>% length() + if (int == 0L) { + graph <- disjoint_union(graph, graph.list[[i]]) + } else { + g <- igraph::union(graph, graph.list[[i]]) + graph <- my_union(g, graph, graph.list[[i]]) + } + } + invisible(graph) +} + + +#' @title Attribute-preserving igraph union +#' @description Transforms the resulting output from igraph::union to adequately recover the original attributes. +#' Apart from vertex attributes, the "label" attribute from edges, used to define object properties, is fixed. +#' @param g The union graph +#' @param g1 First input graph +#' @param g2 Second input graph +#' @return Invisible returns a new graph with fixed attributes +#' @author J. Bedia +#' @keywords internal +#' @export +#' @importFrom magrittr %>% +#' @importFrom igraph vertex_attr vertex_attr_names delete_vertex_attr get.edgelist edge_attr delete_edges add_edges set_vertex_attr +#' @importFrom stats na.exclude + +my_union <- function(g, g1, g2) { + # Vertex attr adjustment + ind.vertex <- match(vertex_attr(g, "name"), c(vertex_attr(g1, "name"), vertex_attr(g2, "name"))) + duplicated.names <- intersect(vertex_attr_names(g1), vertex_attr_names(g2)) + duplicated.names <- duplicated.names[-match("name", duplicated.names)] + if (length(duplicated.names) > 0) { + for (i in 1:length(duplicated.names)) { + all.vnames <- c(vertex_attr(g1, duplicated.names[i]), vertex_attr(g2, duplicated.names[i])) + g <- set_vertex_attr(graph = g, name = duplicated.names[i], value = all.vnames[ind.vertex]) + rmv <- grep(paste0(duplicated.names[i], "_[:1-2:]$"), vertex_attr_names(g), value = TRUE) + g <- delete_vertex_attr(g, rmv[1]) + g <- delete_vertex_attr(g, rmv[2]) + } + # Edge label adjustment + if (!is.null(edge_attr(g, "label_1"))) { + edgelist <- get.edgelist(g) + edgelabels <- edge_attr(g, "label_2") + ind <- which(is.na(edgelabels)) + if (length(ind) > 0) { + edgelabels[ind] <- edge_attr(g, "label_1")[ind] + } + g <- delete_edges(g, E(g)) + for (i in 1:nrow(edgelist)) { + g <- add_edges(g, + c(getNodeIndexbyName(g, edgelist[i,1]), + getNodeIndexbyName(g, edgelist[i,2])), + label = edgelabels[i]) + } + } + } + invisible(g) +} + +#' @title RDF graph serialization to JSON-LD (metaclipR) +#' @description Takes an igraph-class RDF graph and write it in JSON-LD +#' @param graph An i-graph class graph +#' @param output.file Character string. Output path +#' @return A JSON-LD representation of the metadata structure +#' @importFrom igraph V vertex_attr E edge_attr head_of is_igraph +#' @export +#' @family graphical.outputs +#' @author J Bedia +#' @references A useful app to check/test JSON-LD mark + + +graph2json <- function(graph, output.file) { + if (!is_igraph(graph)) stop("'graph' is not an igraph-class object") + f <- output.file + g <- graph + z <- file(f, "w") + cat(c("{","\n"), sep = "", file = z) + cat(c("\t\"@context\": {", "\n"), sep = "", file = z) + # Metaclip imports + cat(c("\t\t\"ds\": ", "\"http://www.metaclip.org/datasource/datasource.owl#\",\n"), + sep = "", file = z) + cat(c("\t\t\"ipcc\": ", "\"http://www.metaclip.org/ipcc_terms/ipcc_terms.owl#\",\n"), + sep = "", file = z) + cat(c("\t\t\"veri\": ", "\"http://www.metaclip.org/verification/verification.owl#\",\n"), + sep = "", file = z) + cat(c("\t\t\"cal\": ", "\"http://www.metaclip.org/calibration/calibration.owl#\",\n"), + sep = "", file = z) + cat(c("\t\t\"go\": ", "\"http://www.metaclip.org/graphical_output/graphical_output.owl#\",\n"), + sep = "", file = z) + # prov-o + cat(c("\t\t\"prov\": ", "\"http://www.w3.org/ns/prov#\",\n"), + sep = "", file = z) + # rdf schema + cat(c("\t\t\"rdfs\": ", "\"http://www.w3.org/2000/01/rdf-schema#\",\n"), + sep = "", file = z) + # dublin core + cat(c("\t\t\"dc\": ", "\"http://www.w3.org/2002/07/owl\",\n"), + sep = "", file = z) + # skos + cat(c("\t\t\"skos\": ", "\"http://www.w3.org/2004/02/skos/core#\"\n"), + sep = "", file = z) + cat("\t},", file = z) + cat("\n\t\"@graph\":[", file = z) + # Todos los vertices del grafo + vertices <- V(g) + # Counter de las clases ya descritas + describedNodeNames <- c() + firstParentNode = TRUE + for (i in 1:length(vertices)) { + vid <- vertex_attr(g, name = "name", index = vertices[i]) + if (!vid %in% describedNodeNames) { + if (!firstParentNode) { + cat("\n,", file = z) + } + cat("{", file = z) + describedNodeNames <- serializeVertex(g, vertex = vertices[i], describedNodeNames, connection = z) + cat("}", file = z) + firstParentNode = FALSE + } + } + cat("]}", file = z) + close(z) +} + + +#' @title Recursive function for graph serialization (metaclipR) +#' @description Recursive function to trace and encode properties of nodes. From metaclipR package +#' @keywords internal +#' @importFrom Rgraphviz from + +serializeVertex <- function(g, vertex, describedNodeNames, connection) { + z <- connection + cat("\n", file = z) + vid <- vertex_attr(g, name = "name", index = vertex) + describedNodeNames <- c(describedNodeNames, vid) + vclass <- vertex_attr(g, name = "className", index = vertex) + label <- vertex_attr(g, name = "label", index = vertex) + node.index <- getNodeIndexbyName(g, vid) + if (isIndividualInstance(vid)) { + cat("\t\"@id\": ", paste0("\"", vid, "\",\n"), file = z) + } else { + cat("\t\"@id\": ", paste0("\"#", vid, "\",\n"), file = z) + } + if (!is.null(vclass)) cat(paste0("\t\"@type\": \"", vclass, "\",\n"), file = z) + if (!is.null(label)) cat(paste0("\t\"rdfs:label\": \"", label, "\""), file = z) + # Propiedades planas + pplist <- getPlainPropertyList(g, node.index) + hasplainproperties <- FALSE + if (length(pplist) > 0) { + hasplainproperties <- TRUE + cat(",\n", file = z) + for (j in 1:length(pplist)) { + cat(paste0("\"", names(pplist)[j], "\":"), paste0("\"", pplist[[j]], "\""), file = z) + if (j < length(pplist)) { + cat(",\n", file = z) + } + } + } + # Arcos que parten de este nodo: E(g)[from(getNodeIndexbyName(g, vid))] + # full.list <- vertex_attr(g, index = vertices[i]) + arcs <- E(g)[from(node.index)] + if (length(arcs) > 0) { # El nodo tiene hijos + cat("\n", file = z) + classproperties <- edge_attr(g, name = "label", arcs) + for (j in 1:length(unique(classproperties))) { + currentProperty <- unique(classproperties)[j] + # numero de hijos con esa propiedad + nchildren <- sum(classproperties %in% currentProperty) + ind.arc <- which(classproperties %in% currentProperty) + cat(paste0(",\n\"", currentProperty, "\": "), file = z) + if (nchildren == 1) { + cat("{", file = z) + child.node <- head_of(g, arcs[ind.arc]) + child.node.name <- vertex_attr(g, name = "name", index = child.node) + # Si child.node ya está en describedNodeNames, skip serialize and point to identifier + # + if (!child.node.name %in% describedNodeNames) { + describedNodeNames <- serializeVertex(g, vertex = child.node, describedNodeNames, z) + } else { + if (isIndividualInstance(child.node.name)) { + cat("\t\"@id\": ", paste0("\"", child.node.name, "\"\n"), file = z) + } else { + cat("\t\"@id\": ", paste0("\"#", child.node.name, "\"\n"), file = z) + } + } + cat("}", file = z) + } else { + cat("[", file = z) + for (k in 1:nchildren) { + child.node <- head_of(g, arcs[ind.arc[k]]) + child.node.name <- vertex_attr(g, name = "name", index = child.node) + cat("{", file = z) + # Si child.node ya está en describedNodeNames, skip serialize and point to identifier + # + if (!child.node.name %in% describedNodeNames) { + describedNodeNames <- serializeVertex(g, child.node, describedNodeNames, z) + } else { + if (isIndividualInstance(child.node.name)) { + cat("\t\"@id\": ", paste0("\"", child.node.name, "\"\n"), file = z) + } else { + cat("\t\"@id\": ", paste0("\"#", child.node.name, "\"\n"), file = z) + } + } + cat("}", file = z) + if (k < nchildren) cat(",", file = z) + } + cat("]", file = z) + } + } + } + return(describedNodeNames) +} + + +#' @title Individual identification (metaclipR) +#' @description Is the node representing and individual instance (\code{TRUE}), or a +#' generic class entity (\code{FALSE})? +#' @return A Logical flag +#' @keywords internal +#' @author J Bedia + +isIndividualInstance <- function(node.name) grepl("^ds\\:|^veri\\:|^go\\:|^cal\\:|^ipcc\\:", node.name) + + +#' @title Get node indices (metaclipR) +#' @description Get the node index positions from their names +#' @param graph \code{\link[igraph]{igraph}} graph +#' @param names Character vector of names of the nodes whose index is queried +#' @family graph.helpers +#' @export +#' @importFrom igraph V make_graph +#' @author J Bedia +#' @return A vector of index positions corresponding to the input names +#' @examples +#' require(igraph) +#' graph <- make_graph(c("A", "B", "B", "C", "C", "D", "Pepe"), directed = FALSE) +#' getNodeIndexbyName(graph, "Pepe") +#' getNodeIndexbyName(graph, c("A","D","Pepe")) + +getNodeIndexbyName <- function(graph, names) { + out <- sapply(1:length(names), function(x) which(V(graph)$name == names[x])) + names(out) <- names + return(out) +} + +#' @title Return all node attributes (metaclipR) +#' @description Return a list of all (non-empty) node attributes +#' @param graph \code{\link[igraph]{igraph}} graph +#' @param vertex.name Character string, with the target vertex name +#' @param vertex.index Alternatively, instead of \code{vertex.name}, an integer +#' indicating the index position of the vertex in the graph. +#' @family graph.helpers +#' @export +#' @seealso \code{\link{getNodeIndexbyName}} +#' @importFrom igraph vertex_attr +#' @author J Bedia +#' @return A list of (non-empty) attributes belonging to the vertex + +getNodeAttributes <- function(graph, vertex.index = NULL, vertex.name = NULL) { + if (is.null(vertex.index)) { + if (is.null(vertex.name)) { + stop("Either a valid 'vertex.index' or 'vertex.name' value is required", call. = FALSE) + } else { + vertex.index <- getNodeIndexbyName(graph, vertex.name) + } + } + lista <- vertex_attr(graph, index = vertex.index) + non.empty.id <- which(sapply(1:length(lista), function(x) !is.na(lista[[x]]))) + return(lista[non.empty.id]) +} + + +#' @title Return most node attributes (metaclipR) +#' @description Return almost all node attributes, but the name, classname, etc... +#' @param g \code{\link[igraph]{igraph}} graph +#' @param node.index An integer indicating the index position of the vertex in the graph. +#' @family graph.helpers +#' @keywords internal +#' @importFrom igraph vertex_attr +#' @author J Bedia +#' @return A list of (non-empty) attributes belonging to the vertex (lists are vectorized) + + +getPlainPropertyList <- function(g, node.index) { + attr.list <- vertex_attr(g, index = node.index) + # Remove already known attrs + ind <- which(!(names(attr.list) %in% c("name", "label", "className", "description"))) + attr.list.s <- attr.list[ind] + plain.property.list <- attr.list.s[which(!is.na(unlist(attr.list.s)))] + return(plain.property.list) +} + + + +#' @importFrom utils read.csv +#' @keywords internal + +pkgVersionCheck <- function(pkg, version) { + pkgv <- package_version(version) + ref <- file.path(find.package(package = "metaclipR"), "pkg_versions.csv") %>% read.csv(stringsAsFactors = FALSE) + if (!pkg %in% ref$package) { + message("NOTE: The specified package is not a known ds:Package. Package version info will be annotated but not checked") + } else { + v <- ref[match(pkg, ref$package), 2:3] + if (pkgv >= package_version(v[1]) && pkgv <= package_version(v[2])) { + message("Valid package version") + } else { + warning("The package version provided is not guaranteed to be properly handled by metaclipR\nType 'knownPackageVersions()' for details.") + } + } +} + + + +#' @title List of valid package versions for metaclipR +#' @description Print a table with the valid version range of known packages. For the listed packages and version ranges, +#' metaclipR fuctions are guaranteed to correctly map arguments/functions to the corresponding METACLIP classes/properties. +#' @importFrom utils read.csv +#' @importFrom magrittr %>% +#' @export + +knownPackageVersions <- function() find.package(package = "metaclipR") %>% file.path("pkg_versions.csv") %>% read.csv(stringsAsFactors = FALSE) %>% print() + +#' @title Set a node name +#' @description Set a node name after checking whether it is an individual instance or not +#' @export +#' @param node.name Proposed node name. This ca be an individual instance +#' @param node.class Class of the node +#' @param vocabulary Vocabulary defining the \code{node.class}. Default to \code{"datasource"}. +#' @return A character string to be set as the \code{"name"} attribute by \code{\link{my_add_vertices}} + +setNodeName <- function(node.name, node.class, vocabulary = "datasource") { + suppressWarnings( + ifelse(node.name %in% suppressMessages(knownClassIndividuals(node.class, vocabulary = vocabulary)), + paste0("ds:", node.name), + paste0(node.name, ".", randomName())) + ) +} + +#' @title Generate a random character string +#' @description Generates a random character string (lowercase) of a specified length. +#' Used to avoid duplicated node names in igraph +#' @param len length of the character string +#' @export +#' @keywords internal +#' @author J Bedia +#' @return A string with the specified number of characters +#' @importFrom magrittr %>% extract +#' @examples +#' randomName() +#' randomName(10) + +randomName <- function(len = 6) { + ind <- sample(1:length(letters)) %>% extract(1:len) + letters[ind] %>% paste(collapse = "") +} + + +#' @title Table of User Data Gateway PUBLIC datasets +#' @description Show the internal table with known Datasets and relevant metadata +#' @export +#' @author J Bedia +#' @return A \code{data.frame} +#' @export +#' @importFrom utils read.csv +#' @examples showUDGDatasources()[c(2,35,60,81),] + +showUDGDatasources <- function() { + read.csv(file.path(find.package("metaclipR"), "datasource.csv"), + stringsAsFactors = FALSE, na.strings = "") +} + + +#' @title List all individuals from a class +#' @description List defined individuals pertaining to a specific class from a METACLIP vocabulary +#' @param classname The parent class from which the individual instances are queried +#' @param vocabulary The target vocabulary name. Possible values are \code{"datasource"} (the default), +#' \code{"calibration"}, \code{"verification"}, \code{"graphical_output"} and \code{"ipcc_terms"}. +#' @param source.vocab Name of the vocabulary the parent class of the individuals belong to. By default, this is the same as the +#' one provided in \code{vocabulary}. +#' @param verbose If set to TRUE, it will print on screen details about the open connections to the remote vocabularies. +#' For testing purposes only. +#' @importFrom utils URLencode +#' @importFrom magrittr %>% +#' @details The function will check the existing individuals in the latest stable target +#' ontology release +#' @return A character vector of individuals for that class +#' @note The function won't work if there is no internet connection, or any other connection problem prevents +#' read access to the ontology file. +#' @export +#' @author J Bedia, D. San-Martín +#' @family ontology.helpers +#' @examples +#' knownClassIndividuals("ModellingCenter") +#' knownClassIndividuals("ETCCDI") +#' # In case a class is misspelled or it has no individuals, +#' # an empty vector is returned, with a warning: +#' knownClassIndividuals("ETCDDI") +#' knownClassIndividuals("Dataset") + +knownClassIndividuals <- function(classname, vocabulary = "datasource", source.vocab = NULL, verbose = FALSE) { + vocabulary <- match.arg(vocabulary, choices = c("datasource", + "calibration", + "verification", + "graphical_output", + "ipcc_terms")) + if (is.null(source.vocab)) source.vocab <- vocabulary + stopifnot(is.logical(verbose)) + refURL <- paste0("http://www.metaclip.org/individuals?vocab=", vocabulary, "&sourceVocab=", source.vocab, "&class=") + message("Reading remote ", vocabulary, " ontology file ...") + destURL <- paste0(refURL, classname) %>% URLencode() %>% url() + on.exit(close(destURL)) + if (isTRUE(verbose)) message(print(destURL)) + out <- tryCatch(suppressWarnings(readLines(destURL, warn = FALSE)), error = function(er) { + er <- NULL + return(er) + }) + if (!is.null(out)) { + a <- gsub("\"|\\[|]", "", out) %>% strsplit(split = ",") %>% unlist() + if (length(a) == 0) warning("No individuals found:\nEither the class does not exist or there are no individuals associated to it") + return(a) + } else { + message("Unreachable remote vocabulary\nLikely reason: unavailable internet connection") + } +} + +knownClass <- function(classname, vocabulary = "datasource", source.vocab = NULL, verbose = FALSE) { + # Ensure the vocabulary argument is one of the predefined choices + vocabulary <- match.arg(vocabulary, choices = c("datasource", + "calibration", + "verification", + "graphical_output", + "ipcc_terms")) + browser() + # If source.vocab is NULL, default to the value of vocabulary + if (is.null(source.vocab)) source.vocab <- vocabulary + + # Ensure that the verbose argument is a logical value + stopifnot(is.logical(verbose)) + + # Construct the base URL for fetching MetaCLIP class data + refURL <- paste0("http://www.metaclip.org/", vocabulary, "/", source.vocab, ".owl#") + + # Append the class name to the URL and encode it properly + destURL <- URLencode(paste0(refURL, classname)) + destURL <- url(destURL) + + # Ensure the connection to the URL is closed when the function exits + on.exit(close(destURL)) + + # Print the constructed URL if verbose is TRUE + if (isTRUE(verbose)) message(print(destURL)) + + # Attempt to read the content from the URL + out <- tryCatch(suppressWarnings(readLines(destURL, warn = FALSE)), error = function(er) { + er <- NULL + return(er) + }) + + # If content was successfully read, process the output + if (!is.null(out)) { + # Clean the output, removing quotes and brackets, and split it into class components + class_info <- gsub("\"|\\[|]", "", out) %>% strsplit(split = ",") %>% unlist() + + # If no class information was found, issue a warning + if (length(class_info) == 0) { + warning("Class not found in the METACLIP vocabularies.:\nEither the class does not exist or there are no classes associated with it.") + } + + # Return the class information + return(class_info) + + } else { + # If the URL couldn't be reached, show an appropriate message + message("Unreachable remote vocabulary\nLikely reason: unavailable internet connection or invalid class.") + } +} + + +#' @title Identify the class belonging of an individual +#' @description Identifies the class belonging of an individual. Superclasses are ignored +#' @param individual.name Name of the individual +#' @param vocabulary The target vocabulary name. Possible values are \code{"datasource"} (the default), +#' \code{"calibration"}, \code{"verification"} and \code{"graphical_output"}. +#' @importFrom utils URLencode +#' @importFrom magrittr %>% extract2 +#' @details The function will check the existing individuals in the latest stable target +#' ontology release +#' @return A character string with the most direct class belonging (superclasses are not indicated) +#' @note The function won't work if there is no internet connection, or any other connection problem prevents +#' read access to the ontology file. +#' @export +#' @author J Bedia, D. San-Martín +#' @family ontology.helpers +#' @examples +#' getIndividualClass("UDG") # The default vocabulary is datasource (can be omitted) +#' getIndividualClass("EQM", vocabulary = "calibration") + +getIndividualClass <- function(individual.name, vocabulary = "datasource") { + vocabulary <- match.arg(vocabulary, choices = c("datasource", + "calibration", + "verification", + "graphical_output", + "ipcc_terms")) + refURL <- paste0("http://www.metaclip.org/individual?vocab=", vocabulary, "&id=") + message("Reading remote ", vocabulary, " ontology file ...") + destURL <- paste0(refURL, individual.name) %>% URLencode() %>% url() + on.exit(close(destURL)) + out <- tryCatch(suppressWarnings(readLines(destURL, warn = FALSE)), error = function(er) { + er <- NULL + return(er) + }) + if (!is.null(out)) { + aux <- gsub("\"|\\[|]", "", out) + if (aux == "") { + a <- NULL + } else { + a <- aux %>% strsplit(split = ",") %>% unlist() %>% extract2(1) %>% gsub(pattern = "http://www.metaclip.org/.*\\.owl#", replacement = "") + if (length(a) == 0) warning("No class found:\nEither the individual does not exist or there are no associated classes to it") + } + return(a) + } else { + message("Unreachable remote vocabulary\nLikely reason: unavailable internet connection") + } +} + + +#' @title Encapsulate JSON in a NetCDF File +#' @description Stores JSON data as a global attribute in a NetCDF file +#' @param json.file Character string. Path to the JSON file to be encapsulated +#' @param output.nc Character string. Output path for the NetCDF file +#' @return The function saves a .nc file with JSON encapsulated as metadata +#' @importFrom RNetCDF create.nc put.att.nc close.nc +#' @export +#' @family netcdf.outputs +#' @examples +#' encapsulate_json_in_netcdf("metadata.json", "output.nc") +#' +encapsulate_json_in_netcdf <- function(json.file, output.nc) { + # Load necessary library + if (!requireNamespace("RNetCDF", quietly = TRUE)) { + stop("RNetCDF package is required but not installed. Please install it.") + } + + # Read JSON content + json_content <- readLines(json.file, warn = FALSE) + json_string <- paste(json_content, collapse = "\n") + + # Create a new NetCDF file + nc <- RNetCDF::create.nc(output.nc) + + # Add JSON data as a global attribute + RNetCDF::put.att.nc(nc, "NC_GLOBAL", "metadata_json", "text", json_string) + + # Close the NetCDF file + RNetCDF::close.nc(nc) + + message("[", Sys.time(), "] The NetCDF file '", output.nc, "' with encapsulated JSON metadata was successfully created.") +} + + + diff --git a/provenance/R/prov_regridding.R b/provenance/R/prov_regridding.R new file mode 100644 index 0000000000000000000000000000000000000000..a172958e6c66145fef8195ac1176ccedf5efdd63 --- /dev/null +++ b/provenance/R/prov_regridding.R @@ -0,0 +1,179 @@ +prov_regridding <- function(recipe, data, graph) { + #Extract the type from previous nodename + Metadata definitions + parent.node <- graph$parentnodename + graph <- graph$graph + type <- strsplit(parent.node, "_")[[1]][[2]] + regtype <- recipe$Analysis$Regrid$method + var <- unlist(strsplit(recipe$Analysis$Variables$name, + ",\\s*"))[[1]] + + if (is.null(regtype)) { + + return(list("graph" = graph, "parentnodename" = parent.node)) + + }else { + regtypeclass <- switch(regtype, + "nn" = "ds:NearestNeighbor", + "bilinear" = "ds:BilinearInterpolation", + "bicubic" = "ds:BicubicInterpolation", + "distance-weighted" = "ds:InverseDistanceWeighting", + "laf" = "ds:InterpolationMethod", + "conservative" = "ds:ConservativeRemapping") + + regtype <- switch(regtype, + "con2" = "Conservative Remapping", + "bilinear" = "Bilinear", + "bicubic" = "Bicubic", + "distance-weighted" = "Distance Weighting", + "laf" = "Spline", + "nn" = "Nearest Neighbor") + + #Provenance for a Reggriding operation to the system's grid + if (recipe$Analysis$Regrid$type == "to_system") { + refgraph <- get_spatial_extent(data, "hcst", var, Reg = TRUE) + refgraphnode <- refgraph$parentnodename + refgraph <- refgraph$graph + + #Interpolation definition + if (type == "obs") { + regnodename <- paste0("Interpolation_", type) + + graph <- add_vertices(graph, + nv = 1, + label = paste(regtype, "interpolation"), + name = regnodename, + className = "ds:Interpolation", + attr = list("dc:description" = "Interpolation to + system, either to forecast or hindcast grid.")) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, + paste0("DatasetSubset_", type)), + getNodeIndexbyName(graph, regnodename)), + label = "ds:hadInterpolation") + + regmethod.nodename <- paste0("InterpolationMethod_", type) + graph <- add_vertices(graph, + nv = 1, + name = regmethod.nodename, + label = regtype, + className = regtypeclass) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, regnodename), + getNodeIndexbyName(graph, regmethod.nodename)), + label = "ds:hadInterpolationMethod") + + #Reference coordinates + graph <- my_union_graph(graph, refgraph) + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, regnodename), + getNodeIndexbyName(graph, refgraphnode)), + label = "ds:usedReferenceCoordinates") + + return(list("graph" = graph, "parentnodename" = regnodename)) + }else { + + return(list("graph" = graph, "parentnodename" = parent.node)) + } + + #Provenance for a Reggriding operation to the reference's grid + }else if (recipe$Analysis$Regrid$type == "to_reference") { + refgraph <- get_spatial_extent(data, "obs", var, Reg = TRUE) + refgraphnode <- refgraph$parentnodename + refgraph <- refgraph$graph + + #Interpolation definition + if (type == "hcst" || type == "fcst") { + regnodename <- paste0("Interpolation_", type) + graph <- add_vertices(graph, + nv = 1, + label = paste(regtype, "interpolation"), + name = regnodename, + className = "ds:Interpolation", + attr = list("dc:description" = "Interpolation + to reference, to the observations grid.")) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, + paste0("DatasetSubset_", type)), + getNodeIndexbyName(graph, regnodename)), + label = "ds:hadInterpolation") + + regmethod.nodename <- paste0("InterpolationMethod_", type) + + graph <- add_vertices(graph, + nv = 1, + name = regmethod.nodename, + label = regtype, + className = regtypeclass) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, regnodename), + getNodeIndexbyName(graph, regmethod.nodename)), + label = "ds:hadInterpolationMethod") + + #Reference coordinates + graph <- my_union_graph(graph, refgraph) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, regnodename), + getNodeIndexbyName(graph, refgraphnode)), + label = "ds:usedReferenceCoordinates") + + return(list("graph" = graph, "parentnodename" = regnodename)) + + }else { + return(list("graph" = graph, "parentnodename" = parent.node)) + } + + }else if (recipe$Analysis$Regrid$type == "none") { + + #Provenance for a Reggriding operation to a third grid + }else { + refgraph <- get_spatial_extent(data, "obs", var, Reg = TRUE) + refgraphnode <- refgraph$parentnodename + refgraph <- refgraph$graph + + #Interpolation definition + if (type == "hcst" || type == "fcst" || type == "obs") { + regnodename <- paste0("Interpolation_", type) + graph <- add_vertices(graph, + nv = 1, + label = paste(regtype, "interpolation"), + name = regnodename, + className = "ds:Interpolation", + attr = list("dc:description" + = paste0( + "Interpolation a + grid defined as:", + recipe$Analysis$Regrid$type + ))) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, + paste0("DatasetSubset_", type)), + getNodeIndexbyName(graph, regnodename)), + label = "ds:hadInterpolation") + regmethod.nodename <- paste0("InterpolationMethod_", "all") + graph <- add_vertices(graph, + nv = 1, + name = regmethod.nodename, + label = regtype, + className = regtypeclass) + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, regnodename), + getNodeIndexbyName(graph, regmethod.nodename)), + label = "ds:hadInterpolationMethod") + + #Reference coordinates + graph <- my_union_graph(graph, refgraph) + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, regnodename), + getNodeIndexbyName(graph, refgraphnode)), + label = "ds:usedReferenceCoordinates") + } + return(list("graph" = graph, "parentnodename" = regnodename)) + } + } +} diff --git a/provenance/R/prov_skill.R b/provenance/R/prov_skill.R new file mode 100644 index 0000000000000000000000000000000000000000..199616f3bc77d491536c428289509c434c642b0f --- /dev/null +++ b/provenance/R/prov_skill.R @@ -0,0 +1,95 @@ +prov_skill <- function(recipe, data, graph, type, Indices) { + if (!Indices) { + #Define parameters + i <- switch(type, + "fcst" = "parentnodename_fcst", + "hcst" = "parentnodename_hcst", + "obs" = "parentnodename_obs") + + #Particular case of computing Skill metrics for several Indices + parentnodename <- graph$parentnodename[[i]] + reference.nodename <- graph$parentnodename$parentnodename_obs + indicesnodenames <- NULL + #Change the nodename of the Verification step in + #case this is not the only verification step!! + + }else { + if (type == "hcst") { + i <- length(graph$parentnodename$indicesnodenames) + parentnodename <- graph$parentnodename$indicesnodenames[[1]] + reference.nodename <- graph$parentnodename$indicesnodenames[[2]] + graph$parentnodename$indicesnodenames <- + graph$parentnodename$indicesnodenames[-c(1, 2)] + indicesnodenames <- graph$parentnodename$indicesnodenames + } else if (type == "obs") { + parentnodename <- graph$parentnodename$indicesnodenames[[2]] + indicesnodenames <- NULL + } + } + graph <- graph$graph + measure <- recipe$Analysis$Workflow$Skill$metric + + if (type == "hcst" || type == "fcst") { + #Verification node + if (!Indices) { + verification.nodename <- paste0("Verification_", type) + }else { + verification.nodename <- paste0("Verification_", type, "_", i) + } + + graph <- add_vertices(graph, + nv = 1, + name = verification.nodename, + label = "Verification", + className = "veri:ForecastVerification") + + #Linking the verification + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parentnodename), + getNodeIndexbyName(graph, verification.nodename)), + label = "veri:hadValidation") + + #Linking the verification with the observations + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, reference.nodename), + getNodeIndexbyName(graph, verification.nodename)), + label = "veri:withReference") + + #Add Verification metrics + measure <- unlist(strsplit(measure, " ")) + archive <- read_yaml("conf/variable-dictionary.yml") + for (names in measure) { + metric <- tolower(names) + if (metric %in% names(archive$metrics)) { + metric.method <- archive$metrics[[metric]]$long_name + metric.class <- paste0("veri:", + archive$metrics[[metric]]$representation) + description <- archive$metrics[[metric]]$definition + }else { + stop("Skill method not registered or not properly defined") + } + if (!Indices) { + metric.nodename <- paste0("Metrics_", type, "_", metric) + }else { + metric.nodename <- paste0("Metrics_", type, "_", metric, "_", i) + } + + graph <- add_vertices(graph, + nv = 1, + name = metric.nodename, + label = metric.method, + className = metric.class, + attr = list("dc:description" = description)) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, metric.nodename), + getNodeIndexbyName(graph, verification.nodename)), + label = "veri:withForecastRepresentation") + } + return(list("graph" = graph, "parentnodename" = verification.nodename, + "indicesnodenames" = indicesnodenames)) + }else { + return(list("graph" = graph, "parentnodename" = parentnodename, + "indicesnodenames" = indicesnodenames)) + } +} \ No newline at end of file diff --git a/provenance/R/prov_visualization.R b/provenance/R/prov_visualization.R new file mode 100644 index 0000000000000000000000000000000000000000..dd0708d5f6aa3ff3db7c129a4eea27472c1b051c --- /dev/null +++ b/provenance/R/prov_visualization.R @@ -0,0 +1,155 @@ +#Currently can only embed the JSON to the metrics graphs and the forcast ensemble.P +#Provenance not defined for most_likely_terciles! +library(stringr) + +prov_visualization <- function(recipe, data, graph, type, outdirs) { + #Define parameters + plots <- strsplit(recipe$Analysis$Workflow$Visualization$plots, ", | |,")[[1]] + i <- switch(type, + "fcst" = "parentnodename_fcst", + "hcst" = "parentnodename_hcst") + parentnodename <- graph$parentnodename[[i]] + graph <- graph$graph + graph.full <- graph + + #Skill metrics graph nodes + if ("skill_metrics" %in% plots) { + measure <- unlist(strsplit(recipe$Analysis$Workflow$Skill$metric, " ")) + nodenames <- list() + for (names in measure){ + graph.plot <- graph + names <- tolower(names) + graph.nodename <- paste0("Graph_", type, "_", names) + label.name <- names + graph.path <- list() + + for (i in 1:length(outdirs)) { + if (names %in% as.list(unlist(strsplit(outdirs[[i]], "[/-]"))) || + names %in% as.list(unlist(strsplit(outdirs[[i]], "[/.]")))) { + graph.path <- append(graph.path, outdirs[[i]]) #All graph paths + } + } + #Graphical representation node + if (!is.null(recipe$Analysis$Workflow$Visualization$projection)) { + projection <- recipe$Analysis$Workflow$Visualization$projection + }else { + projection <- "Cylindrical equidistant" + } + graph.plot <- + add_vertices(graph.plot, + nv = 1, + name = graph.nodename, + label = paste0(label.name, " ", + "Graphical representation"), + className = "go:Map", + description = "A Map class object", + attr = list("go:hasProjection" = projection, + "prov:AtLocation" = + paste(unlist(graph.path), + collapse = " "))) + + #Link graph node with Skill that plots + graph.plot <- add_edges(graph.plot, + c(getNodeIndexbyName(graph.plot, + paste0("Metrics_", type, + "_", names)), + getNodeIndexbyName(graph.plot, graph.nodename)), + label = "go:hadGraphicalRepresentation") + + nodenames <- append(nodenames, list(graph.nodename)) + + graph.plot <- + prov_Command(graph.plot, package = "Visualization", + version = "2.0.0", + fun = "modules/Visualization/Visualization.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name + = c( + unlist( + graph.nodename))) + + #Embbed provenance to the generated image files + for (paths in graph.path) { + path.json <- paste0(str_sub(paths, 1, -4), "json") + graph2json(graph.plot, path.json) + embedJSON(paths, path.json, paths) + } + + graph.full <- my_union_graph(graph.full, graph.plot) + } + } + + #Forecast ensemble graph node + if (!is.null(data$fcst)) { + if ("forecast_ensemble_mean" %in% plots) { + graph.nodename <- paste0("Graph_", "forecast_ensemble") + graph.path <- list() + for (i in 1:length(outdirs)) { + if ("forecast_ensemble_mean" + %in% + as.list(unlist(strsplit(outdirs[[i]], "[/-]")))) { + graph.path <- append(graph.path, outdirs[[i]]) + } + } + graph.plot <- graph + graph.plot <- + add_vertices(graph.plot, + nv = 1, + name = graph.nodename, + label = paste0("Forecast Ensemble", + "Graphical representation"), + className = "go:Map", + attr = list("prov:AtLocation" = + paste(unlist(graph.path), + collapse = " "))) + + + #Find the latest node for fcst (before computing Skills) + known_names <- list("Anomalies_fcst", "Calibration_fcst", + "DatasetSubset_fcst", "Interpolation_fcst") + max_index <- -1 + max_name <- NULL + + for (name in known_names) { + if (name %in% V(graph)$name) { + node_index <- which(V(graph)$name == name) + if (node_index > max_index) { + max_index <- node_index + max_name <- name + } + } + } + lastnode <- max_name + graph.plot <- add_edges(graph.plot, + c(getNodeIndexbyName(graph.plot, lastnode), + getNodeIndexbyName(graph.plot, graph.nodename)), + label = "go:hadGraphicalRepresentation") + + nodenames <- append(nodenames, list(graph.nodename)) + + graph.plot <- + prov_Command(graph.plot, package = "Visualization", + version = "2.0.0", + fun = "modules/Visualization/Visualization.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name + = c( + unlist( + graph.nodename))) + for (paths in graph.path) { + graph2json(graph.plot, path.json) + embedJSON(paths, path.json, paths) + } + graph.full <- my_union_graph(graph.plot, graph.full) + } + } + return(list("graph" = graph.full, + "parentnodename" = parentnodename, "nodenames" = nodenames)) + +} diff --git a/provenance/prov_Aggregation.R b/provenance/prov_Aggregation.R new file mode 100644 index 0000000000000000000000000000000000000000..b467ecad0b78a74aba3810615a9d4c335bca31c0 --- /dev/null +++ b/provenance/prov_Aggregation.R @@ -0,0 +1,76 @@ +source("provenance/R/prov_aggregation.R") + +prov_Aggregation <- function(recipe, data) { + + graph <- data$graph.prov + + #hcst definition (Dataset+Datasubset) + graph_hcst <- prov_aggregation(recipe, data, graph, type = "hcst") + + #obs definition (Dataset+Datasubset) + graph_obs <- prov_aggregation(recipe, data, graph, type = "obs") + + #fcst definition (Dataset+Datasubset) + if (!is.null(data$fcst)) { + graph_fcst <- prov_aggregation(recipe, data, graph, type = "fcst") + } + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- my_union_graph(graph_fcst$graph, graph_hcst$graph, graph_obs$graph) + graph <- prov_Command(graph, package = "Aggregation", + version = "2.0.0", + fun = "modules/Aggregation/Aggregation.R", + arg.list = + list("Recipe", + "Data"), + origin.node.name = + c(graph_fcst$parentnodename, + graph_hcst$parentnodename, + graph_obs$parentnodename)) + graph <- list("graph" = graph, + "parentnodename" + = list("parentnodename_fcst" + = graph_fcst$parentnodename, + "parentnodename_hcst" + = graph_hcst$parentnodename, + "parentnodename_obs" + = graph_obs$parentnodename)) + } else { + graph <- my_union_graph(graph_hcst$graph, graph_obs$graph) + graph <- prov_Command(graph, package = "Aggregation", + version = "2.0.0", + fun = "modules/Aggregation/Aggregation.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(graph_hcst$parentnodename, + graph_obs$parentnodename)) + graph <- list("graph" = graph, + "parentnodename" + = list("parentnodename_hcst"=graph_hcst$parentnodename, + "parentnodename_obs"=graph_obs$parentnodename)) + } + + message("INFO ", "[", Sys.time(), "]", + " Provenance correctly generated for the Aggregation module") + + #Name of the generated JSON according to recipe values + main_dir <- paste0(recipe$Run$output_dir, "/outputs", "/provenance/") + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + + return(graph) + +} diff --git a/provenance/prov_Anomalies.R b/provenance/prov_Anomalies.R new file mode 100644 index 0000000000000000000000000000000000000000..2c74bba2412d5c6d8e6522bbaaf7489ff4eb9b5a --- /dev/null +++ b/provenance/prov_Anomalies.R @@ -0,0 +1,72 @@ +source("provenance/R/prov_anomalies.R") + +prov_Anomalies <- function(recipe, graph) { + + #hcst definition (Dataset+Datasubset) + graph_hcst <- prov_anomalies(recipe, data, graph, type = "hcst") + + #obs definition (Dataset+Datasubset) + graph_obs <- prov_anomalies(recipe, data, graph, type = "obs") + + #fcst definition (Dataset+Datasubset) + if (!is.null(data$fcst)) { + graph_fcst <- prov_anomalies(recipe, data, graph, type = "fcst") + } + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- my_union_graph(graph_fcst$graph, graph_hcst$graph, graph_obs$graph) + graph <- prov_Command(graph, package = "Anomalies", + version = "2.0.0", + fun = "modules/Anomalies/Anomalies.R", + arg.list = + list("Recipe", + "Data"), + origin.node.name = + c(graph_fcst$parentnodename, + graph_hcst$parentnodename, + graph_obs$parentnodename)) + graph <- list("graph" = graph, + "parentnodename" + = list("parentnodename_fcst" + = graph_fcst$parentnodename, + "parentnodename_hcst" + = graph_hcst$parentnodename, + "parentnodename_obs" + = graph_obs$parentnodename)) + } else { + graph <- my_union_graph(graph_hcst$graph, graph_obs$graph) + graph <- prov_Command(graph, package = "Anomalies", + version = "2.0.0", + fun = "modules/Anomalies/Anomalies.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(graph_hcst$parentnodename, + graph_obs$parentnodename)) + graph <- list("graph" = graph, + "parentnodename" + = list("parentnodename_hcst"=graph_hcst$parentnodename, + "parentnodename_obs"=graph_obs$parentnodename)) + } + message("INFO ", "[", Sys.time(), "]", + " Provenance correctly generated for the Anomalies module") + + #Name of the generated JSON according to recipe values + main_dir <- paste0(recipe$Run$output_dir, "/outputs", "/provenance/") + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + return(graph) + +} \ No newline at end of file diff --git a/provenance/prov_Calibration.R b/provenance/prov_Calibration.R new file mode 100644 index 0000000000000000000000000000000000000000..bd5346d9a9b034c193170604ad903040b149f725 --- /dev/null +++ b/provenance/prov_Calibration.R @@ -0,0 +1,76 @@ +source("provenance/R/prov_calibration.R") + +prov_Calibration <- function(recipe, graph) { + + #Check if Calibration is requested + if (recipe$Analysis$Workflow$Calibration$method != "raw") { + #fcst definition (Dataset+Datasubset) + if (!is.null(data$fcst)) { + graph_fcst <- prov_calibration(recipe, data, graph, type = "fcst") + } + #hcst definition (Dataset+Datasubset) + graph_hcst <- prov_calibration(recipe, data, graph, type = "hcst") + + #obs definition (Dataset+Datasubset) + graph_obs <- prov_calibration(recipe, data, graph, type = "obs") + + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- my_union_graph(graph_fcst$graph, + graph_hcst$graph, graph_obs$graph) + graph <- prov_Command(graph, package = "Calibration", + version = "2.0.0", + fun = "modules/Calibration/Calibration.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list( + graph_fcst$parentnodename, + graph_hcst$parentnodename)) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_fcst" = graph_fcst$parentnodename, + "parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = graph_obs$parentnodename)) + } else{ + graph <- my_union_graph(graph_hcst$graph, graph_obs$graph) + graph <- prov_Command(graph, package = "Calibration", + version = "2.0.0", + fun = "modules/Calibration/Calibration.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(graph_hcst$parentnodename)) + + graph <- list("graph" = graph, + "parentnodename" = + list("parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = graph_obs$parentnodename)) + } + + message("INFO ", "[", Sys.time(), "]", + " Provenance correctly generated for the Calibration module") + + #Name of the generated JSON according to recipe values + main_dir <- paste0(recipe$Run$output_dir, "/outputs", "/provenance/") + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + return(graph) + } +} diff --git a/provenance/prov_Command.R b/provenance/prov_Command.R new file mode 100644 index 0000000000000000000000000000000000000000..fee506ed1c08c3c51bc95471de8b7113a7d6274b --- /dev/null +++ b/provenance/prov_Command.R @@ -0,0 +1,153 @@ +#' @title Generate provenance for SUNSET command calls +#' @description Generate provenance information for command calls based on metaclipR function metalcipR.graph.Command() +#' @param graph Current provenance graph. +#' @param package Package name +#' @param version Version name +#' @param function Function name +#' @param arg.list Argument list +#' @param origin.node.name List containing the name of the origin nodes to be linked to the command call +#' @return List containing updated graph. +#' @references https://earth.bsc.es/gitlab/es/sunset/-/wikis/home GitLab repository for SUNSET, a modular tool for climate forecast verification workflows. +#' @references This function is based on the semantics outlined in the ontology within the Metaclip Framework (available at \url{http://www.metaclip.org/}). + +prov_Command <- function(graph, package, version, + fun, arg.list, origin.node.name) { + + #Check the inputs are correctly defined + cmd.node.name <- paste0("Command_", package, version) + # Argument and ArgumentValue + time <- Sys.time() + args <- paste(arg.list, collapse = " ") + graph <- add_vertices_sunset(graph, + name = cmd.node.name, + className = "ds:Command", + label = fun, + attr = list("prov:value" = fun, + "prov:atTime" = as.character(time))) + + # Add the edge linking the step with the command + for (i in 1:length(origin.node.name)) { + graph <- add_edges(graph, + c( + getNodeIndexbyName( + graph, origin.node.name[i]), + getNodeIndexbyName( + graph, cmd.node.name)), + label = "ds:hadCommandCall") + } + + # Add Arguments (Recipe and Data) + if ("Recipe" %in% arg.list) { + recipe.nodename <- paste0("Argument_Recipe_", package) + graph <- add_vertices_sunset(graph, + name = recipe.nodename, + className = "ds:Argument", + label = "Recipe", + attr = + list("dc:description" = + "User-defined configuration file", + "prov:AtLocation" = recipe$Run$output_dir)) + + graph <- add_edges(graph, + c( + getNodeIndexbyName( + graph, recipe.nodename), + getNodeIndexbyName( + graph, cmd.node.name)), + label = "ds:usedArgument") + } + + if ("Data" %in% arg.list) { + recipe.nodename <- paste0("Argument_Data_", package) + graph <- add_vertices_sunset(graph, + name = recipe.nodename, + className = "ds:Argument", + label = "Data", + attr = + list( + "dc:description" = + "s2dv_cube object", + "ds:referenceURL" = + "https://cran.r-project.org/ + web/packages/s2dv/index.html", + "dc:comment" = "output s2dv_cube object + from previous module")) + + graph <- add_edges(graph, + c( + getNodeIndexbyName( + graph, recipe.nodename), + getNodeIndexbyName( + graph, cmd.node.name)), + label = "ds:usedArgument") +} + + + # Package (SUNSET) + pkg.nodename <- paste0("Package_", package) + graph <- add_vertices_sunset(graph = graph, + name = pkg.nodename, + className = "ds:Package", + label = package, + attr = list("ds:hasSoftwareVersion" = version, + "dc:isDefinedBy" = "SUNSET", + "dc:comment" = " SUNSET (SUbseasoNal + to decadal climate forecast + post-processIng and asSEmenT suite) is + a modular tool for subseasonal + to seasonal to decadal forecast + verification workflows. It is + intended to have a modularized + structure, where each module is a + separate part of the code that + performs a specific task, so that + parts of the workflow can be + skipped or reordered. Each command call + node corresponds to one of SUNSET's + modules execution", + "ds:referenceURL" = + "https://earth.bsc.es/gitlab/ + es/sunset/")) + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, cmd.node.name), + getNodeIndexbyName(graph, pkg.nodename)), + label = "ds:fromPackage") + + #Retrive git metadata + #library(git2r) + + #repo <- repository(getwd()) + + #branch <- repository_head(repo) + + #commit <- commits(repo)[[1]] # Latest commit + + #config <- config(repo) + #user_name <- config$global$user.name + #user_email <- config$global$user.email + + + # Package (Library versions) + #pkg.nodename <- paste0("Package_libaries_", randomName()) + #library_versions <- paste(paste0("-", packages[1:10, "Package"], + # "(", packages[1:10, "Version"], ")", "\n"), + #collapse = " ") + + #Maybe we should add somwhere the libraries required y sunset... + + #graph <- my_add_vertices(graph = graph, + # name = pkg.nodename, + # className = "ds:Package", + # label = "Libraries used and their Versions", + # attr = list("ds:hasSoftwareVersion" = + # library_versions)) + #graph <- add_edges(graph, + # c(getNodeIndexbyName(graph, cmd.node.name), + # getNodeIndexbyName(graph, pkg.nodename)), + # label = "ds:fromPackage") + + #Software Agent + #(...) + + return(graph) +} diff --git a/provenance/prov_Downscaling.R b/provenance/prov_Downscaling.R new file mode 100644 index 0000000000000000000000000000000000000000..9287efacd31ef257e6815fe84ec6bd5373299ea1 --- /dev/null +++ b/provenance/prov_Downscaling.R @@ -0,0 +1,111 @@ +#' @title Generate provenance for SUNSET's Downscaling module execution +#' @description Generate provenance information for the execution of SUNSET's Downscaling module. +#' @param recipe Configuration file specifying parameters. +#' @param data s2dv_cube object generated by SUNSET's Downscaling.R function. +#' @return List containing updated graph and parent node name. +#' @details This function generates provenance information for the execution of SUNSET's Downscaling module. +#' @references https://earth.bsc.es/gitlab/es/csdownscale CSTools documentation for CSDownscale. +#' @references This function is based on the semantics outlined in the Calibration ontology within the Metaclip Framework (available at \url{http://www.metaclip.org/}). + +prov_Downscaling <- function(recipe, graph) { + + #Variable definition + parentnodename <- graph$parentnodename$parentnodename_hcst + parentnodename_fcst <- graph$parentnodename$parentnodename_fcst + parentnodename_obs <- graph$parentnodename$parentnodename_obs + down <- recipe$Analysis$Workflow$Downscaling$type + graph <- graph$graph + archive <- read_yaml("conf/variable-dictionary_provenance.yml") + if (down %in% names(archive$downscaling)) { + downsc.method <- archive$downscaling[[down]]$long_name + downsc.class <- paste0("cal:", archive$downscaling[[down]]$representation) + description <- archive$downscaling[[down]]$definition + }else { + stop("Downscaling method not registered or not properly defined") + } + + int <- recipe$Analysis$Workflow$Downscaling$int_method + + #Add Downscaling node definition + downsc.nodename <- paste0("Downscaling_", "hcst") + graph <- add_vertices(graph, + nv = 1, + name = downsc.nodename, + label = downsc.method, + className = downsc.class, + attr = + list( + "dc:description" = description, + "dc:comment" = "For more details, see the + CSTools documentation for CSDownscale", + "ds:renferenceURL" = + "https://earth.bsc.es/gitlab/es/csdownscale")) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parentnodename), + getNodeIndexbyName(graph, downsc.nodename)), + label = "cal:hadCalibration") + + #Add New Spatial Extent + refgraph <- get_spatial_extent(data, type = "hcst", Downsc = TRUE) + graph <- my_union_graph(graph, refgraph$graph) + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, downsc.nodename), + getNodeIndexbyName(graph, refgraph$parentnodename)), + label = "ds:hasHorizontalExtent") + + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- prov_Command(graph, package = "Downscaling", + version = "2.0.0", + fun = "modules/Downscaling/Downscaling.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(downsc.nodename)) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_fcst" = parentnodename_fcst, + "parentnodename_hcst" = downsc.nodename, + "parentnodename_obs" = parentnodename_obs)) + } else { + graph <- prov_Command(graph, package = "Downscaling", + version = "2.0.0", + fun = "modules/Downscaling/Downscaling.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(downsc.nodename)) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_hcst" = downsc.nodename, + "parentnodename_obs" = parentnodename_obs)) + } + message("INFO ", "[", Sys.time(), "]", + " Provenance correctly generated for the Downscaling module") + + #Name of the generated JSON according to recipe values + main_dir <- paste0(recipe$Run$output_dir, "/outputs", "/provenance/") + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + + return(graph) +} \ No newline at end of file diff --git a/provenance/prov_Indices.R b/provenance/prov_Indices.R new file mode 100644 index 0000000000000000000000000000000000000000..181e91030346b63f8251130d4d5914b7722d007f --- /dev/null +++ b/provenance/prov_Indices.R @@ -0,0 +1,118 @@ +prov_Indices <- function(recipe, graph) { + #Indices metadata definitions + index_name <- names(recipe$Analysis$Workflow$Indices) + parentnodename_fcst <- graph$parentnodename$parentnodename_fcst + parentnodename_hcst <- graph$parentnodename$parentnodename_hcst + parentnodename_obs <- graph$parentnodename$parentnodename_obs + i <- 0 + graph <- graph$graph + + nodenames <- list() + indexnodenames <- list() + + for (index in index_name){ + i <- i + 1 + #Match METACLIP class with Index + class <- switch(index, + "NAO" = "ds:NAO", + "Nino1+2" = "ds:Nino1plus2", + "Nino3" = "ds:Nino3", + "Nino3.4" = "ds:Nino3_4", + "Nino4" = "ds:Nino4") + + #Climate Index type node + if (index != "NAO"){ + index.nodename <- class + }else { + index.nodename <- paste0("ClimateIndex_", i) + } + + graph <- add_vertices(graph, + nv = 1, + name = index.nodename, + label = index, + className = class) + + indexnodenames <- append(indexnodenames, index.nodename) + + for (parentnodename in c(parentnodename_hcst, parentnodename_obs)) { + type <- strsplit(parentnodename, "_")[[1]][[2]] + climin.nodename <- paste0("ClimateIndex_", type, "_", i) + graph <- add_vertices(graph, + nv = 1, + name = climin.nodename, + label = "Climate Index Calculation", + className = "ds:ClimateIndexCalculation") + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, parentnodename), + getNodeIndexbyName(graph, climin.nodename)), + label = "ds:hadClimateIndexCalculation") + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, climin.nodename), + getNodeIndexbyName(graph, index.nodename)), + label = "ds:withClimateIndex") + + nodenames <- append(nodenames, climin.nodename) + + } + } + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- prov_Command(graph, package = "Indices", + version = "2.0.0", + fun = "modules/Indices/Indices.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + nodenames) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_fcst" = parentnodename_fcst, + "parentnodename_hcst" = parentnodename_hcst, + "parentnodename_obs" = parentnodename_obs, + "indicesnodenames" = nodenames)) + + } else { + graph <- prov_Command(graph, package = "Indices", + version = "2.0.0", + fun = "modules/Indices/Indices.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + nodenames) + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_hcst" = parentnodename_hcst, + "parentnodename_obs" = parentnodename_obs, + "indicesnodenames" = nodenames)) + } + + message("INFO ", "[", Sys.time(), "]", + "Provenance correctly generated for the Indices module") + + #Name of the generated JSON according to recipe values + main_dir <- paste0(recipe$Run$output_dir, "/outputs", "/provenance/") + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + + return(graph) +} diff --git a/provenance/prov_Loading.R b/provenance/prov_Loading.R new file mode 100644 index 0000000000000000000000000000000000000000..a13ef5c0c87503973eb198663fdc600e5297f12a --- /dev/null +++ b/provenance/prov_Loading.R @@ -0,0 +1,118 @@ +#' @title Directed metadata graph construction for SUNSET's Loading module +#' @description Generate provenance for a SUNSET workflow involving loading of climate +#' forecast, hindcast and observational data. +#' @param recipe Configuration file specifying parameters. +#' @param data s2dv_cube object generated by SUNSET's Loading.R function. +#' @details This function constructs provenance information for a SUNSET workflow, +#' specifically focusing on the loading module. It generates provenance for +#' climate forecast hindcast and observational data, including dataset subsets +#' and reggriding information. The function integrates other provenance generation +#' functions to create a comprehensive metadata graph. +#' @references SUNSET GitLab Repository: https://earth.bsc.es/gitlab/es/sunset/-/wikis/home +#' This repository hosts SUNSET, a modular tool for subseasonal to seasonal to decadal forecast +#' verification workflows. It is structured modularly to allow skipping or reordering +#' workflow parts. +#' @references https://earth.bsc.es/gitlab/es/sunset/-/wikis/home GitLab repository for SUNSET, +#' a modular tool for climate forecast verification workflows. +#' @references This function is based on the semantics outlined in the ontology within the Metaclip +#' Framework (available at \url{http://www.metaclip.org/}). + +source("provenance/R/prov_dataset.R") +source("provenance/R/prov_datasetsubset.R") +source("provenance/R/prov_regridding.R") +source("provenance/prov_Command.R") +source("provenance/R/prov_helpers.R") + +library(igraph) + +prov_Loading <- function(recipe, data) { + #fcst definition (Dataset + Datasubset + Regridding) + if (!is.null(data$fcst)) { + graph_fcst <- prov_dataset(recipe, data, type = "fcst") + graph_fcst <- prov_datasetsubset(recipe, data, graph_fcst) + graph_fcst <- prov_regridding(recipe, data, graph_fcst) + } + #hcst definition (Dataset + Datasubset + Regridding) + graph_hcst <- prov_dataset(recipe, data, type = "hcst") + graph_hcst <- prov_datasetsubset(recipe, data, graph_hcst) + graph_hcst <- prov_regridding(recipe, data, graph_hcst) + + #obs definition (Dataset + Datasubset + Regridding) + graph_obs <- prov_dataset(recipe, data, type = "obs") + graph_obs <- prov_datasetsubset(recipe, data, graph_obs) + graph_obs <- prov_regridding(recipe, data, graph_obs) + + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- my_union_graph(graph_fcst$graph, graph_hcst$graph, graph_obs$graph) + + graph <- prov_Command(graph, package = "Loading", + version = "2.0.0", + fun = "modules/Loading/Loading.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list( + graph_fcst$parentnodename, + graph_hcst$parentnodename, + graph_obs$parentnodename, + "DatasetSubset_fcst", + "DatasetSubset_hcst", + "DatasetSubset_obs")) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_fcst" = graph_fcst$parentnodename, + "parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = graph_obs$parentnodename)) + } else { + graph <- my_union_graph(graph_hcst$graph, graph_obs$graph) + graph <- prov_Command(graph, package = "Loading", + version = "2.0.0", + fun = "modules/Loading/Loading.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list( + graph_hcst$parentnodename, + graph_obs$parentnodename, + "DatasetSubset_hcst", + "DatasetSubset_obs")) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = graph_obs$parentnodename)) + } + + message("INFO ", "[", Sys.time(), "]", + " Provenance correctly generated for the Loading module") + + #Create directory to store de JSON files + main_dir <- paste0(recipe$Run$output_dir, "/outputs", "/provenance/") + if (!dir.exists(main_dir)) { + dir.create(main_dir, recursive = TRUE) + } + + #Name of the generated JSON according to recipe values + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + + return(graph) +} diff --git a/provenance/prov_Scorecards.R b/provenance/prov_Scorecards.R new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/provenance/prov_Skill.R b/provenance/prov_Skill.R new file mode 100644 index 0000000000000000000000000000000000000000..d3c3672d9d6d56cfe9840bef6658143bd1a910d5 --- /dev/null +++ b/provenance/prov_Skill.R @@ -0,0 +1,88 @@ +source("provenance/R/prov_skill.R") + +prov_Skill <- function(recipe, graph) { + #fcst definition (Dataset+Datasubset) + if (!is.null(graph$parentnodename$indicesnodenames)) { + Indices <- TRUE #If True the last step was Indices calculation + }else { + Indices <- FALSE + } + + if (!is.null(data$fcst)) { + graph_fcst <- prov_skill(recipe, data, graph, type = "fcst", + Indices) + } + #obs definition (Dataset+Datasubset) + graph_obs <- prov_skill(recipe, data, graph, type = "obs", + Indices) + + #hcst definition (Dataset+Datasubset) + graph_hcst <- prov_skill(recipe, data, graph, type = "hcst", + Indices) + + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- my_union_graph(graph_fcst$graph, graph_hcst$graph) + + graph <- prov_Command(graph, package = "Skill", + version = "2.0.0", + fun = "modules/Skill/Skill.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(graph_fcst$parentnodename, + graph_hcst$parentnodename)) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_fcst" = graph_fcst$parentnodename, + "parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = graph_obs$parentnodename, + "indicesnodenames" = graph_hcst$indicesnodenames)) + } else { + graph <- graph_hcst$graph + graph <- prov_Command(graph, package = "Skill", + version = "2.0.0", + fun = "modules/Skill/Skill.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(graph_hcst$parentnodename)) + + graph <- list("graph" = graph, + "parentnodename" = + list("parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = graph_obs$parentnodename, + "indicesnodenames" = graph_hcst$indicesnodenames)) + + } + + message("INFO ", "[", Sys.time(), "]", + "Provenance correctly generated for the Skill module") + + #Name of the generated JSON according to recipe values + trimmed_dir <- dirname(dirname(recipe$Run$output_dir)) + + main_dir <- paste0(trimmed_dir, "/outputs", "/provenance/") + + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + + return(graph) + +} \ No newline at end of file diff --git a/provenance/prov_Units.R b/provenance/prov_Units.R new file mode 100644 index 0000000000000000000000000000000000000000..1cfcfa7efdbf79850263d47b1da04fb0bf7b2455 --- /dev/null +++ b/provenance/prov_Units.R @@ -0,0 +1,108 @@ +prov_Units <- function(recipe, data, orig_units) { + + #NEW UNITS (Recipe) + vars <- recipe$Analysis$Variables$name + user_units <- recipe$Analysis$Variable$units + var.freq <- recipe$Analysis$Variables$freq + + #Considering the case of more the one variable being loaded simulateonsly + vars_split <- strsplit(vars, ", ")[[1]] + + graph <- data$graph.prov + parentnodenames <- graph$parentnodename + orig <- parentnodenames + graph <- graph$graph + + if (!is.null(data$fcst)){ + aux_vec <- list("fcst", "hcst", "obs") + } else{ + aux_vec <- list("hcst", "obs") + } + + j <- 1 + for (type in aux_vec){ + description <- "" + for (i in seq_along(user_units)) { + var <- vars_split[i] + unit <- user_units[i][[var]] + description <- paste0(description, " -", var, ": From ", orig_units[[j]][i], " to ", unit, ". ") + } + + unitconversion.nodename <- paste0("UnitConversion_", type) + + parentnodenames[[j]] <- unitconversion.nodename + + graph <- add_vertices_sunset(graph, + nv = 1, + name = unitconversion.nodename, + label = "Unit conversion", + className = "ds:UnitConversion", + attr = list( + "dc:description" = description + ) + ) + + graph <- add_edges(graph, + c(getNodeIndexbyName(graph, orig[[j]]), + getNodeIndexbyName(graph, unitconversion.nodename)), + label = "ds:hadDerivation") + j <- j+1 + } + + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- prov_Command(graph, package = "Units", + version = "2.0.0", + fun = "modules/Units/Units.R", + arg.list = + list("Recipe", + "Data"), + origin.node.name = + c(parentnodenames[[1]], + parentnodenames[[2]], + parentnodenames[[3]])) + graph <- list("graph" = graph, + "parentnodename" + = list("parentnodename_fcst" + = parentnodenames[[1]], + "parentnodename_hcst" + = parentnodenames[[2]], + "parentnodename_obs" + = parentnodenames[[3]])) + } else { + graph <- prov_Command(graph, package = "Units", + version = "2.0.0", + fun = "modules/Units/Units.R", + arg.list = + list( + "Recipe", + "Data"), + origin.node.name = + list(parentnodenames[[1]], + parentnodenames[[2]])) + graph <- list("graph" = graph, + "parentnodename" + = list("parentnodename_hcst"=parentnodenames[[1]], + "parentnodename_obs"=parentnodenames[[2]])) + } + + message("INFO ", "[", Sys.time(), "]", + " Provenance correctly generated for the Units module") + + #Name of the generated JSON according to recipe values + main_dir <- paste0(recipe$Run$output_dir, "/outputs", "/provenance/") + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + return(graph) + +} diff --git a/provenance/prov_Visualization.R b/provenance/prov_Visualization.R new file mode 100644 index 0000000000000000000000000000000000000000..89c3107aba2bb82d2dd25b91f9931e8837d12fa6 --- /dev/null +++ b/provenance/prov_Visualization.R @@ -0,0 +1,59 @@ +source("provenance/R/prov_visualization.R") + +prov_Visualization <- function(recipe, graph, outdirs) { + + #fcst definition + if (!is.null(data$fcst)) { + graph_fcst <- prov_visualization(recipe, data, graph, type = "fcst", + outdirs) + } + + #hcst definition (Dataset+Datasubset) + graph_hcst <- prov_visualization(recipe, data, graph, type = "hcst", outdirs) + + #obs definition + parentnodename_obs <- graph$parentnodename$parentnodename_obs + + #Command Call and output definition + if (!is.null(data$fcst)) { + graph <- my_union_graph(graph_fcst$graph, graph_hcst$graph) + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_fcst" = graph_fcst$parentnodename, + "parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = parentnodename_obs)) + + } else { + graph <- graph_hcst$graph + + graph <- list("graph" = graph, + "parentnodename" = + list( + "parentnodename_hcst" = graph_hcst$parentnodename, + "parentnodename_obs" = parentnodename_obs)) + } + + message("INFO ", "[", Sys.time(), "]", + " Provenance correctly generated for the Visualization module") + message("INFO ", "[", Sys.time(), "]", + " JSON provenance metadata correctly embedded on the images") + + #Name of the generated JSON according to recipe values + trimmed_dir <- dirname(recipe$Run$output_dir) + main_dir <- paste0(trimmed_dir, "/outputs", "/provenance/") + recipe_model <- paste0("sys-", + gsub('\\.', '', recipe$Analysis$Datasets$System$name)) + recipe_split <- paste0("_ref-", recipe$Analysis$Datasets$Reference$name, + "_var-", recipe$Analysis$Variables$name, + "_reg-", recipe$Analysis$Region$name, + "_sdate-", recipe$Analysis$Time$sdate) + + json_name <- paste0("main_provenance", "_", recipe_model, recipe_split) + + #Convert from igraph to JSON format and store in the output directory + graph2json(graph$graph, output.file = paste0(main_dir, + json_name, ".json")) + return(graph) +} diff --git a/recipe_template.yml b/recipe_template.yml index b9cd5300eda36aab5966260156bfcd4afd5dede2..fd60b58444dd003dd6bb7ba9a1a536e2d11faca0 100644 --- a/recipe_template.yml +++ b/recipe_template.yml @@ -4,6 +4,7 @@ Description: Info: # Complete recipe containing all possible fields. Analysis: Horizon: seasonal # Mandatory, str: 'subseasonal', 'seasonal', or 'decadal'. + Provenance: yes # Whether or not to generate a provenance record of the workflow execution. Variables: # name: variable name(s) in the archive (Mandatory, str) # freq: 'monthly_mean', 'daily' or 'daily_mean' (Mandatory, str) @@ -193,3 +194,4 @@ Run: email_address: victoria.agudetse@bsc.es # replace with your email address notify_completed: yes # notify me by email when a job finishes notify_failed: yes # notify me by email when a job fails + orcid: https://orcid.org/0009-0006-0707-6448 # unique, persistent identifier for researchers. Needed to generate provenance when using autosubmit diff --git a/recipes/atomic_recipes/recipe_decadal.yml b/recipes/atomic_recipes/recipe_decadal.yml index 2028fc468766a74bdc32b2ea70e60f89b637bcaa..12eac6030330f58de9436617f0b3a69881641c99 100644 --- a/recipes/atomic_recipes/recipe_decadal.yml +++ b/recipes/atomic_recipes/recipe_decadal.yml @@ -2,6 +2,7 @@ Description: Author: An-Chi Ho '': split version Analysis: + Provenance: no Horizon: Decadal Variables: name: tas diff --git a/recipes/atomic_recipes/recipe_seasonal_downscaling.yml b/recipes/atomic_recipes/recipe_seasonal_downscaling.yml index 18d03185a40f7ad83ca712fc443d3a35e4577132..add1403b7a1660d4c149e4c146ea8b3b6e38b338 100644 --- a/recipes/atomic_recipes/recipe_seasonal_downscaling.yml +++ b/recipes/atomic_recipes/recipe_seasonal_downscaling.yml @@ -2,6 +2,7 @@ Description: Author: V. Agudetse Description: ECMWF-SEAS5 Downscaling Analysis: + Provenance: no Horizon: Seasonal Variables: name: tas diff --git a/recipes/atomic_recipes/recipe_seasonal_provenance.yml b/recipes/atomic_recipes/recipe_seasonal_provenance.yml index 196b49c92bcbd52582bbf4573ae51591e3096ebc..7937d936ae5e4409aff6912aa7165da21615dac2 100644 --- a/recipes/atomic_recipes/recipe_seasonal_provenance.yml +++ b/recipes/atomic_recipes/recipe_seasonal_provenance.yml @@ -3,6 +3,7 @@ Description: '': split version Analysis: Horizon: seasonal + Provenance: yes Variables: name: tas freq: monthly_mean @@ -29,14 +30,19 @@ Analysis: type: to_system Workflow: Anomalies: - compute: no - cross_validation: - save: + compute: yes + cross_validation: yes + save: none + Time_aggregation: + execute: yes + method: average + ini: [1] + end: [2] Calibration: method: bias save: 'all' Skill: - metric: RPSS + metric: mean_bias enscorr rpss crpss enssprerr save: 'all' Probabilities: percentiles: [[1/3, 2/3]] @@ -51,7 +57,7 @@ Analysis: Run: Loglevel: INFO Terminal: yes - output_dir: /tmp/out-logs/ - code_dir: /home/kinow/Development/r/workspace/sunset/ + output_dir: /esarchive/scratch/apuiggro/sunset/outputs + code_dir: /esarchive/scratch/apuiggro/sunset/ filesystem: sample diff --git a/recipes/atomic_recipes/recipe_system5c3s-tas-robinson.yml b/recipes/atomic_recipes/recipe_system5c3s-tas-robinson.yml index f9f7d84e4809c084af1aee715c990006ae762990..42e60aa74cfd4e5d2fb2b899cf20da6d4a2e9a36 100644 --- a/recipes/atomic_recipes/recipe_system5c3s-tas-robinson.yml +++ b/recipes/atomic_recipes/recipe_system5c3s-tas-robinson.yml @@ -2,6 +2,7 @@ Description: Author: V. Agudetse Description: Recipe to test forecast times + robinson projection with SEAS5 Analysis: + Provenance: no Horizon: Seasonal Variables: name: tas diff --git a/recipes/atomic_recipes/recipe_system7c3s-prlr.yml b/recipes/atomic_recipes/recipe_system7c3s-prlr.yml index 590d4499d07b1761737f4c0d2f89bec2bb5e30c7..15a36c996ecf27c5dcaf486b83195a51a08e5089 100644 --- a/recipes/atomic_recipes/recipe_system7c3s-prlr.yml +++ b/recipes/atomic_recipes/recipe_system7c3s-prlr.yml @@ -2,6 +2,7 @@ Description: Author: V. Agudetse Description: Analysis of MF System 7 with precipitation Analysis: + Provenance: no Horizon: Seasonal Variables: name: prlr diff --git a/recipes/atomic_recipes/recipe_test_multivar.yml b/recipes/atomic_recipes/recipe_test_multivar.yml index 7de6bc19b4ff7a3cb9350d0dbd63b1e2494ced9e..e858b15a844b72c82aea42d3ede9aabbe7628399 100644 --- a/recipes/atomic_recipes/recipe_test_multivar.yml +++ b/recipes/atomic_recipes/recipe_test_multivar.yml @@ -2,6 +2,7 @@ Description: Author: V. Agudetse Description: Multiple variables in the same atomic recipe Analysis: + Provenance: no Horizon: Seasonal Variables: name: tas, prlr diff --git a/recipes/examples/recipe_model_decadal_NAO.yml b/recipes/examples/recipe_model_decadal_NAO.yml index cd1b2bdd1b7d797dbf62f5c9235362de4c2632b6..947ddcd62bff5b66d998ae6f052b2b436508261a 100644 --- a/recipes/examples/recipe_model_decadal_NAO.yml +++ b/recipes/examples/recipe_model_decadal_NAO.yml @@ -3,6 +3,7 @@ Description: Info: Test for NAO indices in decadal prediction Analysis: Horizon: Decadal + Provenance: no Variables: name: psl freq: monthly_mean diff --git a/recipes/examples/recipe_subseasonal.yml b/recipes/examples/recipe_subseasonal.yml index d817c69a4317247aa6d2df401ee3dc72990b180a..3f26b3dcc0ee3612b3ffaec88111bda726a47a40 100644 --- a/recipes/examples/recipe_subseasonal.yml +++ b/recipes/examples/recipe_subseasonal.yml @@ -4,6 +4,7 @@ Description: Info: # Complete recipe containing all possible fields. Analysis: Horizon: subseasonal # Mandatory, str: 'seasonal', or 'decadal'. Subseasonal is in development + Provenance: no Variables: # name: variable name(s) in the /esarchive (Mandatory, str) # freq: 'monthly_mean', 'daily' or 'daily_mean' (Mandatory, str) diff --git a/recipes/examples/recipe_tas_seasonal_timeagg.yml b/recipes/examples/recipe_tas_seasonal_timeagg.yml index b62ac79af1dda1461998bfb98972a2fa69423831..a86ec9b7d4fef56519c7cb91a7b1113399697bf7 100644 --- a/recipes/examples/recipe_tas_seasonal_timeagg.yml +++ b/recipes/examples/recipe_tas_seasonal_timeagg.yml @@ -3,6 +3,7 @@ Description: Info: ECVs Oper ESS ECMWF SEAS5 Seasonal Forecast recipe (monthly mean, tas) Analysis: + Provenance: no Horizon: seasonal # Mandatory, str: either subseasonal, seasonal, or decadal Variables: name: tas diff --git a/recipes/examples/test_provenance_1.yml b/recipes/examples/test_provenance_1.yml new file mode 100644 index 0000000000000000000000000000000000000000..54b1143ee3ece266697ca43a19d597924571ce18 --- /dev/null +++ b/recipes/examples/test_provenance_1.yml @@ -0,0 +1,75 @@ +Description: + Author: A.Puiggros + Description: split version +Analysis: + Provenance: yes + Horizon: seasonal + Variables: + - {name: 'tas', freq: 'monthly_mean', units: {tas: 'C'}} + Datasets: + System: + name: ECMWF-SEAS5.1 + Multimodel: no + Reference: + name: ERA5 + Time: + sdate: '1101' + fcst_year: + hcst_start: '2000' + hcst_end: '2006' + ftime_min: 1 + ftime_max: 3 + Region: + name: "EU" + latmin: 20 + latmax: 80 + lonmin: -20 + lonmax: 40 + Regrid: + method: bilinear + type: to_reference + Workflow: + Anomalies: + compute: yes + cross_validation: yes + save: 'none' + Time_aggregation: + execute: yes + method: average + ini: [1] + end: [2] + Calibration: + method: bias + save: 'none' + Downscaling: + type: analogs + int_method: + bc_method: + lr_method: + log_reg_method: + nanalogs: 3 + target_grid: /esarchive/recon/ecmwf/era5/daily_mean/tas_f1h/tas_199301.nc + save: 'all' + Skill: + metric: Mean_Bias EnsCorr RPSS RPS CRPS CRPSS + save: 'none' + Probabilities: + percentiles: [[1/3, 2/3]] + save: 'all' + Indicators: + index: FALSE + Visualization: + plots: skill_metrics + multi_panel: no + projection: 'cylindrical_equidistant' # Options: 'cylindrical_equidistant', 'robinson', 'lambert_europe'. Default is cylindrical equidistant. (Optional, str) + file_format: 'PNG' + ncores: # Optional, int: number of cores, defaults to 1 + remove_NAs: # Optional, bool: Whether NAs are removed, defaults to FALSE + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/apuiggro/sunset_provenance_clean/sunset/outputs + code_dir: /esarchive/scratch/apuiggro/sunset_provenance_clean/sunset + filesystem: sample + diff --git a/split.R b/split.R index faadafd68051b934cafcc2b8f2c2617dee3f849b..0d4153295b7e2bb4459c30937b887e4dfb2f12ff 100755 --- a/split.R +++ b/split.R @@ -30,6 +30,14 @@ recipe <- prepare_outputs(recipe_file = arguments$recipe, uniqueID = !arguments$disable_unique_ID, restructure = FALSE) +#Create base ROCRATE +if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Run$autosubmit && recipe$Analysis$Provenance) { + source("tools/create_rocrate.R") + create_rocrate(recipe) + } +} + # Split recipe into atomic recipes run_parameters <- divide_recipe(recipe) diff --git a/tools/create_rocrate.R b/tools/create_rocrate.R new file mode 100644 index 0000000000000000000000000000000000000000..a3b547151bdeea3209afa1e375901f5b5df9aedf --- /dev/null +++ b/tools/create_rocrate.R @@ -0,0 +1,51 @@ + +create_rocrate <- function(recipe){ + + library(yaml) + library(jsonlite) + + #TODO: Add checks for orcid and Info entries in the recipe! + + #Also have to add create action entity having each file as "result" + json_content <- list( + "@graph" = list( + list( + "@id" = "./", + "license" = "Apache-2.0", + "creator" = list( + "@id" = recipe$Run$auto_conf$orcid + ) + ), + list( + "@id" = recipe$Run$auto_conf$orcid, + "@type" = "Person", + "affiliation" = list( + "@id" = "https://ror.org/05sd8tv96" + ) + ) + ) + ) + + rocrate <- list( + ROCRATE = list( + INPUTS = list(), + OUTPUTS = list("*/*.gif"), + PATCH = list("|") + ) + ) + + out_dir <- paste0(recipe$Run$output_dir, "/outputs/provenance") + + # Create the directory if it does not exist + if (!dir.exists(out_dir)) { + dir.create(out_dir, recursive = TRUE) + } + + if (!dir.exists(paste0(recipe$Run$output_dir, "/logs/recipes"))) { + dir.create(out_dir, recursive = TRUE) + } + + rocrate$ROCRATE$PATCH <- json_content + output_file_name <- paste0(out_dir, "/rocrate", ".yml") + write_yaml(rocrate, output_file_name) +} diff --git a/tools/divide_recipe.R b/tools/divide_recipe.R index 2da7ce1f739990f758e584991d1adce5bd2bf517..d43a8be92d490075ca8b87a3b792a9ac0c3991d7 100644 --- a/tools/divide_recipe.R +++ b/tools/divide_recipe.R @@ -9,6 +9,7 @@ divide_recipe <- function(recipe) { recipe$name, collapse = ""))), Analysis = list(Horizon = recipe$Analysis$Horizon, + Provenance = recipe$Analysis$Provenance, Variables = NULL, Datasets = NULL, Time = NULL, @@ -26,6 +27,9 @@ divide_recipe <- function(recipe) { beta_recipe$Run$tmp_dir <- recipe$Run$tmp_dir } + #expid + expid <- recipe$Run$auto_conf$expid + # duplicate recipe by independent variables: # If a single variable is not given inside a list, rebuild structure if (any(c("name", "freq", "units") %in% names(recipe$Analysis$Variables))) { @@ -217,6 +221,8 @@ divide_recipe <- function(recipe) { chunk_to_recipe <- list() split_to_recipe <- list() total_models <- length(recipe$Analysis$Datasets$System) + recipe_names <- list() + out_dirs <- list() for (reci in 1:length(all_recipes)) { ## TODO: Document recipe_model <- paste0("sys-", @@ -226,8 +232,8 @@ divide_recipe <- function(recipe) { "_var-", all_recipes[[reci]]$Analysis$Variables$name, "_reg-", all_recipes[[reci]]$Analysis$Region$name, "_sdate-", all_recipes[[reci]]$Analysis$Time$sdate) + recipe_name <- paste0(recipe_model, recipe_split) - if (all_recipes[[reci]]$Analysis$Datasets$System$name == 'Multimodel') { recipe_dir <- paste0(recipe$Run$output_dir, "/logs/recipes/multimodel/") @@ -238,9 +244,23 @@ divide_recipe <- function(recipe) { chunk_to_recipe[chunk] <- recipe_name chunk <- chunk + 1 } + write_yaml(all_recipes[[reci]], paste0(recipe_dir, "atomic_recipe_", recipe_name, ".yml")) + + recipe_names <- c(recipe_names, recipe_name) + + out_dir <- recipe$Run$output_dir + } + + #Add to the rocrate.yml individual files + if (!is.null(recipe$Analysis$Provenance)) { + if (recipe$Run$autosubmit && recipe$Analysis$Provenance) { + source("tools/write_rocrate.R") + write_rocrate(recipe_names, out_dir, expid, recipe$Run$code_dir) + } + } # Print information for user info(recipe$Run$logger, @@ -256,4 +276,3 @@ divide_recipe <- function(recipe) { chunk_to_recipe = chunk_to_recipe, split_to_recipe = split_to_recipe)) } - diff --git a/tools/modify_rocrate.R b/tools/modify_rocrate.R new file mode 100644 index 0000000000000000000000000000000000000000..574f3c81e1d8d134ea165f9228bd0eb71457eb6d --- /dev/null +++ b/tools/modify_rocrate.R @@ -0,0 +1,85 @@ +library(jsonlite) + +write_rocrate <- function(expid) { + cat("Starting write_rocrate function...\n") + + roc_dir <- paste0("/esarchive/autosubmit/", expid, "/ro-crate-metadata.json") + + # Check if file exists + if (!file.exists(roc_dir)) { + stop("Error: JSON file does not exist at ", roc_dir) + } + + cat("Reading JSON file from:", roc_dir, "\n") + rocrate <- fromJSON(roc_dir, simplifyVector = FALSE) + + # Add the new entity definition if it doesn't exist + new_entity_id <- "https://w3id.org/ro/wfrun/provenance/0.5" + entity_exists <- any(sapply(rocrate$`@graph`, function(x) x$`@id` == new_entity_id)) + + if (!entity_exists) { + new_entity <- list( + `@id` = new_entity_id, + `@type` = "CreativeWork", + name = "Process Run Crate", + version = "0.5" + ) + rocrate$`@graph` <- c(rocrate$`@graph`, list(new_entity)) + } else { + } + + # Function to add the new profile to conformsTo + add_profile_to_conformsTo <- function(entity) { + if (!is.null(entity$conformsTo)) { + exists_already <- any(sapply(entity$conformsTo, function(x) x$`@id` == new_entity_id)) + if (!exists_already) { + entity$conformsTo <- c(entity$conformsTo, list(list(`@id` = new_entity_id))) + } + } + return(entity) + } + + # Update relevant entities + for (i in seq_along(rocrate$`@graph`)) { + if (rocrate$`@graph`[[i]]$`@id` %in% c("./", "ro-crate-metadata.json")) { + rocrate$`@graph`[[i]] <- add_profile_to_conformsTo(rocrate$`@graph`[[i]]) + } + } + + # Find and modify the experiment_data.yml entity + for (i in seq_along(rocrate$`@graph`)) { + entity <- rocrate$`@graph`[[i]] + if (!is.null(entity$`@id`) && entity$`@id` == "conf/metadata/experiment_data.yml") { + cat("Modifying experiment_data.yml entity...\n") + + if (!"HowTo" %in% entity$`@type`) { + rocrate$`@graph`[[i]]$`@type` <- c(entity$`@type`, "HowTo") + } + + how_to_step_ids <- list() + for (step_entity in rocrate$`@graph`) { + if (!is.null(step_entity$`@type`) && "HowToStep" %in% step_entity$`@type`) { + how_to_step_ids <- c(how_to_step_ids, list(list(`@id` = step_entity$`@id`))) + } + } + + if (length(how_to_step_ids) > 0) { + rocrate$`@graph`[[i]]$step <- how_to_step_ids + } + break + } + } + + output_file <- paste0("/esarchive/autosubmit/", expid, "/ro-crate-metadata.json") + + cat("Writing modified JSON to:", output_file, "\n") + write_json(rocrate, output_file, auto_unbox = TRUE, pretty = TRUE) +} + +# Command-line execution +args <- commandArgs(trailingOnly = TRUE) +if (length(args) == 1) { + write_rocrate(args[1]) +} else { + stop("Usage: Rscript modify_rocrate.R ") +} diff --git a/tools/write_autosubmit_conf.R b/tools/write_autosubmit_conf.R index a6bc5aa04690b5a196c31fd694883bfe5ebb7fcd..7ce1f1e3076af759f4ae66170789202f23332719 100644 --- a/tools/write_autosubmit_conf.R +++ b/tools/write_autosubmit_conf.R @@ -54,6 +54,7 @@ write_autosubmit_conf <- function(recipe, nchunks, # Section 2: expdef ## expid, numchunks, project_type?, project_destination = auto-s2s conf$DEFAULT$EXPID <- expid + conf$DEFAULT$HPCARCH <- platform conf$experiment$DATELIST <- format(Sys.Date(), "%Y%m%d") conf$experiment$NUMCHUNKS <- nchunks conf$local$PROJECT_PATH <- recipe$Run$code_dir @@ -134,9 +135,18 @@ write_autosubmit_conf <- function(recipe, nchunks, if (!is.null(tmp_dir)) { conf$JOBS$transfer_results$DEPENDENCIES <- last_job conf$JOBS$transfer_results$NOTIFY_ON <- notify_on + last_job <- "transfer_results" } else { conf$JOBS$transfer_results <- NULL } + #Provenance transfer job: only include if provenance requested + if (recipe$Analysis$Provenance) { + conf$JOBS$transfer_provenance$DEPENDENCIES <- last_job + conf$JOBS$transfer_provenance$NOTIFY_ON <- notify_on + } else { + conf$JOBS$transfer_provenance <- NULL + } + } else if (conf_type == "platforms") { ## TODO: Allow user to choose platform # Section 4: platform configuration @@ -144,7 +154,8 @@ write_autosubmit_conf <- function(recipe, nchunks, conf$Platforms[[platform]]$USER <- hpc_user unused_machines <- names(conf$Platforms)[!names(conf$Platforms) %in% c("transfer", platform)] conf$Platforms[unused_machines] <- NULL - if (!is.null(tmp_dir)) { + if (!is.null(tmp_dir) || + (recipe$Analysis$Provenance)) { conf$Platforms$transfer$USER <- hpc_user } else { conf$Platforms$transfer <- NULL @@ -175,8 +186,7 @@ write_autosubmit_conf <- function(recipe, nchunks, if (recipe$Run$Terminal) { ## TODO: Change SSH message for other environments (outside BSC) info(recipe$Run$logger, - paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run", - "the following commands:")) + paste("Please SSH into bsceshub01 or bsceshub02 and run the following commands:")) info(recipe$Run$logger, paste("module load", auto_specs$module_version)) info(recipe$Run$logger, @@ -185,9 +195,19 @@ write_autosubmit_conf <- function(recipe, nchunks, paste("autosubmit refresh", expid)) info(recipe$Run$logger, paste("nohup autosubmit run", expid, "& disown")) + #When automatic rocrate generation is implemented in autosubmit this can be removed + if (recipe$Analysis$Provenance){ + info(recipe$Run$logger, + paste("To generate the rocrate object (zip file) run the following command + after the execution is finished:")) + info(recipe$Run$logger, + paste("autosubmit archive --rocrate", expid)) + info(recipe$Run$logger, + paste("The zip file will be saved in /esarchive/autosubmit/2024")) + } + } else { - print(paste("Please SSH into bscesautosubmit01 or bscesautosubmit02 and run", - "the following commands:")) + print(paste("Please SSH into bsceshub01 or bsceshub02 and run the following commands:")) print(paste("module load", auto_specs$module_version)) print(paste("autosubmit create", expid)) print(paste("autosubmit refresh", expid)) diff --git a/tools/write_rocrate.R b/tools/write_rocrate.R new file mode 100644 index 0000000000000000000000000000000000000000..f3d57b5ec57e4d8fbb220960e4e32c4311648add --- /dev/null +++ b/tools/write_rocrate.R @@ -0,0 +1,87 @@ +write_rocrate <- function(recipes, out_dir, expid, repo_dir) { + library(yaml) + library(jsonlite) + + rocrate <- read_yaml(paste0(out_dir, "/outputs/provenance/rocrate.yml")) + + #Add modified entities that will need to be updated when merged to the full ro-crate-metadata.json + new_profile_entity <- list( + "@id" = "https://w3id.org/ro/wfrun/provenance/0.5", + "@type" = "CreativeWork", + name = "Process Run Crate", + version = "0.5" + ) + + rocrate$ROCRATE$PATCH$'@graph' <- c(rocrate$ROCRATE$PATCH$'@graph', list(new_profile_entity)) + + # Add or find Organize Action entity + org_action_idx <- which(sapply(rocrate$ROCRATE$PATCH$'@graph', function(x) x$'@id' == "#organize-action-sunset")) + if (length(org_action_idx) == 0) { + json_organize_action <- list( + "@id" = "#organize-action-sunset", + "@type" = "OrganizeAction", + "agent" = list("@id" = rocrate$ROCRATE$PATCH$'@graph'[[1]]$creator$'@id'), + "instrument" = list("@id" = "conf/metadata/experiment_data.yml"), + "name" = "Run of SUNSET verification jobs", + "object" = list(), + "result" = "#create-action" + ) + rocrate$ROCRATE$PATCH$'@graph' <- c(rocrate$ROCRATE$PATCH$'@graph', list(json_organize_action)) + org_action_idx <- length(rocrate$ROCRATE$PATCH$'@graph') + } + + for (i in seq_along(recipes)) { + recipe_name <- recipes[i] + recipe <- paste0("atomic_recipe_", recipe_name, ".yml") + recipe_dir <- paste0("/tmp/", basename(out_dir), "/logs/recipes/", recipe) + json_name <- paste0("main_provenance_", recipe_name, ".json") + json_dir <- paste0("/tmp/", basename(out_dir), "/outputs/provenance/", json_name) + + # Add control action reference to organize action + control_id <- paste0("#control-action-sunset-", i) + rocrate$ROCRATE$PATCH$'@graph'[[org_action_idx]]$object <- c( + rocrate$ROCRATE$PATCH$'@graph'[[org_action_idx]]$object, + list(list("@id" = control_id)) + ) + + # Add new entities + new_entities <- list( + list( + "@id" = control_id, + "@type" = "ControlAction", + "instrument" = list("@id" = paste0("#how-to-step-sunset-", i)), + "name" = "Orchestrate SUNSET job run", + "object" = list("@id" = paste0("#sunset-create-action-", i)) + ), + list( + "@id" = paste0("#how-to-step-sunset-", i), + "@type" = "HowToStep", + "workExample" = list("@id" = paste0("file:///", repo_dir)) + ), + list( + "@id" = paste0("#metaclip-create-action-", i), + "@type" = "CreateAction", + "instrument" = list("@id" = paste0("file://", repo_dir)), + "name" = "SUNSET job run", + "description" = "SUNSET job execution + Provenance based on METACLIP", + "startTime" = format(Sys.time(), "%Y-%m-%d %H:%M"), + "object" = list(list("@id" = recipe_dir)), + "result" = list(list("@id" = json_dir)) + ) + ) + + rocrate$ROCRATE$PATCH$'@graph' <- c(rocrate$ROCRATE$PATCH$'@graph', new_entities) + rocrate$ROCRATE$OUTPUTS <- c(rocrate$ROCRATE$OUTPUTS, + paste0("file://", json_dir), + paste0("file://", recipe_dir)) + } + + rocrate$ROCRATE$PATCH <- toJSON(rocrate$ROCRATE$PATCH, pretty = TRUE, auto_unbox = TRUE) + + # Remove rocrate.yml from SUNSET outputs folder + rocrate_path <- paste0(out_dir, "/outputs/provenance/rocrate.yml") + if (file.exists(rocrate_path)) { + file.remove(rocrate_path) +} + write_yaml(rocrate, paste0("/esarchive/autosubmit/", expid, "/conf/rocrate.yml")) +} \ No newline at end of file diff --git a/use_cases/ex1_4_provenance_autosubmit/Figures/autosubmit_graph.png b/use_cases/ex1_4_provenance_autosubmit/Figures/autosubmit_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..227850151d2721e21aad5c4fd6c2ad0f38267e69 Binary files /dev/null and b/use_cases/ex1_4_provenance_autosubmit/Figures/autosubmit_graph.png differ diff --git a/use_cases/ex1_4_provenance_autosubmit/Figures/autosubmit_graph2.png b/use_cases/ex1_4_provenance_autosubmit/Figures/autosubmit_graph2.png new file mode 100644 index 0000000000000000000000000000000000000000..438172f40ba4d7f7f9b919bd2cbf09b9d0dddaa8 Binary files /dev/null and b/use_cases/ex1_4_provenance_autosubmit/Figures/autosubmit_graph2.png differ diff --git a/use_cases/ex1_4_provenance_autosubmit/Figures/metaclip_graph.png b/use_cases/ex1_4_provenance_autosubmit/Figures/metaclip_graph.png new file mode 100644 index 0000000000000000000000000000000000000000..be72cb7a47a900142b35f79d65a4c17e9b6d0987 Binary files /dev/null and b/use_cases/ex1_4_provenance_autosubmit/Figures/metaclip_graph.png differ diff --git a/use_cases/ex1_4_provenance_autosubmit/Figures/scorecard-1_ECMWF-SEAS51_ERA5_tas_1993-2003.png b/use_cases/ex1_4_provenance_autosubmit/Figures/scorecard-1_ECMWF-SEAS51_ERA5_tas_1993-2003.png new file mode 100644 index 0000000000000000000000000000000000000000..cac4cb286d1c7da18571e9fee2c5f17f6399f04a Binary files /dev/null and b/use_cases/ex1_4_provenance_autosubmit/Figures/scorecard-1_ECMWF-SEAS51_ERA5_tas_1993-2003.png differ diff --git a/use_cases/ex1_4_provenance_autosubmit/Figures/scorecard-2_ECMWF-SEAS51_ERA5_tas_1993-2003.png b/use_cases/ex1_4_provenance_autosubmit/Figures/scorecard-2_ECMWF-SEAS51_ERA5_tas_1993-2003.png new file mode 100644 index 0000000000000000000000000000000000000000..c4d25fa4e2faee55d694e9d8af7d7c7417dab0c0 Binary files /dev/null and b/use_cases/ex1_4_provenance_autosubmit/Figures/scorecard-2_ECMWF-SEAS51_ERA5_tas_1993-2003.png differ diff --git a/use_cases/ex1_4_provenance_autosubmit/Files/ro-crate-metadata.json b/use_cases/ex1_4_provenance_autosubmit/Files/ro-crate-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ff3d4d44977be490204291d6a3810e1096c57c2d --- /dev/null +++ b/use_cases/ex1_4_provenance_autosubmit/Files/ro-crate-metadata.json @@ -0,0 +1,2956 @@ +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "conformsTo": [ + { + "@id": "https://w3id.org/ro/wfrun/process/0.1" + }, + { + "@id": "https://w3id.org/ro/wfrun/workflow/0.1" + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" + } + ], + "creator": { + "@id": "https://orcid.org/0009-0006-0707-6448" + }, + "datePublished": "2024-11-29T12:01:02+00:00", + "description": "Test multiprov", + "hasPart": [ + { + "@id": "conf/metadata/experiment_data.yml" + }, + { + "@id": "conf/rocrate.yml" + }, + { + "@id": "conf/.jobss_a83j.yml.swp" + }, + { + "@id": "conf/jobs_a83j.yml" + }, + { + "@id": "conf/.jobs_a83j.yml.swn" + }, + { + "@id": "conf/.jobs_a83j.yml.swp" + }, + { + "@id": "conf/autosubmit_a83j.yml" + }, + { + "@id": "conf/proj_a83j.yml" + }, + { + "@id": "conf/.as_misc.yml.swp" + }, + { + "@id": "conf/.platforms_a83j.yml.swo" + }, + { + "@id": "conf/.version.yml.swp" + }, + { + "@id": "conf/version.yml" + }, + { + "@id": "conf/.nohup.out.swp" + }, + { + "@id": "conf/platforms_a83j.yml" + }, + { + "@id": "conf/.platforms_a83j.yml.swn" + }, + { + "@id": "conf/.jobs_a83j.yml.swo" + }, + { + "@id": "conf/expdef_a83j.yml" + }, + { + "@id": "conf/.platforms_a83j.yml.swp" + }, + { + "@id": "conf/.expdef_a83j.yml.swp" + }, + { + "@id": "conf/metadata/experiment_data.yml.bak" + }, + { + "@id": "conf/metadata/.experiment_data.yml.swp" + }, + { + "@id": "conf/metadata/.experiment_data.yml.swn" + }, + { + "@id": "conf/metadata/.experiment_data.yml.swo" + }, + { + "@id": "conf/" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE_COMPLETED" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES_STAT" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION.cmd" + }, + { + "@id": "tmp/autosubmit.lock" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION_STAT" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION_COMPLETED" + }, + { + "@id": "tmp/a83j_SCORECARDS_COMPLETED" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION_STAT" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS_COMPLETED" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES_COMPLETED" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE_STAT" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION.cmd" + }, + { + "@id": "tmp/a83j_SCORECARDS_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE.cmd" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS.cmd" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION_STAT" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION_STAT" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES.cmd" + }, + { + "@id": "tmp/a83j_SCORECARDS.cmd" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION.cmd" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION_TOTAL_STATS" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION_COMPLETED" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION_COMPLETED" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION_COMPLETED" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION_COMPLETED" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION_STAT" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION.cmd" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION.cmd" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION_COMPLETED" + }, + { + "@id": "tmp/a83j_SCORECARDS_STAT" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS_STAT" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION.cmd" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION_STAT" + }, + { + "@id": "tmp/ASLOGS/20241128_141956_run.log" + }, + { + "@id": "tmp/ASLOGS/jobs_active_status.log" + }, + { + "@id": "tmp/ASLOGS/submit_MN5.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_130252_create.log" + }, + { + "@id": "tmp/ASLOGS/20241128_141956_run_err.log" + }, + { + "@id": "tmp/ASLOGS/submit_772b01e3ed7bcababb7a969075937e28.sh" + }, + { + "@id": "tmp/ASLOGS/20241127_185127_run.log" + }, + { + "@id": "tmp/ASLOGS/20241127_185127_run_err.log" + }, + { + "@id": "tmp/ASLOGS/20241129_102021_refresh_err.log" + }, + { + "@id": "tmp/ASLOGS/20241127_184831_create.log" + }, + { + "@id": "tmp/ASLOGS/submit_783dceb25f287729434a53d54ee0bbb0.sh" + }, + { + "@id": "tmp/ASLOGS/submit_5871e3014e89c27529b6edc0ba6850cb.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_120907_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104106_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104004_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104308_refresh.log" + }, + { + "@id": "tmp/ASLOGS/submit_82136b9beaa9e37a885fa1323629b7f2.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_130405_run.log" + }, + { + "@id": "tmp/ASLOGS/submit_53703b48752de3574abdac1a8a9e7045.sh" + }, + { + "@id": "tmp/ASLOGS/20241127_190041_run.log" + }, + { + "@id": "tmp/ASLOGS/jobs_failed_status.log" + }, + { + "@id": "tmp/ASLOGS/20241128_121216_run.log" + }, + { + "@id": "tmp/ASLOGS/20241128_141926_refresh.log" + }, + { + "@id": "tmp/ASLOGS/20241128_141741_create.log" + }, + { + "@id": "tmp/ASLOGS/20241129_102044_run.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104053_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_093956_refresh_err.log" + }, + { + "@id": "tmp/ASLOGS/submit_5c518d28f55b93b7673b3dcc4817c7bf.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_121121_refresh_err.log" + }, + { + "@id": "tmp/ASLOGS/submit_64541dcaa8ce743d974b1eabb0b6857f.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_104308_refresh_err.log" + }, + { + "@id": "tmp/ASLOGS/20241129_101930_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_130347_refresh_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_120907_create.log" + }, + { + "@id": "tmp/ASLOGS/20241127_190041_run_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_094015_run_err.log" + }, + { + "@id": "tmp/ASLOGS/20241129_102044_run_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_121121_refresh.log" + }, + { + "@id": "tmp/ASLOGS/submit_98d7e0bae197e6ac4b5757865e5bfe62.sh" + }, + { + "@id": "tmp/ASLOGS/20241127_184831_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_141926_refresh_err.log" + }, + { + "@id": "tmp/ASLOGS/submit_27d168042092fa83c88a96b7333c5df2.sh" + }, + { + "@id": "tmp/ASLOGS/submit_fa5da7bc7cc0e30546a450f2e1b45ea1.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_094015_run.log" + }, + { + "@id": "tmp/ASLOGS/submit_TRANSFER.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_104322_run_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_093827_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104106_create.log" + }, + { + "@id": "tmp/ASLOGS/20241128_141741_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104138_create.log" + }, + { + "@id": "tmp/ASLOGS/20241128_093956_refresh.log" + }, + { + "@id": "tmp/ASLOGS/submit_9d0952f3fb1ac8d96f077820e72c37c3.sh" + }, + { + "@id": "tmp/ASLOGS/submit_ed8d52dbf464e327e3fe73ae95a774c8.sh" + }, + { + "@id": "tmp/ASLOGS/20241128_104004_create.log" + }, + { + "@id": "tmp/ASLOGS/20241128_130405_run_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_130347_refresh.log" + }, + { + "@id": "tmp/ASLOGS/20241129_101930_create.log" + }, + { + "@id": "tmp/ASLOGS/20241128_130252_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_121216_run_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_093827_create.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104053_create.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104138_create_err.log" + }, + { + "@id": "tmp/ASLOGS/20241128_104322_run.log" + }, + { + "@id": "tmp/ASLOGS/20241129_102021_refresh.log" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/rocrate.yml" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.json" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.json" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.json" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.yml" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/ex1_4-recipe.yml" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.yml" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.yml" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128130416.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128121245.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RESULTS.20241129105711.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128094025.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241129102053.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128094043.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128121245.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128104405.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128130434.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128121227.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128142032.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_3_VERIFICATION.20241129102111.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_PROVENANCE.20241129105742.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128130416.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_3_VERIFICATION.20241129102111.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128130434.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128094025.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128094043.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_PROVENANCE.20241129105742.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241129102053.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128130434.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_1_VERIFICATION.20241129102111.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128142008.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128104332.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128142032.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128142031.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RESULTS.20241129105711.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128094042.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128142032.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_1_VERIFICATION.20241129102111.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128104332.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128130434.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128142008.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128121227.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_2_VERIFICATION.20241129102111.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128142031.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128121245.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_SCORECARDS.20241129105603.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_SCORECARDS.20241129105603.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128142032.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_2_VERIFICATION.20241129102111.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128121245.out" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128104405.err" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128094042.out" + }, + { + "@id": "tmp/" + }, + { + "@id": "plot/a83j_20241128_1040.pdf" + }, + { + "@id": "plot/a83j_20241128_1417.pdf" + }, + { + "@id": "plot/a83j_20241128_1209.pdf" + }, + { + "@id": "plot/a83j_20241127_1848.pdf" + }, + { + "@id": "plot/a83j_20241128_0938.pdf" + }, + { + "@id": "plot/a83j_20241128_1041.pdf" + }, + { + "@id": "plot/a83j_20241128_1303.pdf" + }, + { + "@id": "plot/a83j_20241129_1019.pdf" + }, + { + "@id": "plot/" + }, + { + "@id": "status/a83j_20241128_1303.txt" + }, + { + "@id": "status/a83j_20241128_1209.txt" + }, + { + "@id": "status/a83j_20241128_1417.txt" + }, + { + "@id": "status/a83j_20241129_1020.txt" + }, + { + "@id": "status/a83j_20241128_1040.txt" + }, + { + "@id": "status/a83j_20241128_1041.txt" + }, + { + "@id": "status/a83j_20241128_0938.txt" + }, + { + "@id": "status/a83j_20241127_1848.txt" + }, + { + "@id": "status/" + }, + { + "@id": "pkl/.job_list_a83j.pkl.swp" + }, + { + "@id": "pkl/job_packages_a83j.db" + }, + { + "@id": "pkl/job_list_a83j.pkl" + }, + { + "@id": "pkl/" + }, + { + "@id": "proj/auto-s2s/" + } + ], + "license": "Apache-2.0", + "mainEntity": { + "@id": "conf/metadata/experiment_data.yml" + }, + "mentions": { + "@id": "#create-action" + } + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "./" + }, + "conformsTo": [ + { + "@id": "https://w3id.org/ro/crate/1.1" + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" + } + ] + }, + { + "@id": "https://w3id.org/ro/wfrun/process/0.1", + "@type": "CreativeWork", + "name": "Process Run Crate", + "version": "0.1" + }, + { + "@id": "https://w3id.org/ro/wfrun/workflow/0.1", + "@type": "CreativeWork", + "name": "Workflow Run Crate", + "version": "0.1" + }, + { + "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0", + "@type": "CreativeWork", + "name": "Workflow RO-Crate", + "version": "1.0" + }, + { + "@id": "conf/metadata/experiment_data.yml", + "@type": [ + "File", + "SoftwareSourceCode", + "ComputationalWorkflow" + ], + "hasPart": [ + { + "@id": "file:///esarchive/scratch/apuiggro/sunset" + } + ], + "input": [ + { + "@id": "#DEFAULT.EXPID-param" + }, + { + "@id": "#DEFAULT.HPCARCH-param" + }, + { + "@id": "#EXPERIMENT.DATELIST-param" + }, + { + "@id": "#EXPERIMENT.MEMBERS-param" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZEUNIT-param" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZE-param" + }, + { + "@id": "#EXPERIMENT.NUMCHUNKS-param" + }, + { + "@id": "#EXPERIMENT.CHUNKINI-param" + }, + { + "@id": "#EXPERIMENT.CALENDAR-param" + }, + { + "@id": "#CONFIG.EXPID-param" + }, + { + "@id": "#CONFIG.AUTOSUBMIT_VERSION-param" + }, + { + "@id": "#CONFIG.MAXWAITINGJOBS-param" + }, + { + "@id": "#CONFIG.TOTALJOBS-param" + }, + { + "@id": "#CONFIG.SAFETYSLEEPTIME-param" + }, + { + "@id": "#CONFIG.RETRIALS-param" + }, + { + "@id": "#PROJECT.PROJECT_TYPE-param" + }, + { + "@id": "#PROJECT.PROJECT_DESTINATION-param" + }, + { + "@id": "#LOCAL.PROJECT_PATH-param" + } + ], + "name": "conf/metadata/experiment_data", + "programmingLanguage": { + "@id": "#autosubmit" + } + }, + { + "@id": "#autosubmit", + "@type": "ComputerLanguage", + "alternateName": "AS", + "citation": "https://doi.org/10.1109/HPCSim.2016.7568429", + "name": "Autosubmit", + "url": "https://autosubmit.readthedocs.io/", + "version": "4.1.11" + }, + { + "@id": "conf/rocrate.yml", + "@type": "File", + "contentSize": 2644, + "dateModified": "2024-11-29T09:18:44", + "name": "rocrate.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.jobss_a83j.yml.swp", + "@type": "File", + "contentSize": 4096, + "dateModified": "2024-11-29T11:10:44", + "name": ".jobss_a83j.yml.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/jobs_a83j.yml", + "@type": "File", + "contentSize": 1005, + "dateModified": "2024-11-29T11:11:57", + "name": "jobs_a83j.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.jobs_a83j.yml.swn", + "@type": "File", + "contentSize": 12288, + "dateModified": "2024-11-29T11:47:36", + "name": ".jobs_a83j.yml.swn", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.jobs_a83j.yml.swp", + "@type": "File", + "contentSize": 12288, + "dateModified": "2024-11-29T11:22:25", + "name": ".jobs_a83j.yml.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/autosubmit_a83j.yml", + "@type": "File", + "contentSize": 262, + "dateModified": "2024-11-29T10:46:39", + "name": "autosubmit_a83j.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/proj_a83j.yml", + "@type": "File", + "contentSize": 245, + "dateModified": "2024-11-29T09:18:45", + "name": "proj_a83j.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.as_misc.yml.swp", + "@type": "File", + "contentSize": 12288, + "dateModified": "2024-11-29T11:09:37", + "name": ".as_misc.yml.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.platforms_a83j.yml.swo", + "@type": "File", + "contentSize": 12288, + "dateModified": "2024-11-29T11:23:16", + "name": ".platforms_a83j.yml.swo", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.version.yml.swp", + "@type": "File", + "contentSize": 4096, + "dateModified": "2024-11-29T11:24:29", + "name": ".version.yml.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/version.yml", + "@type": "File", + "contentSize": 37, + "dateModified": "2024-11-27T17:48:38", + "name": "version.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.nohup.out.swp", + "@type": "File", + "contentSize": 16384, + "dateModified": "2024-11-29T11:25:34", + "name": ".nohup.out.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/platforms_a83j.yml", + "@type": "File", + "contentSize": 293, + "dateModified": "2024-11-29T10:22:54", + "name": "platforms_a83j.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.platforms_a83j.yml.swn", + "@type": "File", + "contentSize": 4096, + "dateModified": "2024-11-29T11:35:51", + "name": ".platforms_a83j.yml.swn", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.jobs_a83j.yml.swo", + "@type": "File", + "contentSize": 12288, + "dateModified": "2024-11-29T11:36:35", + "name": ".jobs_a83j.yml.swo", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/expdef_a83j.yml", + "@type": "File", + "contentSize": 578, + "dateModified": "2024-11-29T11:23:35", + "name": "expdef_a83j.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.platforms_a83j.yml.swp", + "@type": "File", + "contentSize": 12288, + "dateModified": "2024-11-29T11:10:19", + "name": ".platforms_a83j.yml.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/.expdef_a83j.yml.swp", + "@type": "File", + "contentSize": 12288, + "dateModified": "2024-11-29T11:24:07", + "name": ".expdef_a83j.yml.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/metadata/experiment_data.yml.bak", + "@type": "File", + "contentSize": 5300, + "dateModified": "2024-11-29T09:57:43", + "name": "experiment_data.yml.bak", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/metadata/.experiment_data.yml.swp", + "@type": "File", + "contentSize": 16384, + "dateModified": "2024-11-29T11:37:51", + "name": ".experiment_data.yml.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/metadata/.experiment_data.yml.swn", + "@type": "File", + "contentSize": 16384, + "dateModified": "2024-11-29T11:47:10", + "name": ".experiment_data.yml.swn", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/metadata/.experiment_data.yml.swo", + "@type": "File", + "contentSize": 4096, + "dateModified": "2024-11-29T11:20:17", + "name": ".experiment_data.yml.swo", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "conf/", + "@type": "Dataset" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES_TOTAL_STATS", + "@type": "File", + "contentSize": 329, + "dateModified": "2024-11-29T09:21:53", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION_TOTAL_STATS", + "@type": "File", + "contentSize": 54, + "dateModified": "2024-11-29T09:55:06", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_3_VERIFICATION_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION_TOTAL_STATS", + "@type": "File", + "contentSize": 188, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:57:58", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_PROVENANCE_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION_TOTAL_STATS", + "@type": "File", + "contentSize": 54, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_1_VERIFICATION_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION_TOTAL_STATS", + "@type": "File", + "contentSize": 262, + "dateModified": "2024-11-28T13:56:26", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS_TOTAL_STATS", + "@type": "File", + "contentSize": 54, + "dateModified": "2024-11-29T09:58:15", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RESULTS_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:21:53", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION.cmd", + "@type": "File", + "contentSize": 2411, + "dateModified": "2024-11-29T09:21:09", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_3_VERIFICATION.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/autosubmit.lock", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T09:43:28", + "encodingFormat": "text/plain", + "name": "autosubmit.lock", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_1_VERIFICATION_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION_TOTAL_STATS", + "@type": "File", + "contentSize": 54, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_2_VERIFICATION_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T13:55:26", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_SCORECARDS_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:57:10", + "encodingFormat": "text/plain", + "name": "a83j_SCORECARDS_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:57:42", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RESULTS_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:21:09", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:58:15", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_PROVENANCE_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION.cmd", + "@type": "File", + "contentSize": 2411, + "dateModified": "2024-11-28T13:20:24", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_SCORECARDS_TOTAL_STATS", + "@type": "File", + "contentSize": 69, + "dateModified": "2024-11-29T09:57:14", + "encodingFormat": "text/plain", + "name": "a83j_SCORECARDS_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE.cmd", + "@type": "File", + "contentSize": 2364, + "dateModified": "2024-11-29T09:57:42", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_PROVENANCE.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_PROVENANCE_TOTAL_STATS", + "@type": "File", + "contentSize": 54, + "dateModified": "2024-11-29T09:58:15", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_PROVENANCE_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS.cmd", + "@type": "File", + "contentSize": 2100, + "dateModified": "2024-11-29T09:57:10", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RESULTS.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_2_VERIFICATION_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_RECIPES.cmd", + "@type": "File", + "contentSize": 2269, + "dateModified": "2024-11-29T09:20:52", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_SCORECARDS.cmd", + "@type": "File", + "contentSize": 2258, + "dateModified": "2024-11-29T09:56:01", + "encodingFormat": "text/plain", + "name": "a83j_SCORECARDS.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION.cmd", + "@type": "File", + "contentSize": 2411, + "dateModified": "2024-11-29T09:21:09", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_2_VERIFICATION.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION_TOTAL_STATS", + "@type": "File", + "contentSize": 151, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION_TOTAL_STATS", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:56:01", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_1_VERIFICATION_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T13:55:10", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:54:56", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_3_VERIFICATION_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_2_VERIFICATION_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T13:55:10", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_3_VERIFICATION_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:55:06", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_3_VERIFICATION_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION.cmd", + "@type": "File", + "contentSize": 2411, + "dateModified": "2024-11-28T13:20:24", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_1_VERIFICATION.cmd", + "@type": "File", + "contentSize": 2411, + "dateModified": "2024-11-29T09:21:09", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_1_VERIFICATION.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241129_fc0_2_VERIFICATION_COMPLETED", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:55:12", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_2_VERIFICATION_COMPLETED", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_SCORECARDS_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:57:14", + "encodingFormat": "text/plain", + "name": "a83j_SCORECARDS_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_TRANSFER_RESULTS_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:58:14", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RESULTS_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_3_VERIFICATION.cmd", + "@type": "File", + "contentSize": 2411, + "dateModified": "2024-11-28T13:20:24", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION.cmd", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/a83j_20241128_fc0_1_VERIFICATION_STAT", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T13:56:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION_STAT", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_141956_run.log", + "@type": "File", + "contentSize": 248976, + "dateModified": "2024-11-28T13:58:25", + "encodingFormat": "text/plain", + "name": "20241128_141956_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/jobs_active_status.log", + "@type": "File", + "contentSize": 607, + "dateModified": "2024-11-29T09:57:58", + "encodingFormat": "text/plain", + "name": "jobs_active_status.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_MN5.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T11:25:30", + "encodingFormat": "text/plain", + "name": "submit_MN5.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_130252_create.log", + "@type": "File", + "contentSize": 3028, + "dateModified": "2024-11-28T12:03:41", + "encodingFormat": "text/plain", + "name": "20241128_130252_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_141956_run_err.log", + "@type": "File", + "contentSize": 116, + "dateModified": "2024-11-28T13:58:14", + "encodingFormat": "text/plain", + "name": "20241128_141956_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_772b01e3ed7bcababb7a969075937e28.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T13:55:33", + "encodingFormat": "text/plain", + "name": "submit_772b01e3ed7bcababb7a969075937e28.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241127_185127_run.log", + "@type": "File", + "contentSize": 3608, + "dateModified": "2024-11-27T17:51:35", + "encodingFormat": "text/plain", + "name": "20241127_185127_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241127_185127_run_err.log", + "@type": "File", + "contentSize": 723, + "dateModified": "2024-11-27T17:51:35", + "encodingFormat": "text/plain", + "name": "20241127_185127_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241129_102021_refresh_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:20:28", + "encodingFormat": "text/plain", + "name": "20241129_102021_refresh_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241127_184831_create.log", + "@type": "File", + "contentSize": 3469, + "dateModified": "2024-11-27T17:49:44", + "encodingFormat": "text/plain", + "name": "20241127_184831_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_783dceb25f287729434a53d54ee0bbb0.sh", + "@type": "File", + "contentSize": 131, + "dateModified": "2024-11-28T13:55:26", + "encodingFormat": "text/plain", + "name": "submit_783dceb25f287729434a53d54ee0bbb0.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_5871e3014e89c27529b6edc0ba6850cb.sh", + "@type": "File", + "contentSize": 444, + "dateModified": "2024-11-28T12:04:32", + "encodingFormat": "text/plain", + "name": "submit_5871e3014e89c27529b6edc0ba6850cb.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_120907_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T11:09:37", + "encodingFormat": "text/plain", + "name": "20241128_120907_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104106_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T09:41:13", + "encodingFormat": "text/plain", + "name": "20241128_104106_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104004_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T09:40:15", + "encodingFormat": "text/plain", + "name": "20241128_104004_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104308_refresh.log", + "@type": "File", + "contentSize": 183, + "dateModified": "2024-11-28T09:43:17", + "encodingFormat": "text/plain", + "name": "20241128_104308_refresh.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_82136b9beaa9e37a885fa1323629b7f2.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T12:04:34", + "encodingFormat": "text/plain", + "name": "submit_82136b9beaa9e37a885fa1323629b7f2.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_130405_run.log", + "@type": "File", + "contentSize": 252678, + "dateModified": "2024-11-28T12:44:01", + "encodingFormat": "text/plain", + "name": "20241128_130405_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_53703b48752de3574abdac1a8a9e7045.sh", + "@type": "File", + "contentSize": 444, + "dateModified": "2024-11-28T09:44:03", + "encodingFormat": "text/plain", + "name": "submit_53703b48752de3574abdac1a8a9e7045.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241127_190041_run.log", + "@type": "File", + "contentSize": 3608, + "dateModified": "2024-11-27T18:00:49", + "encodingFormat": "text/plain", + "name": "20241127_190041_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/jobs_failed_status.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:57:58", + "encodingFormat": "text/plain", + "name": "jobs_failed_status.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_121216_run.log", + "@type": "File", + "contentSize": 229861, + "dateModified": "2024-11-28T11:49:15", + "encodingFormat": "text/plain", + "name": "20241128_121216_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_141926_refresh.log", + "@type": "File", + "contentSize": 183, + "dateModified": "2024-11-28T13:19:35", + "encodingFormat": "text/plain", + "name": "20241128_141926_refresh.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_141741_create.log", + "@type": "File", + "contentSize": 3028, + "dateModified": "2024-11-28T13:19:16", + "encodingFormat": "text/plain", + "name": "20241128_141741_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241129_102044_run.log", + "@type": "File", + "contentSize": 251694, + "dateModified": "2024-11-29T09:58:17", + "encodingFormat": "text/plain", + "name": "20241129_102044_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104053_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T09:41:00", + "encodingFormat": "text/plain", + "name": "20241128_104053_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_093956_refresh_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T08:40:02", + "encodingFormat": "text/plain", + "name": "20241128_093956_refresh_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_5c518d28f55b93b7673b3dcc4817c7bf.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T11:12:43", + "encodingFormat": "text/plain", + "name": "submit_5c518d28f55b93b7673b3dcc4817c7bf.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_121121_refresh_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T11:11:29", + "encodingFormat": "text/plain", + "name": "20241128_121121_refresh_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_64541dcaa8ce743d974b1eabb0b6857f.sh", + "@type": "File", + "contentSize": 444, + "dateModified": "2024-11-28T11:12:43", + "encodingFormat": "text/plain", + "name": "submit_64541dcaa8ce743d974b1eabb0b6857f.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104308_refresh_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T09:43:16", + "encodingFormat": "text/plain", + "name": "20241128_104308_refresh_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241129_101930_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:19:55", + "encodingFormat": "text/plain", + "name": "20241129_101930_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_130347_refresh_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T12:03:58", + "encodingFormat": "text/plain", + "name": "20241128_130347_refresh_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_120907_create.log", + "@type": "File", + "contentSize": 3110, + "dateModified": "2024-11-28T11:11:13", + "encodingFormat": "text/plain", + "name": "20241128_120907_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241127_190041_run_err.log", + "@type": "File", + "contentSize": 723, + "dateModified": "2024-11-27T18:00:49", + "encodingFormat": "text/plain", + "name": "20241127_190041_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_094015_run_err.log", + "@type": "File", + "contentSize": 399, + "dateModified": "2024-11-28T09:12:56", + "encodingFormat": "text/plain", + "name": "20241128_094015_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241129_102044_run_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:20:50", + "encodingFormat": "text/plain", + "name": "20241129_102044_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_121121_refresh.log", + "@type": "File", + "contentSize": 183, + "dateModified": "2024-11-28T11:11:31", + "encodingFormat": "text/plain", + "name": "20241128_121121_refresh.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_98d7e0bae197e6ac4b5757865e5bfe62.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T09:44:06", + "encodingFormat": "text/plain", + "name": "submit_98d7e0bae197e6ac4b5757865e5bfe62.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241127_184831_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-27T17:48:38", + "encodingFormat": "text/plain", + "name": "20241127_184831_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_141926_refresh_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T13:19:33", + "encodingFormat": "text/plain", + "name": "20241128_141926_refresh_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_27d168042092fa83c88a96b7333c5df2.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T08:40:41", + "encodingFormat": "text/plain", + "name": "submit_27d168042092fa83c88a96b7333c5df2.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_fa5da7bc7cc0e30546a450f2e1b45ea1.sh", + "@type": "File", + "contentSize": 140, + "dateModified": "2024-11-29T09:57:42", + "encodingFormat": "text/plain", + "name": "submit_fa5da7bc7cc0e30546a450f2e1b45ea1.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_094015_run.log", + "@type": "File", + "contentSize": 195765, + "dateModified": "2024-11-28T09:13:07", + "encodingFormat": "text/plain", + "name": "20241128_094015_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_TRANSFER.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T11:25:30", + "encodingFormat": "text/plain", + "name": "submit_TRANSFER.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104322_run_err.log", + "@type": "File", + "contentSize": 399, + "dateModified": "2024-11-28T10:25:28", + "encodingFormat": "text/plain", + "name": "20241128_104322_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_093827_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T08:38:56", + "encodingFormat": "text/plain", + "name": "20241128_093827_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104106_create.log", + "@type": "File", + "contentSize": 1651, + "dateModified": "2024-11-28T09:41:14", + "encodingFormat": "text/plain", + "name": "20241128_104106_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_141741_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T13:17:56", + "encodingFormat": "text/plain", + "name": "20241128_141741_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104138_create.log", + "@type": "File", + "contentSize": 3028, + "dateModified": "2024-11-28T09:41:58", + "encodingFormat": "text/plain", + "name": "20241128_104138_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_093956_refresh.log", + "@type": "File", + "contentSize": 183, + "dateModified": "2024-11-28T08:40:12", + "encodingFormat": "text/plain", + "name": "20241128_093956_refresh.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_9d0952f3fb1ac8d96f077820e72c37c3.sh", + "@type": "File", + "contentSize": 444, + "dateModified": "2024-11-28T08:40:41", + "encodingFormat": "text/plain", + "name": "submit_9d0952f3fb1ac8d96f077820e72c37c3.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/submit_ed8d52dbf464e327e3fe73ae95a774c8.sh", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-29T09:57:42", + "encodingFormat": "text/plain", + "name": "submit_ed8d52dbf464e327e3fe73ae95a774c8.sh", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104004_create.log", + "@type": "File", + "contentSize": 2978, + "dateModified": "2024-11-28T09:40:17", + "encodingFormat": "text/plain", + "name": "20241128_104004_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_130405_run_err.log", + "@type": "File", + "contentSize": 399, + "dateModified": "2024-11-28T12:43:51", + "encodingFormat": "text/plain", + "name": "20241128_130405_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_130347_refresh.log", + "@type": "File", + "contentSize": 183, + "dateModified": "2024-11-28T12:04:00", + "encodingFormat": "text/plain", + "name": "20241128_130347_refresh.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241129_101930_create.log", + "@type": "File", + "contentSize": 3110, + "dateModified": "2024-11-29T09:20:19", + "encodingFormat": "text/plain", + "name": "20241129_101930_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_130252_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T12:03:23", + "encodingFormat": "text/plain", + "name": "20241128_130252_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_121216_run_err.log", + "@type": "File", + "contentSize": 399, + "dateModified": "2024-11-28T11:49:04", + "encodingFormat": "text/plain", + "name": "20241128_121216_run_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_093827_create.log", + "@type": "File", + "contentSize": 3261, + "dateModified": "2024-11-28T08:39:17", + "encodingFormat": "text/plain", + "name": "20241128_093827_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104053_create.log", + "@type": "File", + "contentSize": 1651, + "dateModified": "2024-11-28T09:41:01", + "encodingFormat": "text/plain", + "name": "20241128_104053_create.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104138_create_err.log", + "@type": "File", + "contentSize": 0, + "dateModified": "2024-11-28T09:41:45", + "encodingFormat": "text/plain", + "name": "20241128_104138_create_err.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241128_104322_run.log", + "@type": "File", + "contentSize": 254557, + "dateModified": "2024-11-28T10:25:39", + "encodingFormat": "text/plain", + "name": "20241128_104322_run.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ASLOGS/20241129_102021_refresh.log", + "@type": "File", + "contentSize": 183, + "dateModified": "2024-11-29T09:20:35", + "encodingFormat": "text/plain", + "name": "20241129_102021_refresh.log", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/rocrate.yml", + "@type": "File", + "contentSize": 2644, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "rocrate.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.json", + "@type": "File", + "contentSize": 16803, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.json", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.json", + "@type": "File", + "contentSize": 16803, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.json", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.json", + "@type": "File", + "contentSize": 16803, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.json", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.yml", + "@type": "File", + "contentSize": 2122, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/ex1_4-recipe.yml", + "@type": "File", + "contentSize": 2638, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "ex1_4-recipe.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.yml", + "@type": "File", + "contentSize": 2122, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/ex1_4-recipe_20241129101842/logs/recipes/atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.yml", + "@type": "File", + "contentSize": 2122, + "dateModified": "2024-11-29T09:57:45", + "encodingFormat": "text/plain", + "name": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.yml", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128130416.err", + "@type": "File", + "contentSize": 3967, + "dateModified": "2024-11-28T12:05:16", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128130416.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128121245.err", + "@type": "File", + "contentSize": 32566, + "dateModified": "2024-11-28T11:48:36", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION.20241128121245.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RESULTS.20241129105711.err", + "@type": "File", + "contentSize": 2234, + "dateModified": "2024-11-29T09:58:14", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RESULTS.20241129105711.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128094025.err", + "@type": "File", + "contentSize": 3967, + "dateModified": "2024-11-28T08:41:24", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128094025.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241129102053.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:21:53", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241129102053.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128094043.out", + "@type": "File", + "contentSize": 2331, + "dateModified": "2024-11-28T09:12:32", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION.20241128094043.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128121245.err", + "@type": "File", + "contentSize": 32566, + "dateModified": "2024-11-28T11:48:36", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128121245.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128104405.out", + "@type": "File", + "contentSize": 5043, + "dateModified": "2024-11-28T10:24:41", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128104405.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128130434.out", + "@type": "File", + "contentSize": 5000, + "dateModified": "2024-11-28T12:43:26", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION.20241128130434.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128121227.err", + "@type": "File", + "contentSize": 3967, + "dateModified": "2024-11-28T11:13:26", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128121227.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128142032.err", + "@type": "File", + "contentSize": 31772, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION.20241128142032.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_3_VERIFICATION.20241129102111.out", + "@type": "File", + "contentSize": 2627, + "dateModified": "2024-11-29T09:55:06", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_3_VERIFICATION.20241129102111.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_PROVENANCE.20241129105742.err", + "@type": "File", + "contentSize": 4012, + "dateModified": "2024-11-29T09:58:15", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_PROVENANCE.20241129105742.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128130416.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T12:05:16", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128130416.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_3_VERIFICATION.20241129102111.err", + "@type": "File", + "contentSize": 31326, + "dateModified": "2024-11-29T09:55:06", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_3_VERIFICATION.20241129102111.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128130434.err", + "@type": "File", + "contentSize": 32566, + "dateModified": "2024-11-28T12:40:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128130434.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128094025.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T08:41:24", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128094025.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128094043.err", + "@type": "File", + "contentSize": 31343, + "dateModified": "2024-11-28T09:12:32", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION.20241128094043.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_PROVENANCE.20241129105742.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:58:15", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_PROVENANCE.20241129105742.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241129102053.err", + "@type": "File", + "contentSize": 3967, + "dateModified": "2024-11-29T09:21:53", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241129102053.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128130434.out", + "@type": "File", + "contentSize": 5010, + "dateModified": "2024-11-28T12:40:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128130434.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_1_VERIFICATION.20241129102111.err", + "@type": "File", + "contentSize": 32164, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_1_VERIFICATION.20241129102111.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128142008.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T13:21:08", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128142008.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128104332.err", + "@type": "File", + "contentSize": 3967, + "dateModified": "2024-11-28T09:44:32", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128104332.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128142032.err", + "@type": "File", + "contentSize": 32610, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION.20241128142032.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128142031.err", + "@type": "File", + "contentSize": 32610, + "dateModified": "2024-11-28T13:56:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128142031.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RESULTS.20241129105711.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-29T09:58:14", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RESULTS.20241129105711.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128094042.err", + "@type": "File", + "contentSize": 32194, + "dateModified": "2024-11-28T09:12:32", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128094042.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128142032.out", + "@type": "File", + "contentSize": 5105, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION.20241128142032.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_1_VERIFICATION.20241129102111.out", + "@type": "File", + "contentSize": 2637, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_1_VERIFICATION.20241129102111.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128104332.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T09:44:32", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128104332.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_3_VERIFICATION.20241128130434.err", + "@type": "File", + "contentSize": 31728, + "dateModified": "2024-11-28T12:43:26", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_3_VERIFICATION.20241128130434.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128142008.err", + "@type": "File", + "contentSize": 3967, + "dateModified": "2024-11-28T13:21:08", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128142008.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_TRANSFER_RECIPES.20241128121227.out", + "@type": "File", + "contentSize": 22, + "dateModified": "2024-11-28T11:13:26", + "encodingFormat": "text/plain", + "name": "a83j_TRANSFER_RECIPES.20241128121227.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_2_VERIFICATION.20241129102111.out", + "@type": "File", + "contentSize": 2631, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_2_VERIFICATION.20241129102111.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128142031.out", + "@type": "File", + "contentSize": 5115, + "dateModified": "2024-11-28T13:56:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128142031.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128121245.out", + "@type": "File", + "contentSize": 5043, + "dateModified": "2024-11-28T11:48:36", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128121245.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_SCORECARDS.20241129105603.out", + "@type": "File", + "contentSize": 130, + "dateModified": "2024-11-29T09:57:14", + "encodingFormat": "text/plain", + "name": "a83j_SCORECARDS.20241129105603.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_SCORECARDS.20241129105603.err", + "@type": "File", + "contentSize": 27198, + "dateModified": "2024-11-29T09:57:14", + "encodingFormat": "text/plain", + "name": "a83j_SCORECARDS.20241129105603.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128142032.out", + "@type": "File", + "contentSize": 5109, + "dateModified": "2024-11-28T13:55:25", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION.20241128142032.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241129_fc0_2_VERIFICATION.20241129102111.err", + "@type": "File", + "contentSize": 32164, + "dateModified": "2024-11-29T09:56:07", + "encodingFormat": "text/plain", + "name": "a83j_20241129_fc0_2_VERIFICATION.20241129102111.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_2_VERIFICATION.20241128121245.out", + "@type": "File", + "contentSize": 5037, + "dateModified": "2024-11-28T11:48:36", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_2_VERIFICATION.20241128121245.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128104405.err", + "@type": "File", + "contentSize": 32566, + "dateModified": "2024-11-28T10:24:41", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128104405.err", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/LOG_a83j/a83j_20241128_fc0_1_VERIFICATION.20241128094042.out", + "@type": "File", + "contentSize": 2341, + "dateModified": "2024-11-28T09:12:32", + "encodingFormat": "text/plain", + "name": "a83j_20241128_fc0_1_VERIFICATION.20241128094042.out", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "tmp/", + "@type": "Dataset" + }, + { + "@id": "plot/a83j_20241128_1040.pdf", + "@type": "File", + "contentSize": 10865, + "dateModified": "2024-11-28T09:40:17", + "encodingFormat": "application/pdf", + "name": "a83j_20241128_1040.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/a83j_20241128_1417.pdf", + "@type": "File", + "contentSize": 10865, + "dateModified": "2024-11-28T13:17:59", + "encodingFormat": "application/pdf", + "name": "a83j_20241128_1417.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/a83j_20241128_1209.pdf", + "@type": "File", + "contentSize": 10865, + "dateModified": "2024-11-28T11:09:40", + "encodingFormat": "application/pdf", + "name": "a83j_20241128_1209.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/a83j_20241127_1848.pdf", + "@type": "File", + "contentSize": 10974, + "dateModified": "2024-11-27T17:48:41", + "encodingFormat": "application/pdf", + "name": "a83j_20241127_1848.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/a83j_20241128_0938.pdf", + "@type": "File", + "contentSize": 10865, + "dateModified": "2024-11-28T08:38:59", + "encodingFormat": "application/pdf", + "name": "a83j_20241128_0938.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/a83j_20241128_1041.pdf", + "@type": "File", + "contentSize": 10865, + "dateModified": "2024-11-28T09:41:46", + "encodingFormat": "application/pdf", + "name": "a83j_20241128_1041.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/a83j_20241128_1303.pdf", + "@type": "File", + "contentSize": 10865, + "dateModified": "2024-11-28T12:03:26", + "encodingFormat": "application/pdf", + "name": "a83j_20241128_1303.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/a83j_20241129_1019.pdf", + "@type": "File", + "contentSize": 11052, + "dateModified": "2024-11-29T09:20:00", + "encodingFormat": "application/pdf", + "name": "a83j_20241129_1019.pdf", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "plot/", + "@type": "Dataset" + }, + { + "@id": "status/a83j_20241128_1303.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-28T12:03:26", + "encodingFormat": "text/plain", + "name": "a83j_20241128_1303.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/a83j_20241128_1209.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-28T11:09:40", + "encodingFormat": "text/plain", + "name": "a83j_20241128_1209.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/a83j_20241128_1417.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-28T13:17:59", + "encodingFormat": "text/plain", + "name": "a83j_20241128_1417.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/a83j_20241129_1020.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-29T09:20:00", + "encodingFormat": "text/plain", + "name": "a83j_20241129_1020.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/a83j_20241128_1040.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-28T09:40:17", + "encodingFormat": "text/plain", + "name": "a83j_20241128_1040.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/a83j_20241128_1041.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-28T09:41:46", + "encodingFormat": "text/plain", + "name": "a83j_20241128_1041.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/a83j_20241128_0938.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-28T08:38:59", + "encodingFormat": "text/plain", + "name": "a83j_20241128_0938.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/a83j_20241127_1848.txt", + "@type": "File", + "contentSize": 252, + "dateModified": "2024-11-27T17:48:41", + "encodingFormat": "text/plain", + "name": "a83j_20241127_1848.txt", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "status/", + "@type": "Dataset" + }, + { + "@id": "pkl/.job_list_a83j.pkl.swp", + "@type": "File", + "contentSize": 28672, + "dateModified": "2024-11-29T11:59:04", + "encodingFormat": "application/binary", + "name": ".job_list_a83j.pkl.swp", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "pkl/job_packages_a83j.db", + "@type": "File", + "contentSize": 16384, + "dateModified": "2024-11-29T09:19:56", + "encodingFormat": "application/binary", + "name": "job_packages_a83j.db", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "pkl/job_list_a83j.pkl", + "@type": "File", + "contentSize": 47104, + "dateModified": "2024-11-29T09:57:58", + "encodingFormat": "application/binary", + "name": "job_list_a83j.pkl", + "sdDatePublished": "2024-11-29T12:01:02+00:00" + }, + { + "@id": "pkl/", + "@type": "Dataset" + }, + { + "@id": "#create-action", + "@type": "CreateAction", + "actionStatus": { + "@id": "http://schema.org/PotentialActionStatus" + }, + "description": "Test multiprov", + "endTime": "2024-11-29T10:55:48", + "object": [ + { + "@id": "#DEFAULT.EXPID-pv" + }, + { + "@id": "#DEFAULT.HPCARCH-pv" + }, + { + "@id": "#EXPERIMENT.DATELIST-pv" + }, + { + "@id": "#EXPERIMENT.MEMBERS-pv" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZEUNIT-pv" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZE-pv" + }, + { + "@id": "#EXPERIMENT.NUMCHUNKS-pv" + }, + { + "@id": "#EXPERIMENT.CHUNKINI-pv" + }, + { + "@id": "#EXPERIMENT.CALENDAR-pv" + }, + { + "@id": "#CONFIG.EXPID-pv" + }, + { + "@id": "#CONFIG.AUTOSUBMIT_VERSION-pv" + }, + { + "@id": "#CONFIG.MAXWAITINGJOBS-pv" + }, + { + "@id": "#CONFIG.TOTALJOBS-pv" + }, + { + "@id": "#CONFIG.SAFETYSLEEPTIME-pv" + }, + { + "@id": "#CONFIG.RETRIALS-pv" + }, + { + "@id": "#PROJECT.PROJECT_TYPE-pv" + }, + { + "@id": "#PROJECT.PROJECT_DESTINATION-pv" + }, + { + "@id": "#LOCAL.PROJECT_PATH-pv" + } + ], + "startTime": "2024-11-29T10:57:45" + }, + { + "@id": "file:///esarchive/scratch/apuiggro/sunset", + "@type": "SoftwareSourceCode", + "abstract": "The Autosubmit project. It contains the templates used\nby Autosubmit for the scripts used in the workflow, as well as any other\nsource code used by the scripts (i.e. any files sourced, or other source\ncode compiled or executed in the workflow).", + "codeRepository": "file:///esarchive/scratch/apuiggro/sunset", + "codeSampleType": "template", + "name": "file:///esarchive/scratch/apuiggro/sunset", + "programmingLanguage": "Any", + "runtimePlatform": "Autosubmit 4.1.11", + "sdDatePublished": "2024-11-29T12:01:02+00:00", + "targetProduct": "Autosubmit", + "version": "" + }, + { + "@id": "#DEFAULT.EXPID-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "DEFAULT.EXPID", + "valueRequired": "True" + }, + { + "@id": "#DEFAULT.EXPID-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#DEFAULT.EXPID-param" + }, + "name": "DEFAULT.EXPID", + "value": "a83j" + }, + { + "@id": "#DEFAULT.HPCARCH-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "DEFAULT.HPCARCH", + "valueRequired": "True" + }, + { + "@id": "#DEFAULT.HPCARCH-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#DEFAULT.HPCARCH-param" + }, + "name": "DEFAULT.HPCARCH", + "value": "nord3v2" + }, + { + "@id": "#EXPERIMENT.DATELIST-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "EXPERIMENT.DATELIST", + "valueRequired": "True" + }, + { + "@id": "#EXPERIMENT.DATELIST-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#EXPERIMENT.DATELIST-param" + }, + "name": "EXPERIMENT.DATELIST", + "value": "20241129" + }, + { + "@id": "#EXPERIMENT.MEMBERS-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "EXPERIMENT.MEMBERS", + "valueRequired": "True" + }, + { + "@id": "#EXPERIMENT.MEMBERS-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#EXPERIMENT.MEMBERS-param" + }, + "name": "EXPERIMENT.MEMBERS", + "value": "fc0" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZEUNIT-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "EXPERIMENT.CHUNKSIZEUNIT", + "valueRequired": "True" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZEUNIT-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#EXPERIMENT.CHUNKSIZEUNIT-param" + }, + "name": "EXPERIMENT.CHUNKSIZEUNIT", + "value": "month" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZE-param", + "@type": "FormalParameter", + "additionalType": "Integer", + "name": "EXPERIMENT.CHUNKSIZE", + "valueRequired": "True" + }, + { + "@id": "#EXPERIMENT.CHUNKSIZE-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#EXPERIMENT.CHUNKSIZE-param" + }, + "name": "EXPERIMENT.CHUNKSIZE", + "value": 1 + }, + { + "@id": "#EXPERIMENT.NUMCHUNKS-param", + "@type": "FormalParameter", + "additionalType": "Integer", + "name": "EXPERIMENT.NUMCHUNKS", + "valueRequired": "True" + }, + { + "@id": "#EXPERIMENT.NUMCHUNKS-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#EXPERIMENT.NUMCHUNKS-param" + }, + "name": "EXPERIMENT.NUMCHUNKS", + "value": 3 + }, + { + "@id": "#EXPERIMENT.CHUNKINI-param", + "@type": "FormalParameter", + "additionalType": "Integer", + "name": "EXPERIMENT.CHUNKINI", + "valueRequired": "True" + }, + { + "@id": "#EXPERIMENT.CHUNKINI-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#EXPERIMENT.CHUNKINI-param" + }, + "name": "EXPERIMENT.CHUNKINI", + "value": 1 + }, + { + "@id": "#EXPERIMENT.CALENDAR-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "EXPERIMENT.CALENDAR", + "valueRequired": "True" + }, + { + "@id": "#EXPERIMENT.CALENDAR-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#EXPERIMENT.CALENDAR-param" + }, + "name": "EXPERIMENT.CALENDAR", + "value": "standard" + }, + { + "@id": "#CONFIG.EXPID-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "CONFIG.EXPID", + "valueRequired": "True" + }, + { + "@id": "#CONFIG.EXPID-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#CONFIG.EXPID-param" + }, + "name": "CONFIG.EXPID", + "value": "a83j" + }, + { + "@id": "#CONFIG.AUTOSUBMIT_VERSION-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "CONFIG.AUTOSUBMIT_VERSION", + "valueRequired": "True" + }, + { + "@id": "#CONFIG.AUTOSUBMIT_VERSION-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#CONFIG.AUTOSUBMIT_VERSION-param" + }, + "name": "CONFIG.AUTOSUBMIT_VERSION", + "value": "4.1.11" + }, + { + "@id": "#CONFIG.MAXWAITINGJOBS-param", + "@type": "FormalParameter", + "additionalType": "Integer", + "name": "CONFIG.MAXWAITINGJOBS", + "valueRequired": "True" + }, + { + "@id": "#CONFIG.MAXWAITINGJOBS-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#CONFIG.MAXWAITINGJOBS-param" + }, + "name": "CONFIG.MAXWAITINGJOBS", + "value": 7 + }, + { + "@id": "#CONFIG.TOTALJOBS-param", + "@type": "FormalParameter", + "additionalType": "Integer", + "name": "CONFIG.TOTALJOBS", + "valueRequired": "True" + }, + { + "@id": "#CONFIG.TOTALJOBS-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#CONFIG.TOTALJOBS-param" + }, + "name": "CONFIG.TOTALJOBS", + "value": 7 + }, + { + "@id": "#CONFIG.SAFETYSLEEPTIME-param", + "@type": "FormalParameter", + "additionalType": "Integer", + "name": "CONFIG.SAFETYSLEEPTIME", + "valueRequired": "True" + }, + { + "@id": "#CONFIG.SAFETYSLEEPTIME-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#CONFIG.SAFETYSLEEPTIME-param" + }, + "name": "CONFIG.SAFETYSLEEPTIME", + "value": 10 + }, + { + "@id": "#CONFIG.RETRIALS-param", + "@type": "FormalParameter", + "additionalType": "Integer", + "name": "CONFIG.RETRIALS", + "valueRequired": "True" + }, + { + "@id": "#CONFIG.RETRIALS-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#CONFIG.RETRIALS-param" + }, + "name": "CONFIG.RETRIALS", + "value": 0 + }, + { + "@id": "#PROJECT.PROJECT_TYPE-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "PROJECT.PROJECT_TYPE", + "valueRequired": "True" + }, + { + "@id": "#PROJECT.PROJECT_TYPE-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#PROJECT.PROJECT_TYPE-param" + }, + "name": "PROJECT.PROJECT_TYPE", + "value": "local" + }, + { + "@id": "#PROJECT.PROJECT_DESTINATION-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "PROJECT.PROJECT_DESTINATION", + "valueRequired": "True" + }, + { + "@id": "#PROJECT.PROJECT_DESTINATION-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#PROJECT.PROJECT_DESTINATION-param" + }, + "name": "PROJECT.PROJECT_DESTINATION", + "value": "auto-s2s" + }, + { + "@id": "#LOCAL.PROJECT_PATH-param", + "@type": "FormalParameter", + "additionalType": "Text", + "name": "LOCAL.PROJECT_PATH", + "valueRequired": "True" + }, + { + "@id": "#LOCAL.PROJECT_PATH-pv", + "@type": "PropertyValue", + "exampleOfWork": { + "@id": "#LOCAL.PROJECT_PATH-param" + }, + "name": "LOCAL.PROJECT_PATH", + "value": "/esarchive/scratch/apuiggro/sunset" + }, + { + "@id": "proj/auto-s2s/", + "@type": "Dataset" + }, + { + "@id": "https://orcid.org/0009-0006-0707-6448", + "@type": "Person", + "affiliation": { + "@id": "https://ror.org/05sd8tv96" + } + }, + { + "@id": "#metaclip-create-action", + "@type": "CreateAction", + "description": "SUNSET provenance based on METACLIP", + "endTime": "2024-11-29 10:18", + "instrument": { + "@id": "file:///esarchive/scratch/apuiggro/sunset" + }, + "isPartOf": { + "@id": "#create-action" + }, + "object": [ + { + "@id": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.yml" + }, + { + "@id": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.yml" + }, + { + "@id": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.yml" + } + ], + "result": [ + { + "@id": "/tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.json" + }, + { + "@id": "/tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.json" + }, + { + "@id": "/tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.json" + } + ] + } + ] +} \ No newline at end of file diff --git a/use_cases/ex1_4_provenance_autosubmit/ex1_4-handson.md b/use_cases/ex1_4_provenance_autosubmit/ex1_4-handson.md new file mode 100644 index 0000000000000000000000000000000000000000..f8f04b1796005e6e5ca42540c668f4db5682bf35 --- /dev/null +++ b/use_cases/ex1_4_provenance_autosubmit/ex1_4-handson.md @@ -0,0 +1,295 @@ +# Hands-on 1.4: Provenance generation when running with autosubmit + +Author: Albert Puiggros Figueras + +## Goal + +Create a SUNSET recipe to generate a set of JSON files containing the provenance information of the workflow execution combining autosubmit rocrate structure with the METACLIP ontologies. For this, we compute some skill metrics and scorecards plots with SUNSET, using Autosubmit to dispatch jobs in parallel. In the recipe, we request 3 start dates for January, Feburary and March (0101, 0201,0301). SUNSET will split the recipe into 3 atomic recipes, and Autosubmit will run 3 jobs, which process the verification, for each recipe in parallel. When all the scorecards are generted, the transfer_provenance job will be triggered transfering the SUNSET-generated provenance files to the autosubmits experiment folder to, lastly, create an rocrate object where the entire process description is encapsulated. + +## Why provenance? + +Creating an RO-Crate object for your workflow is highly beneficial for reproducibility, transparency, and long-term storage. It encapsulates the entire process, including data, scripts, execution steps, and provenance information, ensuring others can easily reproduce and understand the results. By adhering to open standards, RO-Crates are interoperable with various tools, making data sharing and integration seamless. In this case, they store rich metadata, such as Autosubmit configuration files and Autosubmit job structures, which enhance discoverability and reuse. Fruther, it includes a set of JSON files that describe with detail the SUNSET worflow executed (one JSON file per job). For publications, RO-Crates provide a citable, machine-readable package of the workflow, ensuring durability and accessibility for future research or collaborative efforts. + +## 0. Cloning the SUNSET repository (branch Dev-Provenance) + +If you're completely new to SUNSET, the first step is to create a copy of the tool in your local environment. +Open a terminal and `cd` to the directory where you would like to store your local copy of SUNSET. For example: `/esarchive/scratch//git/`. If a directory does not exist yet, you can create it with the `mkdir` shell command. + +```shell +# Clone the GitLab repository to create a local copy of the code +git clone https://earth.bsc.es/gitlab/es/sunset.git + +``` +You should see a git folder "sunset" under the current directory. Now you have all the code, recipes, and scripts for running SUNSET. + +## 1. Create Autosubmit experiment +Since we're going to use Autosubmit to dispatch jobs, we need to have an Autosubmit experiment. Note that SUNSET uses Autosubmit >= 4.1.11 + +```shell +module load autosubmit/4.1.11-foss-2021b-Python-3.9.6 +autosubmit expid -H mn5 -d "SUNSET use case 1_4" +``` +You will see the messages like below: + +```shell +Autosubmit is running with 4.1.11 +The new experiment "a83y" has been registered. +Generating folder structure... +Experiment folder: /esarchive/autosubmit/a81u +Generating config files... +Experiment a83y created +``` +Note the experiment ID (in this snippet above a83y) down. We need it for the recipe later. + +## 2. Modifying the recipe + +The template recipe for this use case can be found in [ex1_4-recipe.yml](use_cases/ex1_4_autosubmit_provenance/ex1_4-recipe.yml). +You should at least edit some items in the "Run" section: +- `output_dir`: The directory you want to save the outputs and logs +- `code_dir`: The directory where your SUNSET code is stored (i.e., the git folder) +- `auto_conf$script`: The path to the script ex1_2-recipe.yml +- `auto_conf$expid`: The experiment "xxxx" you just created +- `auto_conf$hpc_user`: You user ID on Nord3, which should be bsc032xxx +- `auto_conf$email_address`: Your email. You can also adjust other email notification parts up to your preference. + +In the recipe, we ask for anomaly calculation after loading the data, calculate the skill scores and save the result for scorecards. In the Scorecard section, three regions are requested. + +Note the use of parameter `Provenance` to request the provenance generation. Currently, provenance is available for the Loading and Aggregation. Further modules will have this provenance feature in the future, aiming to cover the entirely SUNSET workflows. Further, you have to include your personal orcid. If doesn't dispose of one yo can create one here [https://orcid.org/](https://orcid.org/): + +```shell +orcid: https://orcid.org/0009-0006-0707-6448 +``` +Turn the notifications off due to a current bugg in autosubmit. + +```shell + email_notifications: no +``` + +Issue: [Issue 1468](https://earth.bsc.es/gitlab/es/autosubmit/-/issues/1468) + +## 3. The user-defined script + +We need to have a script to define the modules to use and the steps of the workflow. Note that the script is for data loading and verification parts. The Scorecards module doesn't need to be included in this script. The Statistics module is called in order to compute some statistics that are needed to correctly to the ensemble correlation aggregation for the Scorecards. + +The prepare_outputs() function is already incorporated into the launcher script (see the next section for details about launcher), so it does not need to be included in the user-defined script in this case. +In its place, we will use the function read_atomic_recipe(). The recipe path is passed as an argument onto the R script. Below is an example of what the script should look like. + +Nothing need to be specified in teh script regarding provenance operations. Only with the recipe specifications is enough. + +NOTE: Don't try to copy and paste this code on an R terminal; it will not work without properly pre-processing the recipe and creating the necessary output folders first. + +The beginning of the user-defined script should look like this: + +```R +# Load modules +source("modules/Loading/Loading.R") +source("modules/Anomalies/Anomalies.R") +source("modules/Skill/Skill.R") +source("modules/Statistics/Statistics.R") +source("modules/Scorecards/Scorecards_calculations.R") +source("modules/Saving/Saving.R") + +# Read recipe +## (leave this part as-is! Autosubmit will automatically pass the atomic recipe as a parameter) +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] +recipe <- read_atomic_recipe(recipe_file) +``` + +The rest of the user-defined script can be written in the same way as any other SUNSET script. We load the data, calculate the anomalies, then compute the skill metrics and statistics, and we call `Scorecards_calculations()` to do some specific computations and save the result as netCDF files for Scorecards. + +```R +# Load data +data <- Loading(recipe) +# Compute tos anomalies +data <- Anomalies(recipe, data) +# Compute skill metrics +skill_metrics <- Skill(recipe, data) +# Compute statistics +statistics <- Statistics(recipe, data) +# Pre-computations required for the Scorecards +Scorecards_calculations(recipe, data = data, + skill_metrics = skill_metrics, + statistics = statistics) + +``` +Check the example script at [ex1_4-script.yml](use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R). +You can execute it as-is or copy it and modify it according to your needs. + +## 4. Launch jobs and Use Autosubmit + +We will start the jobs with the launcher. The SUNSET Launcher is a bash script named launch_SUNSET.sh that can be found in the main directory of the SUNSET repository. It runs in two steps: + +1. Run the recipe checks, split the recipe into atomic recipes and create the directory for the outputs. +2. Modify the Autosubmit configuration of your experiment according to the parameters in the recipe. + +The bash script needs two inputs: (1) [recipe](#2-modifying-the-recipe) (2) [R script](#3-the-user-defined-script). + + On your workstation or Nord3 under the SUNSET code directory, run: + +```shell +bash launch_SUNSET.sh use_cases/ex1_4_provenance_autosubmit/ex1_4-recipe.yml use_cases/ex1_4_provenance_autosubmit/ex1_4-script.R +``` +You will see the messages similar to below: +```shell +[1] "Saving all outputs to:" +[1] "/esarchive/scratch/apuiggro/sunset/outputsex1_4-recipe_20241203113236" +INFO [2024-12-03 11:32:36] SUNSET: SUbseasoNal to decadal climate forecast post-processing and asSEssmenT suite + Copyright (C) 2024 BSC-CNS + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program. If not, see . +INFO [2024-12-03 11:32:38] Checking recipe: use_cases/ex1_4_provenance_autosubmit/ex1_4-recipe.yml +WARN [2024-12-03 11:32:38] The element 'fcst_year' is not defined in the recipe. No forecast year will be used. +INFO [2024-12-03 11:32:38] ##### RECIPE CHECK SUCCESSFULL ##### +INFO [2024-12-03 11:32:39] Splitting recipe in single verifications. +INFO [2024-12-03 11:32:39] The main recipe has been divided into 3 single model atomic recipes, plus 0 multi-model atomic recipes. +INFO [2024-12-03 11:32:39] Check output directory /esarchive/scratch/apuiggro/sunset/outputs/ex1_4-recipe_20241203113236/logs/recipes/ to see all the individual atomic recipes. +INFO [2024-12-03 11:32:40] ##### AUTOSUBMIT CONFIGURATION WRITTEN FOR a83y ##### +INFO [2024-12-03 11:32:40] You can check your experiment configuration at: /esarchive/autosubmit/a83y/conf/ +INFO [2024-12-03 11:32:40] Please SSH into bsceeshub01 or bsceshub02 and run the following commands: +INFO [2024-12-03 11:32:40] module load autosubmit/4.1.11-foss-2021b-Python-3.9.6 +INFO [2024-12-03 11:32:40] autosubmit create a83y +INFO [2024-12-03 11:32:40] autosubmit refresh a83y +INFO [2024-12-03 11:32:40] nohup autosubmit run a83y & disown +INFO [2024-12-03 11:32:40] To generate the rocrate object (zip file) run the following command + after the execution is finished: +INFO [2024-12-03 11:32:40] autosubmit archive --rocrate a83y +INFO [2024-12-03 11:32:40] The zip file will be saved in /esarchive/autosubmit/2024 +``` +You can see some useful information, like the the path to atomic recipes, the Autosubmit configuration files, and most importantly, follow the last lines to launch your experiment. + +```shell +ssh bsceshub01.bsc.es +(enter Hub machine) +module load autosubmit/4.1.11-foss-2021b-Python-3.9.6 +autosubmit create a83y +autosubmit refresh a83y +nohup autosubmit run a83y & disown +``` + +Then, you can go to [Autosubmit GUI](https://earth.bsc.es/autosubmitapp/) to check the experiment status. + +![Autosubmit graph.](Figures/autosubmit_graph.png) + +As you can see, the Scorecards job is dependent on the Verification jobs. Once the 3 verification jobs are finished, the Scorecards job will start, followed by the transfer jobs. + + +# 5. Archive the autosubmit experiment and create rocrate object + +After the experiment has executed correctly you can archive it requesting rocrate : + +```shell +autosubmit archive --rocrate a83y -v + +``` + +This will generate the zip file containing all the provenance metadata and the corresponding files. At the time, autosubmit stores the zip in a folder: + +```shell +/esarchive/autosubmit/2024 + +``` + +The output consists on a folder with a top file [ro-crate-metadata.json](Files/ro-crate-metadata.json) that contains all the rocrate provenance. + +```shell +. +|- ro-crate-metadata.json +|- conf +| |- autosubmit_a83y.yml +| |- expdef_a83y.yml +| |- jobs_a83y.yml +| ... +|- status +| |- a83y_20241122_1047.txt +| |- a83y_20241122_1140.txt +|- tmp + |- ex1_4-recipe_20241203113236 + |- logs + |- recipes + |- atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.yml + |- atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.yml + |- atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.yml + |- ex1_4-recipe.yml + |- outputs + |- provenance + |- main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.json + |- main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.json + |- main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.json + ... +``` +This is the structure of [ro-crate-metadata.json](Files/ro-crate-metadata.json): + +```shell +{ + "@context": "https://w3id.org/ro/crate/1.1/context", + "@graph": [ + { + "@id": "./", + "@type": "Dataset", + "conformsTo": [ + { "@id": "https://w3id.org/ro/wfrun/process/0.1" }, + { "@id": "https://w3id.org/ro/wfrun/workflow/0.1" }, + { "@id": "https://w3id.org/workflowhub/workflow-ro-crate/1.0" } + ], + "creator": { "@id": "https://orcid.org/0009-0006-0707-6448" }, + "datePublished": "2024-11-29T12:01:02+00:00", + "description": "Test multiprov", + "hasPart": [ + (...) + ], + "license": "Apache-2.0", + "mainEntity": { "@id": "conf/metadata/experiment_data.yml" }, + "mentions": [ + { "@id": "#create-action" } + ] + }, + + (...) + + { + "@id": "#metaclip-create-action", + "@type": "CreateAction", + "description": "SUNSET provenance based on METACLIP", + "endTime": "2024-11-29T10:18:00Z", + "instrument": { + "@id": "file:///esarchive/scratch/apuiggro/sunset" + }, + "isPartOf": { + "@id": "#create-action" + }, + "object": [ + { "@id": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.yml" }, + { "@id": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.yml" }, + { "@id": "atomic_recipe_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.yml" } + ], + "result": [ + { "@id": "/tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0101.json" }, + { "@id": "/tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201.json" }, + { "@id": "/tmp/ex1_4-recipe_20241129101842/outputs/provenance/main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0301.json" } + ] + } + ] +} +``` + +Each json file in `tmp/ex1_4-recipe_20241203113236/outputs/provenance` contains the provenance of each individual job (as an atomic execution of a SUNSET workflow) run with autosubmit. When using the METACLIP intepreter [METACLIP Intepreter](https://metaclip.org/) it can be interactively visualized. Each node represents an entity (dataset, reggriding operation, command call, etc) and its related to other nodes with edges. Each edge represents a relation between entities. Entities can be further expanded to obtain more specific metadata related to the METACLIP definitions or further entity properties established by SUNSET. + +Here is an example: + +![METACLIP intepreter visualization of `main_provenance_sys-ECMWF-SEAS51_ref-ERA5_var-tas_reg-global_sdate-0201`.](Figures/metaclip_graph.png) + +Combined with METACLIP, the Ro-Crate-defined provenance provides a nearly complete representation of the entire workflow execution. However, note that provenance is currently unavailable for the Skill and Anomalies modules. + +This same use case can be executed with nord3v2 machine instead of mn5 olny by modifying the values of +`recipe$Run$filesystem` to esarchive and `recipe$Run$auto_conf$platform`to nord3v2. With this setting, the workflow structure looks like: + +![Autosubmit graph nord3v2.](Figures/autosubmit_graph2.png) diff --git a/use_cases/ex1_4_provenance_autosubmit/ex1_4-recipe.yml b/use_cases/ex1_4_provenance_autosubmit/ex1_4-recipe.yml new file mode 100644 index 0000000000000000000000000000000000000000..1e435d90ee974b64163c2aa1df250a973ae2d326 --- /dev/null +++ b/use_cases/ex1_4_provenance_autosubmit/ex1_4-recipe.yml @@ -0,0 +1,92 @@ +Description: + Author: Albert Puiggros + Info: Compute Skills and Plot Scorecards with Autosubmit + Provenance +Analysis: + Horizon: seasonal + Provenance: yes + Variables: + - {name: tas, freq: monthly_mean} + Datasets: + System: # multiple systems for single model, split if Multimodel = F + - {name: ECMWF-SEAS5.1} + Multimodel: + execute: False # single option + Reference: + - {name: ERA5} + Time: + sdate: # list, split + - '0101' + - '0201' + - '0301' + fcst_year: + hcst_start: '1993' # single option + hcst_end: '2003' # single option + ftime_min: 1 # single option + ftime_max: 6 # single option + Region: # multiple lists, split? Add region name if length(Region) > 1 + - {name: "global", latmin: -90, latmax: 90, lonmin: 0, lonmax: 359.9} + Regrid: + method: bilinear + type: to_system + Workflow: + Anomalies: + compute: yes + cross_validation: no + save: 'none' + Calibration: + method: raw + save: 'none' + Skill: + metric: mean_bias EnsCorr rps rpss crps crpss EnsSprErr # list, don't split + cross_validation: yes + save: 'all' + Statistics: + metric: cov std n_eff spread + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] + save: 'none' + Scorecards: + execute: yes # yes/no + regions: + Extra-tropical NH: {lon.min: 0, lon.max: 360, lat.min: 30, lat.max: 90} + Tropics: {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: 30} + Extra-tropical SH : {lon.min: 0, lon.max: 360, lat.min: -30, lat.max: -90} + start_months: 'all' + metric: mean_bias enscorr rpss crpss enssprerr + metric_aggregation: 'score' + inf_to_na: TRUE # Optional, bool: set inf values in data to NA, default is FALSE table_label: NULL + fileout_label: NULL + col1_width: NULL + col2_width: NULL + calculate_diff: FALSE + ncores: 8 + remove_NAs: no # bool, don't split + Output_format: Scorecards # string, don't split + +###################################################################### ########## +## Run CONFIGURATION +################################################################################ +Run: + Loglevel: INFO + Terminal: yes + filesystem: gpfs + output_dir: /esarchive/scratch/apuiggro/sunset_provenance_clean/sunset/outputs + code_dir: /esarchive/scratch/apuiggro/sunset_provenance_clean/sunset + autosubmit: yes + # fill only if using autosubmit + auto_conf: + script: ./use_cases/ex1_4_provenance_autosubmit/ex1_4-script.R + expid: a8w2 # replace with your EXPID + hpc_user: bsc032533 # replace with your hpc username + wallclock: 03:00 # hh:mm + processors_per_job: 4 + custom_directives: ['#SBATCH --exclusive'] + platform: mn5 + email_notifications: no # enable/disable email notifications. Change it if you want to. + email_address: albert.puiggros@bsc.es # replace with your email address + notify_completed: no # notify me by email when a job finishes + notify_failed: no # notify me by email when a job fails + orcid: https://orcid.org/0009-0006-0707-6448 + + diff --git a/use_cases/ex1_4_provenance_autosubmit/ex1_4-script.R b/use_cases/ex1_4_provenance_autosubmit/ex1_4-script.R new file mode 100644 index 0000000000000000000000000000000000000000..a9c3749c9dfdc73058b14ef5345d04694c560ffd --- /dev/null +++ b/use_cases/ex1_4_provenance_autosubmit/ex1_4-script.R @@ -0,0 +1,35 @@ +############################################################################### +## Author: An-Chi Ho +## Description: Computes some skill metrics and plots scorecards with Autosubmit +## Instructions: Follow the steps described in: +## use_cases/ex1_2_autosubmit_scorecards/ex1_2-handson.md +## This script should be called by bash script launch_SUNSET.sh. +############################################################################### + +# Load modules +source("modules/Loading/Loading.R") +source("modules/Anomalies/Anomalies.R") +source("modules/Skill/Skill.R") +source("modules/Statistics/Statistics.R") +source("modules/Scorecards/Scorecards_calculations.R") +source("modules/Saving/Saving.R") +source("provenance/prov_Loading.R") + +library(igraph) + +# Read recipe +args = commandArgs(trailingOnly = TRUE) +recipe_file <- args[1] +recipe <- read_atomic_recipe(recipe_file) +# Load data +data <- Loading(recipe) +# Compute tos anomalies +data <- Anomalies(recipe, data) +# Compute skill metrics +skill_metrics <- Skill(recipe, data) +# Compute statistics +statistics <- Statistics(recipe, data) +# Pre-computations required for the scorecards +Scorecards_calculations(recipe, data = data, + skill_metrics = skill_metrics, + statistics = statistics)