From 3563915679487d2775a161825603ac79450a3a1c Mon Sep 17 00:00:00 2001 From: Nadia Milders Date: Thu, 26 Oct 2023 14:38:40 +0200 Subject: [PATCH 1/7] merged tas-tos data blending into SUNSET --- conf/archive.yml | 359 ++++++++---- conf/variable-dictionary.yml | 13 +- modules/Loading/Loading.R | 9 +- modules/Loading/R/load_tas_tos.R | 530 ++++++++++++++++++ modules/Loading/R/mask_tas_tos.R | 65 +-- .../atomic_recipes/recipe_tas-tos_nadia.yml | 55 ++ 6 files changed, 874 insertions(+), 157 deletions(-) create mode 100644 modules/Loading/R/load_tas_tos.R create mode 100644 recipes/atomic_recipes/recipe_tas-tos_nadia.yml diff --git a/conf/archive.yml b/conf/archive.yml index 0b643ae9..4ea8c4c9 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -5,42 +5,36 @@ esarchive: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/ecmwf/system5c3s/" - daily_mean: {"tas":"daily_mean/tas_f6h/", "rsds":"daily/rsds_s0-24h/", - "prlr":"daily/prlr_s0-24h/", "tasmin":"daily/tasmin/", - "tasmax":"daily/tasmax/", "sfcWind":"daily_mean/sfcWind_f6h/", - "ta300":"daily_mean/ta300_f12h/", "ta500":"daily_mean/ta500_f12h/", - "ta850":"daily_mean/ta850_f12h/", "g300":"daily_mean/g300_f12h/", - "g500":"daily_mean/g500_f12h/", "g850":"daily_mean/g850_f12h/", - "tdps":"daily_mean/tdps_f6h/", "hurs":"daily_mean/hurs_f6h/"} - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "rsds":"monthly_mean/rsds_s0-24h/", - "prlr":"monthly_mean/prlr_s0-24h/", - "sfcWind":"monthly_mean/sfcWind_f6h/", - "tasmin":"monthly_mean/tasmin_f24h/", - "tasmax":"monthly_mean/tasmax_f24h/", - "ta300":"monthly_mean/ta300_f12h/", "ta500":"monthly_mean/ta500_f12h/", - "ta850":"monthly_mean/ta850_f12h/", "g300":"monthly_mean/g300_f12h/", - "g500":"monthly_mean/g500_f12h/", "g850":"monthly_mean/g500_f12h/", - "tdps":"monthly_mean/tdps_f6h/", "psl":"monthly_mean/psl_f6h/", - "tos":"monthly_mean/tos_f6h/"} + daily_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", + "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/", + "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", + "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", + "tdps":"_f6h/", "hurs":"_f6h/"} + monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", + "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/", + "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", + "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", + "tdps":"_f6h/", "tos":"_f6h/", "sic":"_f24h/"} nmember: fcst: 51 hcst: 25 calendar: "proleptic_gregorian" time_stamp_lag: "0" reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" + land_sea_mask: "/esarchive/exp/ecmwf/system5c3s/constant/lsm/lsm.nc" # $var$ ECMWF-SEAS5.1: name: "ECMWF SEAS5 (v5.1)" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/ecmwf/system51c3s/" - daily_mean: {"tas":"daily_mean/tas_f6h/", "prlr":"daily/prlr_s0-24h/", - "sfcWind":"daily_mean/sfcWind_f6h/", - "uas":"daily_mean/uas_f6h/", "vas":"daily_mean/vas_f6h/", - "psl":"daily_mean/psl_f6h/", "tdps":"daily_mean/tdps_f6h/"} - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "rsds":"monthly_mean/rsds_s0-24h/", - "prlr":"monthly_mean/prlr_s0-24h/", "sfcWind":"monthly_mean/sfcWind_f6h/", - "tasmin":"monthly_mean/tasmin_f24h/", "tasmax":"monthly_mean/tasmax_f24h/", - "uas":"monthly_mean/uas_f6h/", "vas":"monthly_mean/vas_f6h/", - "psl":"monthly_mean/psl_f6h/", "tdps":"monthly_mean/tdps_f6h/"} + daily_mean: {"tas":"_f6h/", "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", + "tdps":"_f6h/"} + monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", "prlr":"_s0-24h/", + "sfcWind":"_f6h/", "tasmin":"_f24h/", "tasmax":"_f24h/", + "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", + "tdps":"_f6h/"} nmember: fcst: 51 hcst: 25 @@ -51,9 +45,9 @@ esarchive: name: "Meteo-France System 7" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/meteofrance/system7c3s/" - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "g500":"monthly_mean/g500_f12h/", - "prlr":"monthly_mean/prlr_f24h/", "sfcWind": "monthly_mean/sfcWind_f6h/", - "tasmax":"monthly_mean/tasmax_f6h/", "tasmin": "monthly_mean/tasmin_f6h/"} + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/", + "tasmax":"_f6h/", "tasmin": "_f6h/"} nmember: fcst: 51 hcst: 25 @@ -64,9 +58,9 @@ esarchive: name: "DWD GCFS 2.1" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/dwd/system21_m1/" - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", - "g500":"monthly_mean/g500_f12h/", "sfcWind":"monthly_mean/sfcWind_f6h/", - "tasmin":"monthly_mean/tasmin_f24h/", "tasmax":"monthly_mean/tasmax_f24h/"} + monthly_mean: {"tas":"_f6h/", "prlr":"_f24h/", + "g500":"_f12h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/"} nmember: fcst: 50 hcst: 30 @@ -77,9 +71,9 @@ esarchive: name: "CMCC-SPS3.5" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/cmcc/system35c3s/" - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", - "g500":"monthly_mean/g500_f12h/", "sfcWind":"monthly_mean/sfcWind_f6h/", - "tasmin":"monthly_mean/tasmin_f24h/", "tasmax":"monthly_mean/tasmax_f24h/"} + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/", + "tasmax":"_f24h/", "tasmin":"_f24h"} nmember: fcst: 50 hcst: 40 @@ -90,8 +84,8 @@ esarchive: name: "JMA System 2" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/jma/system2c3s/" - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", - "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} nmember: fcst: 10 hcst: 10 @@ -102,8 +96,8 @@ esarchive: name: "ECCC CanCM4i" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/eccc/eccc1/" - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", - "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} nmember: fcst: 10 hcst: 10 @@ -114,8 +108,8 @@ esarchive: name: "UK MetOffice GloSea 6 (v6.0)" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/ukmo/glosea6_system600-c3s/" - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "tasmin":"monthly_mean/tasmin_f24h/", - "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f24h/"} + monthly_mean: {"tas":"_f6h/", "tasmin":"_f24h/", + "tasmax":"_f24h/", "prlr":"_f24h/"} nmember: fcst: 62 hcst: 28 @@ -126,8 +120,8 @@ esarchive: name: "NCEP CFSv2" institution: "NOAA NCEP" #? src: "exp/ncep/cfs-v2/" - monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", - "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} nmember: fcst: 20 hcst: 20 @@ -139,114 +133,265 @@ esarchive: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/era5/" - daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", - "rsds":"daily_mean/rsds_f1h-r1440x721cds/", - "prlr":"daily_mean/prlr_f1h-r1440x721cds/", - "g300":"daily_mean/g300_f1h-r1440x721cds/", - "g500":"daily_mean/g500_f1h-r1440x721cds/", - "g850":"daily_mean/g850_f1h-r1440x721cds/", - "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax_f1h-r1440x721cds/", - "tasmin":"daily/tasmin_f1h-r1440x721cds/", - "ta300":"daily_mean/ta300_f1h-r1440x721cds/", - "ta500":"daily_mean/ta500_f1h-r1440x721cds/", - "ta850":"daily_mean/ta850_f1h-r1440x721cds/", - "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", - "psl":"monthly_mean/psl_f1h-r1440x721cds/", - "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", - "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", - "g300":"monthly_mean/g300_f1h-r1440x721cds/", - "g500":"monthly_mean/g500_f1h-r1440x721cds/", - "g850":"monthly_mean/g850_f1h-r1440x721cds/", - "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", - "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", - "ta300":"montly_mean/ta300_f1h-r1440x721cds/", - "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", - "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", - "tos":"monthly_mean/tos_f1h-r1440x721cds/"} + daily_mean: {"tas":"_f1h-r1440x721cds/", + "rsds":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", + "g300":"_f1h-r1440x721cds/", + "g500":"_f1h-r1440x721cds/", + "g850":"_f1h-r1440x721cds/", + "sfcWind":"_f1h-r1440x721cds/", + "tasmax":"_f1h-r1440x721cds/", + "tasmin":"_f1h-r1440x721cds/", + "ta300":"_f1h-r1440x721cds/", + "ta500":"_f1h-r1440x721cds/", + "ta850":"_f1h-r1440x721cds/", + "hurs":"_f1h-r1440x721cds/"} + monthly_mean: {"tas":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", + "rsds":"_f1h-r1440x721cds/", + "g300":"_f1h-r1440x721cds/", + "g500":"_f1h-r1440x721cds/", + "g850":"_f1h-r1440x721cds/", + "sfcWind":"_f1h-r1440x721cds/", + "tasmax":"_f1h-r1440x721cds/", + "tasmin":"_f1h-r1440x721cds/", + "ta300":"_f1h-r1440x721cds/", + "ta500":"_f1h-r1440x721cds/", + "ta850":"_f1h-r1440x721cds/", + "tos":"_f1h-r1440x721cds", + "sic":"_f1h-r1440x721cds", + } calendar: "standard" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" ERA5-Land: name: "ERA5-Land" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", - "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", - "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", - "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", - "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", - "tdps":"monthly_mean/tdps_f1h/"} + daily_mean: {"tas":"_f1h/", "rsds":"_f1h/", + "prlr":"_f1h/", "sfcWind":"_f1h/"} + monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/", + "tasmax":"_f24h/", "prlr":"_f1h/", + "sfcWind":"_f1h/", "rsds":"_f1h/", + "tdps":"_f1h/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" UERRA: name: "ECMWF UERRA" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"daily_mean/tas_f6h/"} - monthly_mean: {"tas":"monthly_mean/tas_f6h/"} + daily_mean: {"tas":"_f6h/"} + monthly_mean: {"tas":"_f6h/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" CERRA: name: "ECMWF CERRA" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/cerra/" - daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", - "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", - "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", - "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", - "tas":"monthly_mean/tas_f3h-r2631x1113/", - "winddir":"monthly_mean/winddir_f3h-r2631x1113/", - "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", - "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} + daily_mean: {"hur":"_f3h-r2631x1113/", "ps":"_f3h-r2631x1113/", "sfcWind":"_f3h-r2631x1113/", + "tas":"_f3h-r2631x1113/", "winddir":"_f3h-r2631x1113/"} + monthly_mean: {"hur":"_f3h-r2631x1113/", "ps":"_f3h-r2631x1113/", "sfcWind":"_f3h-r2631x1113/", + "tas":"_f3h-r2631x1113/", "winddir":"_f3h-r2631x1113/","tasmin":"_f24h-r2631x1113/","tasmax":"_f24h-r2631x1113/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" CERRA-Land: name: "ECMWF CERRA-Land" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/cerraland/" - daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} - monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} + daily_mean: {"prlr":"_f6h-r2631x1113/"} + monthly_mean: {"prlr":"_f6h-r2631x1113/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" - HadCRUT5: - name: "HadCRUT5" - institution: "Met Office" - src: "obs/ukmo/hadcrut_v5.0_analysis/" - monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" BEST: name: "BEST" institution: "European Centre for Medium-Range Weather Forecasts" src: "obs/berkeleyearth/berkeleyearth/" - daily_mean: {"tas":"daily_mean/tas/"} - monthly_mean: {"tas":"monthly_mean/tas/"} + daily_mean: {"tas":"/"} + monthly_mean: {"tas":"/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" + + + mars: - src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" + src: "/esarchive/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" - src: "GRIB_system5_tas_CORRECTED/" - monthly_mean: {"tas":""} + src: "exp/ecmwf/system5c3s/" + daily_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", + "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/", + "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", + "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", + "tdps":"_f6h/", "hurs":"_f6h/"} + monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", + "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/", + "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", + "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", + "tdps":"_f6h/"} + nmember: + fcst: 51 + hcst: 25 + calendar: "proleptic_gregorian" + time_stamp_lag: "0" + reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" + ECMWF-SEAS5.1: + name: "ECMWF SEAS5 (v5.1)" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/ecmwf/system51c3s/" + daily_mean: {"tas":"_f6h/", "prlr":"_s0-24h/", "sfcWind":"_f6h/", + "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", + "tdps":"_f6h/"} + monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", "prlr":"_s0-24h/", + "sfcWind":"_f6h/", "tasmin":"_f24h/", "tasmax":"_f24h/", + "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", + "tdps":"_f6h/"} + nmember: + fcst: 51 + hcst: 25 + calendar: "proleptic_gregorian" + time_stamp_lag: "0" + reference_grid: "conf/grid_description/griddes_system51c3s.txt" + Meteo-France-System7: + name: "Meteo-France System 7" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/meteofrance/system7c3s/" + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/", + "tasmax":"_f6h/", "tasmin": "_f6h/"} nmember: fcst: 51 - hcst: 51 + hcst: 25 + time_stamp_lag: "+1" + calendar: "proleptic_gregorian" + reference_grid: "conf/grid_description/griddes_system7c3s.txt" + DWD-GCFS2.1: + name: "DWD GCFS 2.1" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/dwd/system21_m1/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f24h/", + "g500":"_f12h/", "sfcWind":"_f6h/", + "tasmin":"_f24h/", "tasmax":"_f24h/"} + nmember: + fcst: 50 + hcst: 30 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_system21_m1.txt" + CMCC-SPS3.5: + name: "CMCC-SPS3.5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/cmcc/system35c3s/" + monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", + "prlr":"_f24h/", "sfcWind": "_f6h/", + "tasmax":"_f24h/", "tasmin":"_f24h"} + nmember: + fcst: 50 + hcst: 40 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_system35c3s.txt" + JMA-CPS2: + name: "JMA System 2" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/jma/system2c3s/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 10 + hcst: 10 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_system2c3s.txt" + ECCC-CanCM4i: + name: "ECCC CanCM4i" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/eccc/eccc1/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 10 + hcst: 10 + calendar: "proleptic_gregorian" + time_stamp_lag: "+1" + reference_grid: "conf/grid_description/griddes_eccc1.txt" + UK-MetOffice-Glosea600: + name: "UK MetOffice GloSea 6 (v6.0)" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "exp/ukmo/glosea6_system600-c3s/" + monthly_mean: {"tas":"_f6h/", "tasmin":"_f24h/", + "tasmax":"_f24h/", "prlr":"_f24h/"} + nmember: + fcst: 62 + hcst: 28 calendar: "proleptic_gregorian" time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" + reference_grid: "conf/grid_description/griddes_ukmo600.txt" + NCEP-CFSv2: + name: "NCEP CFSv2" + institution: "NOAA NCEP" #? + src: "exp/ncep/cfs-v2/" + monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", + "tasmax":"_f6h/", "tasmin":"_f6h/"} + nmember: + fcst: 20 + hcst: 20 + calendar: "gregorian" + time_stamp_lag: "0" + reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" Reference: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" - src: "GRIB_era5_tas/" - monthly_mean: {"tas":""} + src: "recon/ecmwf/era5/" + daily_mean: {"tas":"_f1h-r1440x721cds/", + "rsds":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", + "g300":"_f1h-r1440x721cds/", + "g500":"_f1h-r1440x721cds/", + "g850":"_f1h-r1440x721cds/", + "sfcWind":"_f1h-r1440x721cds/", + "tasmax":"_f1h-r1440x721cds/", + "tasmin":"_f1h-r1440x721cds/", + "ta300":"_f1h-r1440x721cds/", + "ta500":"_f1h-r1440x721cds/", + "ta850":"_f1h-r1440x721cds/", + "hurs":"_f1h-r1440x721cds/"} + monthly_mean: {"tas":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", + "rsds":"_f1h-r1440x721cds/", + "g300":"_f1h-r1440x721cds/", + "g500":"_f1h-r1440x721cds/", + "g850":"_f1h-r1440x721cds/", + "sfcWind":"_f1h-r1440x721cds/", + "tasmax":"_f1h-r1440x721cds/", + "tasmin":"_f1h-r1440x721cds/", + "ta300":"_f1h-r1440x721cds/", + "ta500":"_f1h-r1440x721cds/", + "ta850":"_f1h-r1440x721cds/"} calendar: "standard" - reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + ERA5-Land: + name: "ERA5-Land" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/era5land/" + daily_mean: {"tas":"_f1h/", "rsds":"_f1h/", + "prlr":"_f1h/", "sfcWind":"_f1h/"} + monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/", + "tasmax":"_f24h/", "prlr":"_f1h/", + "sfcWind":"_f1h/", "rsds":"_f1h/", + "tdps":"_f1h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" + UERRA: + name: "ECMWF UERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/uerra_mescan/" + daily_mean: {"tas":"_f6h/"} + monthly_mean: {"tas":"_f6h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" diff --git a/conf/variable-dictionary.yml b/conf/variable-dictionary.yml index 0bfbffe0..c440eac1 100644 --- a/conf/variable-dictionary.yml +++ b/conf/variable-dictionary.yml @@ -204,13 +204,16 @@ vars: long_name: "Surface Upward Sensible Heat Flux" standard_name: "surface_upward_sensible_heat_flux" accum: no -## Adding new variable - tasanomaly: + tas-tos: units: "K" - long_name: "Near-Surface Air Temperature Anomaly" - standard_name: "air_temperature_anom" + long_name: "Blended air - sea temperature" + standard_name: "air_sea_temperature" + accum: no + sic: + units: "1" + long_name: "Sea Ice Concentration" + standard_name: "sea_ice_concentration" accum: no - # Coordinates diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R index d2919594..b4adca5e 100644 --- a/modules/Loading/Loading.R +++ b/modules/Loading/Loading.R @@ -12,8 +12,13 @@ Loading <- function(recipe) { # Case: esarchive time_horizon <- tolower(recipe$Analysis$Horizon) if (time_horizon == "seasonal") { - source("modules/Loading/R/load_seasonal.R") - data <- load_seasonal(recipe) + if(recipe$Analysis$Variables$name == 'tas-tos') { + source("modules/Loading/R/load_tas_tos.R") + data <- load_tas_tos(recipe) + } else { + source("modules/Loading/R/load_seasonal.R") + data <- load_seasonal(recipe) + } } else if (time_horizon == "decadal") { source("modules/Loading/R/load_decadal.R") data <- load_decadal(recipe) diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R new file mode 100644 index 00000000..ae39c6bc --- /dev/null +++ b/modules/Loading/R/load_tas_tos.R @@ -0,0 +1,530 @@ +source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") +# Load required libraries/funs +source("modules/Loading/R/dates2load.R") +source("modules/Loading/R/get_timeidx.R") +source("modules/Loading/R/check_latlon.R") + +source('modules/Loading/R/mask_tas_tos.R') + +## TODO: Source new s2dv_cube version +## TODO: Eliminate dim_var dimension (merge_across_dims?) + +load_tas_tos <- function(recipe) { + + # ------------------------------------------- + # Set params ----------------------------------------- + + hcst.inityear <- recipe$Analysis$Time$hcst_start + hcst.endyear <- recipe$Analysis$Time$hcst_end + lats.min <- recipe$Analysis$Region$latmin + lats.max <- recipe$Analysis$Region$latmax + lons.min <- recipe$Analysis$Region$lonmin + lons.max <- recipe$Analysis$Region$lonmax + ref.name <- recipe$Analysis$Datasets$Reference$name + exp.name <- recipe$Analysis$Datasets$System$name + + data_order <- c('dat', 'var', 'sday', 'sweek', 'syear', 'time', 'latitude', 'longitude', 'ensemble') + + variable <- c("tas", "tos", "sic") + sic.threshold <- recipe$Analysis$Variables$sic_threshold + store.freq <- recipe$Analysis$Variables$freq + + # get sdates array + ## LOGGER: Change dates2load to extract logger from recipe? + sdates <- dates2load(recipe, recipe$Run$logger) + + idxs <- NULL + idxs$hcst <- get_timeidx(sdates$hcst, + recipe$Analysis$Time$ftime_min, + recipe$Analysis$Time$ftime_max, + time_freq=store.freq) + + if (!(is.null(sdates$fcst))) { + idxs$fcst <- get_timeidx(sdates$fcst, + recipe$Analysis$Time$ftime_min, + recipe$Analysis$Time$ftime_max, + time_freq=store.freq) + } + + ## TODO: Examine this verifications part, verify if it's necessary + # stream <- verifications$stream + # sdates <- verifications$fcst.sdate + + ## TODO: define fcst.name + ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name + + # get esarchive datasets dict: + ## TODO: Adapt to 'filesystem' option in recipe + archive <- read_yaml("conf/archive.yml")$esarchive + exp_descrip <- archive$System[[exp.name]] + + freq.hcst <- unlist(exp_descrip[[store.freq]][variable[1]]) + reference_descrip <- archive$Reference[[ref.name]] + freq.obs <- unlist(reference_descrip[[store.freq]][variable[1]]) + obs.dir <- reference_descrip$src + fcst.dir <- exp_descrip$src + hcst.dir <- exp_descrip$src + fcst.nmember <- exp_descrip$nmember$fcst + hcst.nmember <- exp_descrip$nmember$hcst + + ## TODO: it is necessary? + ##if ("accum" %in% names(reference_descrip)) { + ## accum <- unlist(reference_descrip$accum[store.freq][[1]]) + ##} else { + ## accum <- FALSE + ##} + + var_dir_obs <- reference_descrip[[store.freq]][variable] + var_dir_exp <- exp_descrip[[store.freq]][variable] + + # ----------- + obs.path <- paste0(archive$src, + obs.dir, store.freq, "/$var$", "$var_dir$", + "/$var$_$file_date$.nc") + + hcst.path <- paste0(archive$src, + hcst.dir, store.freq, "/$var$", "$var_dir$", + "$var$_$file_date$.nc") + + fcst.path <- paste0(archive$src, + hcst.dir, store.freq, "/$var$", "$var_dir$", + "/$var$_$file_date$.nc") + + # Define regrid parameters: + #------------------------------------------------------------------- + regrid_params <- get_regrid_params(recipe, archive) + + # Longitude circular sort and latitude check + #------------------------------------------------------------------- + circularsort <- check_latlon(lats.min, lats.max, lons.min, lons.max) + + if (recipe$Analysis$Variables$freq == "monthly_mean"){ + split_multiselected_dims = TRUE + } else { + split_multiselected_dims = FALSE + } + + # Load hindcast data without regrid + #------------------------------------------------------------------- + hcst <- Start(dat = hcst.path, + var = variable, + var_dir = var_dir_exp, + file_date = sdates$hcst, + time = idxs$hcst, + var_dir_depends = 'var', + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude'), + ensemble = c('member', 'ensemble')), + ensemble = indices(1:hcst.nmember), + metadata_dims = 'var', # change to just 'var'? + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = split_multiselected_dims, + retrieve = TRUE) + + + # Remove var_dir dimension + if ("var_dir" %in% names(dim(hcst))) { + hcst <- Subset(hcst, along = "var_dir", indices = 1, drop = "selected") + } + + if (recipe$Analysis$Variables$freq == "daily_mean") { + # Adjusts dims for daily case, could be removed if startR allows + # multidim split + names(dim(hcst))[which(names(dim(hcst)) == 'file_date')] <- "syear" + default_dims <- c(dat = 1, var = 1, sday = 1, + sweek = 1, syear = 1, time = 1, + latitude = 1, longitude = 1, ensemble = 1) + default_dims[names(dim(hcst))] <- dim(hcst) + dim(hcst) <- default_dims + # Change time attribute dimensions + default_time_dims <- c(sday = 1, sweek = 1, syear = 1, time = 1) + names(dim(attr(hcst, "Variables")$common$time))[which(names( + dim(attr(hcst, "Variables")$common$time)) == 'file_date')] <- "syear" + default_time_dims[names(dim(attr(hcst, "Variables")$common$time))] <- + dim(attr(hcst, "Variables")$common$time) + dim(attr(hcst, "Variables")$common$time) <- default_time_dims + } + + # Define sea-ice grid points based of sea-ice concentration threshold + ice_hcst <- hcst[,3,,,,,,,] >= sic.threshold + + # Replace Tos with Tas for data points with sea ice + hcst[,2,,,,,,,][ice_hcst] <- hcst[,1,,,,,,,][ice_hcst] + + + # Convert hcst to s2dv_cube object + ## TODO: Give correct dimensions to $Dates + ## (sday, sweek, syear instead of file_date) + hcst <- as.s2dv_cube(hcst) + # Adjust dates for models where the time stamp goes into the next month + if (recipe$Analysis$Variables$freq == "monthly_mean") { + hcst$attrs$Dates[] <- hcst$attrs$Dates - seconds(exp_descrip$time_stamp_lag) + } + + # Combine hcst tas and tos data + #------------------------------------------------------------------- + + hcst <- mask_tas_tos(input_data = hcst, region = c(lons.min, lons.max,lats.min, lats.max), + mask_path = archive$System[[exp.name]]$land_sea_mask, lsm_var_name = 'lsm', + lon = hcst$coords$longitude, lat = hcst$coords$latitude, + lon_dim = 'longitude', lat_dim = 'latitude', ncores = NULL) + + hcst$dims[['var']] <- dim(hcst$data)[['var']] + hcst$attrs$Variable$varName <- 'tas-tos' + + hcst$data <- Reorder(hcst$data, data_order) + + # Load forecast data without regrid + #------------------------------------------------------------------- + if (!is.null(recipe$Analysis$Time$fcst_year)) { + # the call uses file_date instead of fcst_syear so that it can work + # with the daily case and the current version of startR not allowing + # multiple dims split + + fcst <- Start(dat = fcst.path, + var = variable, + var_dir = var_dir_exp, + var_dir_depends = 'var', + file_date = sdates$fcst, + time = idxs$fcst, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude'), + ensemble = c('member', 'ensemble')), + ensemble = indices(1:fcst.nmember), + metadata_dims = 'var', + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = split_multiselected_dims, + retrieve = TRUE) + + if ("var_dir" %in% names(dim(fcst))) { + fcst <- Subset(fcst, along = "var_dir", indices = 1, drop = "selected") + } + + if (recipe$Analysis$Variables$freq == "daily_mean") { + # Adjusts dims for daily case, could be removed if startR allows + # multidim split + names(dim(fcst))[which(names(dim(fcst)) == 'file_date')] <- "syear" + default_dims <- c(dat = 1, var = 1, sday = 1, + sweek = 1, syear = 1, time = 1, + latitude = 1, longitude = 1, ensemble = 1) + default_dims[names(dim(fcst))] <- dim(fcst) + dim(fcst) <- default_dims + # Change time attribute dimensions + default_time_dims <- c(sday = 1, sweek = 1, syear = 1, time = 1) + names(dim(attr(fcst, "Variables")$common$time))[which(names( + dim(attr(fcst, "Variables")$common$time)) == 'file_date')] <- "syear" + default_time_dims[names(dim(attr(fcst, "Variables")$common$time))] <- + dim(attr(fcst, "Variables")$common$time) + dim(attr(fcst, "Variables")$common$time) <- default_time_dims + } + + # Define sea-ice grid points based of sea-ice concentration threshold + ice_fcst <- fcst[,3,,,,,,,] >= sic.threshold + + # Replace Tos with Tas for datapoints with sea ice + fcst[,2,,,,,,,][ice_fcst] <- fcst[,1,,,,,,,][ice_fcst] + + + # Convert fcst to s2dv_cube + fcst <- as.s2dv_cube(fcst) + # Adjust dates for models where the time stamp goes into the next month + if (recipe$Analysis$Variables$freq == "monthly_mean") { + fcst$attrs$Dates[] <- + fcst$attrs$Dates - seconds(exp_descrip$time_stamp_lag) + } + + # Combine fcst tas and tos data + #------------------------------------------------------------------- + + fcst <- mask_tas_tos(input_data = fcst, region = c(lons.min, lons.max,lats.min, lats.max), + mask_path = archive$System[[exp.name]]$land_sea_mask, lsm_var_name = 'lsm', + lon = fcst$coords$longitude, lat = fcst$coords$latitude, + lon_dim = 'longitude', lat_dim = 'latitude', ncores = NULL) + + fcst$dims[['var']] <- dim(fcst$data)[['var']] + fcst$attrs$Variable$varName <- 'tas-tos' + + fcst$data <- Reorder(fcst$data, data_order) + + } else { + fcst <- NULL + } + + # Load obs data without regrid + #------------------------------------------------------------------- + + # Obtain dates and date dimensions from the loaded hcst data to make sure + # the corresponding observations are loaded correctly. + dates <- hcst$attrs$Dates + dim(dates) <- hcst$dims[c("sday", "sweek", "syear", "time")] + + # Separate Start() call for monthly vs daily data + if (store.freq == "monthly_mean") { + + dates_file <- format(as.Date(dates, '%Y%m%d'), "%Y%m") + dim(dates_file) <- dim(dates) + + + # Define variables for blended tas-tos datasets + if (recipe$Analysis$Datasets$Reference$name == 'BEST'){ + variable <- 'tas' + var_dir_obs <- reference_descrip[[store.freq]][variable] + } + + obs <- Start(dat = obs.path, + var = variable, + var_dir = var_dir_obs, + var_dir_depends = 'var', + file_date = dates_file, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude')), + metadata_dims = 'var', + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = TRUE, + retrieve = TRUE) + + + } else if (store.freq == "daily_mean") { + + # Get year and month for file_date + dates_file <- sapply(dates, format, '%Y%m') + dim(dates_file) <- dim(dates) + # Set hour to 12:00 to ensure correct date retrieval for daily data + lubridate::hour(dates) <- 12 + lubridate::minute(dates) <- 00 + # Restore correct dimensions + dim(dates) <- dim(dates_file) + + obs <- Start(dat = obs.path, + var = variable, + var_dir = var_dir_obs, + var_dir_depends = 'var', + file_date = sort(unique(dates_file)), + time = dates, + time_var = 'time', + time_across = 'file_date', + merge_across_dims = TRUE, + merge_across_dims_narm = TRUE, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude')), + metadata_dims = 'var', + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'file_date'), + split_multiselected_dims = TRUE, + retrieve = TRUE) + + } + + # Remove var_dir dimension + if ("var_dir" %in% names(dim(obs))) { + obs <- Subset(obs, along = "var_dir", indices = 1, drop = "selected") + } + # Adds ensemble dim to obs (for consistency with hcst/fcst) + default_dims <- c(dat = 1, var = 1, sday = 1, + sweek = 1, syear = 1, time = 1, + latitude = 1, longitude = 1, ensemble = 1) + default_dims[names(dim(obs))] <- dim(obs) + dim(obs) <- default_dims + + if(!recipe$Analysis$Datasets$Reference$name %in% c('HadCRUT4','HadCRUT5','BEST','GISTEMPv4')){ + + # Define sea-ice grid points based of sea-ice concentration threshold + ice_obs <- (obs[,3,,,,,,,]) >= sic.threshold + + # Replace NA values with False + ice_obs[is.na(ice_obs)] <- FALSE + + # Replace Tos with Tas for datapoints with sea ice + obs[,2,,,,,,,][ice_obs] <- obs[,1,,,,,,,][ice_obs] + } + + # Convert obs to s2dv_cube + obs <- as.s2dv_cube(obs) + + + # Combine obs tas and tos data + #------------------------------------------------------------------- + + if(!recipe$Analysis$Datasets$Reference$name %in% c('HadCRUT4','HadCRUT5','BEST','GISTEMPv4')){ + + obs <- mask_tas_tos(input_data = obs, region = c(lons.min, lons.max,lats.min, lats.max), + mask_path = archive$Reference[[ref.name]]$land_sea_mask, lsm_var_name = 'sftof', + lon = obs$coords$longitude, lat = obs$coords$latitude, + lon_dim = 'longitude', lat_dim = 'latitude', ncores = NULL) + + obs$dims[['var']] <- dim(obs$data)[['var']] + obs$attrs$Variable$varName <- 'tas-tos' + + } ## close if on reference name + + obs$data <- Reorder(obs$data, data_order) + + + # Regrid data + #------------------------------------------------------------------- + + # Regrid reference to system grid: + if(recipe$Analysis$Regrid$type == 'to_system'){ + + aux <- CDORemap(data_array = obs$data, ## Not regridding to desired grid when latitudes are ordered descending + lons = obs$coords$longitude, lats = obs$coords$latitude, + grid = regrid_params$obs.gridtype, method = recipe$Analysis$Regrid$method, + avoid_writes = TRUE, crop = c(lons.min, lons.max,lats.min, lats.max), + force_remap = TRUE) + + obs$data <- aux$data_array + obs$coords$longitude <- aux$lons + obs$coords$latitude <- aux$lats + obs$dims['longitude'] <- dim(aux$data_array)['longitude'] + obs$dims['latitude'] <- dim(aux$data_array)['latitude'] + rm(aux) + } + + # Regrid system to reference grid: + if(recipe$Analysis$Regrid$type == 'to_reference'){ + + aux <- CDORemap(data_array = hcst$data, + lons = hcst$coords$longitude, lats = hcst$coords$latitude, + grid = regrid_params$fcst.gridtype, method = recipe$Analysis$Regrid$method, + avoid_writes = TRUE, crop = TRUE, + force_remap = TRUE) + + hcst$data <- aux$data_array + hcst$coords$longitude <- aux$lons + hcst$coords$latitude <- aux$lats + hcst$dims['longitude'] <- dim(aux$data_array)['longitude'] + hcst$dims['latitude'] <- dim(aux$data_array)['latitude'] + rm(aux) + + if (!is.null(recipe$Analysis$Time$fcst_year)) { + aux <- CDORemap(data_array = fcst$data, + lons = fcst$coords$longitude, lats = fcst$coords$latitude, + grid = regrid_params$fcst.gridtype, method = recipe$Analysis$Regrid$method, + avoid_writes = TRUE, crop = TRUE, + force_remap = TRUE) + + fcst$data <- aux$data_array + fcst$coords$longitude <- aux$lons + fcst$coords$latitude <- aux$lats + fcst$dims['longitude'] <- dim(aux$data_array)['longitude'] + fcst$dims['latitude'] <- dim(aux$data_array)['latitude'] + rm(aux) + } + } + + # Regrid all data to user defined grid: + if(!recipe$Analysis$Regrid$type %in% c('to_system','to_reference')){ + + aux <- CDORemap(data_array = hcst$data, + lons = hcst$coords$longitude, lats = hcst$coords$latitude, + grid = regrid_params$fcst.gridtype, method = recipe$Analysis$Regrid$method, + avoid_writes = TRUE, crop = TRUE, + force_remap = TRUE) + + hcst$data <- aux$data_array + hcst$coords$longitude <- aux$lons + hcst$coords$latitude <- aux$lats + hcst$dims['longitude'] <- dim(aux$data_array)['longitude'] + hcst$dims['latitude'] <- dim(aux$data_array)['latitude'] + rm(aux) + + if (!is.null(recipe$Analysis$Time$fcst_year)) { + aux <- CDORemap(data_array = fcst$data, + lons = fcst$coords$longitude, lats = fcst$coords$latitude, + grid = regrid_params$fcst.gridtype, method = recipe$Analysis$Regrid$method, + avoid_writes = TRUE, crop = TRUE, + force_remap = TRUE) + + fcst$data <- aux$data_array + fcst$coords$longitude <- aux$lons + fcst$coords$latitude <- aux$lats + fcst$dims['longitude'] <- dim(aux$data_array)['longitude'] + fcst$dims['latitude'] <- dim(aux$data_array)['latitude'] + rm(aux) + } + + aux <- CDORemap(data_array = obs$data, + lons = obs$coords$longitude, lats = obs$coords$latitude, + grid = regrid_params$obs.gridtype, method = recipe$Analysis$Regrid$method, + avoid_writes = TRUE, crop = TRUE, + force_remap = TRUE) + + obs$data <- aux$data_array + obs$coords$longitude <- aux$lons + obs$coords$latitude <- aux$lats + obs$dims['longitude'] <- dim(aux$data_array)['longitude'] + obs$dims['latitude'] <- dim(aux$data_array)['latitude'] + rm(aux) + } + + + # Check for consistency between hcst and obs grid + if (!(recipe$Analysis$Regrid$type == 'none')) { + if (!isTRUE(all.equal(as.vector(hcst$lat), as.vector(obs$lat)))) { + lat_error_msg <- paste("Latitude mismatch between hcst and obs.", + "Please check the original grids and the", + "regrid parameters in your recipe.") + error(recipe$Run$logger, lat_error_msg) + hcst_lat_msg <- paste0("First hcst lat: ", hcst$lat[1], + "; Last hcst lat: ", hcst$lat[length(hcst$lat)]) + info(recipe$Run$logger, hcst_lat_msg) + obs_lat_msg <- paste0("First obs lat: ", obs$lat[1], + "; Last obs lat: ", obs$lat[length(obs$lat)]) + info(recipe$Run$logger, obs_lat_msg) + stop("hcst and obs don't share the same latitudes.") + } + if (!isTRUE(all.equal(as.vector(hcst$lon), as.vector(obs$lon)))) { + lon_error_msg <- paste("Longitude mismatch between hcst and obs.", + "Please check the original grids and the", + "regrid parameters in your recipe.") + error(recipe$Run$logger, lon_error_msg) + hcst_lon_msg <- paste0("First hcst lon: ", hcst$lon[1], + "; Last hcst lon: ", hcst$lon[length(hcst$lon)]) + info(recipe$Run$logger, hcst_lon_msg) + obs_lon_msg <- paste0("First obs lon: ", obs$lon[1], + "; Last obs lon: ", obs$lon[length(obs$lon)]) + info(recipe$Run$logger, obs_lon_msg) + stop("hcst and obs don't share the same longitudes.") + + } + } + + + # Compute anomalies if requested + # Print a summary of the loaded data for the user, for each object + if (recipe$Run$logger$threshold <= 2) { + data_summary(hcst, recipe) + data_summary(obs, recipe) + if (!is.null(fcst)) { + data_summary(fcst, recipe) + } + } + info(recipe$Run$logger, + "##### DATA LOADING COMPLETED SUCCESSFULLY #####") + + return(list(hcst = hcst, fcst = fcst, obs = obs)) + +} diff --git a/modules/Loading/R/mask_tas_tos.R b/modules/Loading/R/mask_tas_tos.R index a2eeb0b6..cc693795 100644 --- a/modules/Loading/R/mask_tas_tos.R +++ b/modules/Loading/R/mask_tas_tos.R @@ -2,29 +2,28 @@ library(multiApply) library(startR) library(s2dv) -mask_tas_tos <- function(input_data, grid, lon, lat, region = region , - lon_dim = 'lon', lat_dim = 'lat', ncores = NULL){ +mask_tas_tos <- function(input_data, mask_path, lsm_var_name = lsm_var_name, lon, lat, region = region, + lon_dim = 'lon', lat_dim = 'lat', + ncores = NULL){ - - mask <- .load_mask(grid = grid, lon_dim = lon_dim, lat_dim = lat_dim, + + mask <- .load_mask(mask_path = mask_path, lsm_var_name = lsm_var_name, lon_dim = lon_dim, lat_dim = lat_dim, sea_value = 1, land_value = 0, region = region) - ## TO DO: improve the check and correct lats stopifnot(all(lon == mask$lon)) stopifnot(max(abs(as.numeric(round(lat,2) - round(mask$lat,2)))) < 0.1) # stopifnot(all(lat == mask$lat)) - + tas <- Subset(input_data$data, along = 'var', indices = 1) tos <- Subset(input_data$data, along = 'var', indices = 2) - tas_tos <- multiApply::Apply(data = list(tas, tos), - target_dims = c(lon_dim, lat_dim), - fun = .mask_tas_tos, - mask = mask$mask, - sea_value = 1, - ncores = ncores)$output1 - input_data$data <- tas_tos - + input_data$data <- multiApply::Apply(data = list(tas, tos), + target_dims = c(lon_dim, lat_dim), + fun = .mask_tas_tos, + mask = mask$mask, + sea_value = 1, + ncores = ncores)$output1 + return(input_data) } @@ -33,52 +32,32 @@ mask_tas_tos <- function(input_data, grid, lon, lat, region = region , return(data_tas) } -.load_mask <- function(grid, mask_path = NULL, land_value = 0, sea_value = 1, +.load_mask <- function(mask_path = mask_path, lsm_var_name = lsm_var_name, land_value = 0, sea_value = 1, lon_dim = 'lon', lat_dim = 'lat', region = region){ - if (is.null(mask_path)){ - mask_sea_land_path <- '/esarchive/exp/ecmwf/system5c3s/constant/lsm/lsm.nc' ## /esarchive/recon/ecmwf/era5land/constant/lsm-r3600x1801cds/lsm.nc' - } else if (is.character(mask_path)){ - mask_sea_land_path <- mask_path - } else { - stop("mask_path must be NULL (to use the default mask and interpolate it to - the specified grid) or a string with the mask's path you want to load") - } - lons.min <- region[1] lons.max <- region[2] lats.min <- region[3] lats.max <- region[4] - - ## TO DO: - ## Fix region filter for lat and lon - ## Fix 'number' parameter for mask - - data <- startR::Start(dat = mask_sea_land_path, - var = 'lsm', - lon = 'all', - lat = 'all', - number = 1, ## needed to add for ensemble member dimension of lsm.nc - # lon = values(list(lons.min, lons.max)), - # lat = values(list(lats.min, lats.max)), - transform = CDORemapper, transform_extra_cells = 2, - transform_params = list(grid = grid, method = 'con', crop = region), - transform_vars = c('lat','lon'), + data <- startR::Start(dat = mask_path, + var = lsm_var_name, + lon = values(list(lons.min, lons.max)), + lat = values(list(lats.min, lats.max)), return_vars = list(lat = NULL, lon = NULL), synonims = list(lon = c('lon','longitude'), lat = c('lat','latitude')), - lat_reorder = Sort(decreasing = FALSE), - lon_reorder = CircularSort(0,359.9), + lat_reorder = Sort(decreasing = TRUE), + lon_reorder = CircularSort(0,360), num_procs = 1, retrieve = TRUE) - + mask <- list(mask = drop(data), lon = as.numeric(attr(data,'Variables')$common$lon), lat = as.numeric(attr(data,'Variables')$common$lat)) mask$mask[data <= 0.5] <- sea_value mask$mask[data > 0.5] <- land_value - names(dim(mask$mask)) <- c(lon_dim, lat_dim) + names(dim(mask$mask)) <- c(lon_dim, lat_dim) return(mask) } diff --git a/recipes/atomic_recipes/recipe_tas-tos_nadia.yml b/recipes/atomic_recipes/recipe_tas-tos_nadia.yml new file mode 100644 index 00000000..466659a2 --- /dev/null +++ b/recipes/atomic_recipes/recipe_tas-tos_nadia.yml @@ -0,0 +1,55 @@ +Description: + Author: V. Agudetse + +Analysis: + Horizon: Seasonal + Variables: + name: tas-tos + sic_threshold: 0.15 ## sea ice threshold for tas-tos blending, default = 0.15 + freq: monthly_mean + Datasets: + System: + name: ECMWF-SEAS5 + Multimodel: False + Reference: + name: BEST + Time: + sdate: '0101' + #fcst_year: + hcst_start: '2014' + hcst_end: '2016' + ftime_min: 1 + ftime_max: 1 + Region: + latmin: -90 + latmax: 90 + lonmin: 0 + lonmax: 359.9 + Regrid: + method: bilinear + type: to_system + Workflow: + Calibration: + method: raw + save: 'none' + Anomalies: + compute: yes + cross_validation: no + save: 'none' + Skill: + metric: mean_bias EnsCorr RPS RPSS CRPS CRPSS enssprerr + cross_validation: no + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3]] + save: 'none' + Indicators: + index: no + ncores: 15 + remove_NAs: yes + Output_format: scorecards +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/nmilders/scorecards_data/test/ + code_dir: /esarchive/scratch/nmilders/gitlab/git_clones/s2s-suite_tas-tos/ -- GitLab From ae6bf4af1573a6d6e9df6fd06608313953a62267 Mon Sep 17 00:00:00 2001 From: Nadia Milders Date: Thu, 26 Oct 2023 17:47:41 +0200 Subject: [PATCH 2/7] included sic_threshold default value --- modules/Loading/R/load_tas_tos.R | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R index ae39c6bc..93d2ce69 100644 --- a/modules/Loading/R/load_tas_tos.R +++ b/modules/Loading/R/load_tas_tos.R @@ -23,12 +23,18 @@ load_tas_tos <- function(recipe) { ref.name <- recipe$Analysis$Datasets$Reference$name exp.name <- recipe$Analysis$Datasets$System$name - data_order <- c('dat', 'var', 'sday', 'sweek', 'syear', 'time', 'latitude', 'longitude', 'ensemble') - variable <- c("tas", "tos", "sic") - sic.threshold <- recipe$Analysis$Variables$sic_threshold store.freq <- recipe$Analysis$Variables$freq + if(is.null(recipe$Analysis$Variables$sic_threshold)){ + sic.threshold = 0.15 + } else { + sic.threshold <- recipe$Analysis$Variables$sic_threshold + } + + data_order <- c('dat', 'var', 'sday', 'sweek', 'syear', 'time', 'latitude', 'longitude', 'ensemble') + + # get sdates array ## LOGGER: Change dates2load to extract logger from recipe? sdates <- dates2load(recipe, recipe$Run$logger) -- GitLab From c40b8414c1d387817c3790488d2b2cf5df786958 Mon Sep 17 00:00:00 2001 From: Nadia Milders Date: Wed, 8 Nov 2023 16:36:37 +0100 Subject: [PATCH 3/7] Including tas-tos unit test --- .../recipe-seasonal_monthly_1_tas-tos.yml | 56 ++++ .../testthat/test-seasonal_monthly_tas-tos.R | 251 ++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml create mode 100644 tests/testthat/test-seasonal_monthly_tas-tos.R diff --git a/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml b/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml new file mode 100644 index 00000000..768b8e71 --- /dev/null +++ b/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml @@ -0,0 +1,56 @@ +Description: + Author: V. Agudetse + +Analysis: + Horizon: Seasonal + Variables: + name: tas-tos + sic_threshold: 0.15 + freq: monthly_mean + Datasets: + System: + name: ECMWF-SEAS5 + Multimodel: False + Reference: + name: ERA5 + Time: + sdate: '1101' + fcst_year: '2020' + hcst_start: '1993' + hcst_end: '1996' + ftime_min: 1 + ftime_max: 3 + Region: + latmin: 17 + latmax: 20 + lonmin: 12 + lonmax: 15 + Regrid: + method: bilinear + type: to_system + Workflow: + Anomalies: + compute: no + cross_validation: + save: 'none' + Calibration: + method: mse_min + save: 'all' + Skill: + metric: RPSS CRPSS EnsCorr Corr_individual_members Enscorr_specs + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3], [1/10, 9/10]] + save: 'all' + Indicators: + index: no + Visualization: + plots: skill_metrics most_likely_terciles forecast_ensemble_mean + multi_panel: yes + projection: cylindrical_equidistant + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: ./tests/out-logs/ + code_dir: /esarchive/scratch/nmilders/gitlab/git_clones/auto-s2s/ ##/esarchive/scratch/vagudets/repos/auto-s2s/ diff --git a/tests/testthat/test-seasonal_monthly_tas-tos.R b/tests/testthat/test-seasonal_monthly_tas-tos.R new file mode 100644 index 00000000..c942cc79 --- /dev/null +++ b/tests/testthat/test-seasonal_monthly_tas-tos.R @@ -0,0 +1,251 @@ +context("Seasonal monthly data") + +source("modules/Loading/Loading.R") +source("modules/Calibration/Calibration.R") +source("modules/Skill/Skill.R") +source("modules/Saving/Saving.R") +source("modules/Visualization/Visualization.R") + +recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml" +recipe <- prepare_outputs(recipe_file, disable_checks = F) +archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive + +# Load datasets +suppressWarnings({invisible(capture.output( +data <- Loading(recipe) +))}) + +# Calibrate data +suppressWarnings({invisible(capture.output( +calibrated_data <- Calibration(recipe, data) +))}) + +# Compute skill metrics +suppressWarnings({invisible(capture.output( +skill_metrics <- Skill(recipe, calibrated_data) +))}) + +suppressWarnings({invisible(capture.output( +probs <- Probabilities(recipe, calibrated_data) +))}) + +# Saving +suppressWarnings({invisible(capture.output( +Saving(recipe = recipe, data = calibrated_data, + skill_metrics = skill_metrics, probabilities = probs) +))}) + +# Plotting +suppressWarnings({invisible(capture.output( +Visualization(recipe = recipe, data = calibrated_data, + skill_metrics = skill_metrics, probabilities = probs, + significance = T) +))}) +outdir <- get_dir(recipe = recipe, variable = data$hcst$attrs$Variable$varName) + +# ------- TESTS -------- + +test_that("1. Loading", { + +expect_equal( +is.list(data), +TRUE +) +expect_equal( +names(data), +c("hcst", "fcst", "obs") +) +expect_equal( +class(data$hcst), +"s2dv_cube" +) +expect_equal( +class(data$fcst), +"s2dv_cube" +) +expect_equal( +class(data$obs), +"s2dv_cube" +) +expect_equal( +names(data$hcst), +c("data", "dims", "coords", "attrs") +) +expect_equal( +names(data$hcst), +names(data$fcst) +) +expect_equal( +names(data$hcst), +names(data$obs) +) +expect_equal( +dim(data$hcst$data), +c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 4, time = 3, latitude = 3, longitude = 3, ensemble = 25) +) +expect_equal( +dim(data$fcst$data), +c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 1, time = 3, latitude = 3, longitude = 3, ensemble = 51) +) +expect_equal( +dim(data$hcst$attrs$Dates), +c(sday = 1, sweek = 1, syear = 4, time = 3) +) +expect_equal( +as.vector(drop(data$hcst$data)[1:2,1:2,1,2,3]), +c(293.9651, 295.9690, 290.6771, 290.7957), +tolerance = 0.0001 +) +expect_equal( +mean(data$hcst$data), +290.8758, +tolerance = 0.0001 +) +expect_equal( +range(data$hcst$data), +c(284.7413, 299.6219), +tolerance = 0.0001 +) +expect_equal( +(data$hcst$attrs$Dates)[1], +as.POSIXct("1993-11-30 23:59:59", tz = 'UTC') +) +expect_equal( +(data$hcst$attrs$Dates)[2], +as.POSIXct("1994-11-30 23:59:59", tz = 'UTC') +) +expect_equal( +(data$hcst$attrs$Dates)[5], +as.POSIXct("1993-12-31 23:59:59", tz = 'UTC') +) +expect_equal( +(data$obs$attrs$Dates)[10], +as.POSIXct("1995-01-15 12:00:00", tz = 'UTC') +) + +}) + +test_that("2. Calibration", { + +expect_equal( +is.list(calibrated_data), +TRUE +) +expect_equal( +names(calibrated_data), +c("hcst", "obs", "fcst") +) +expect_equal( +class(calibrated_data$hcst), +"s2dv_cube" +) +expect_equal( +class(calibrated_data$fcst), +"s2dv_cube" +) +expect_equal( +dim(calibrated_data$hcst$data), +c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 4, time = 3, latitude = 3, longitude = 3, ensemble = 25) +) +expect_equal( +dim(calibrated_data$fcst$data), +c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 1, time = 3, latitude = 3, longitude = 3, ensemble = 51) +) +expect_equal( +mean(calibrated_data$fcst$data), +291.6433, +tolerance = 0.0001 +) +expect_equal( +mean(calibrated_data$hcst$data), +290.9006, +tolerance = 0.0001 +) +expect_equal( +as.vector(drop(calibrated_data$hcst$data)[1, , 2, 3, 4]), +c(291.8887, 287.0233, 289.8808), +tolerance = 0.0001 +) +expect_equal( +range(calibrated_data$fcst$data), +c(283.8926, 299.0644), +tolerance = 0.0001 +) + +}) + + +#====================================== +test_that("3. Metrics", { + +expect_equal( +is.list(skill_metrics), +TRUE +) +expect_equal( +names(skill_metrics), +c("rpss", "rpss_significance", "crpss", "crpss_significance", "enscorr", + "enscorr_significance", "corr_individual_members", "corr_individual_members_significance", + "enscorr_specs") +) +expect_equal( +class(skill_metrics$rpss), +"array" +) +expect_equal( +dim(skill_metrics$rpss), +c(var = 1, time = 3, latitude = 3, longitude = 3) +) +expect_equal( +dim(skill_metrics$rpss_significance), +dim(skill_metrics$rpss) +) +expect_equal( +as.vector(skill_metrics$rpss[, , 2, 3]), +c(-0.2918857, -1.4809143, -1.3842286), +tolerance = 0.0001 +) +expect_equal( +as.vector(skill_metrics$rpss_significance[, , 2, 3]), +rep(FALSE, 3) +) + +}) + +test_that("4. Saving", { +outputs <- paste0(recipe$Run$output_dir, "/outputs/") +expect_equal( +all(basename(list.files(outputs, recursive = T)) %in% +c("tas_19931101.nc", "tas_19941101.nc", "tas_19951101.nc", + "tas_19961101.nc", "tas_20201101.nc", "tas-corr_month11.nc", + "tas-obs_19931101.nc", "tas-obs_19941101.nc", "tas-obs_19951101.nc", + "tas-obs_19961101.nc", "tas-percentiles_month11.nc", "tas-probs_19931101.nc", + "tas-probs_19941101.nc", "tas-probs_19951101.nc", "tas-probs_19961101.nc", + "tas-probs_20201101.nc", "tas-skill_month11.nc")), +TRUE +) +expect_equal( +length(list.files(outputs, recursive = T)), +17 +) + +}) + +test_that("5. Visualization", { +plots <- paste0(recipe$Run$output_dir, "/plots/") +expect_equal( +all(basename(list.files(plots, recursive = T)) %in% +c("crpss-november.png", "enscorr_specs-november.png", "enscorr-november.png", + "forecast_ensemble_mean-20201101.png", "forecast_most_likely_tercile-20201101.png", + "rpss-november.png")), +TRUE +) +expect_equal( +length(list.files(plots, recursive = T)), +6 +) + +}) + +# Delete files +unlink(recipe$Run$output_dir, recursive = T) -- GitLab From 4c330305802a04886afe66fa06f2478ac03b7242 Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Tue, 14 Nov 2023 13:24:33 +0100 Subject: [PATCH 4/7] Remove/Add TODOs, reformat code --- modules/Loading/R/load_tas_tos.R | 60 +++++++++++--------------------- 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R index 93d2ce69..7ba7fa7c 100644 --- a/modules/Loading/R/load_tas_tos.R +++ b/modules/Loading/R/load_tas_tos.R @@ -3,11 +3,8 @@ source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") source("modules/Loading/R/dates2load.R") source("modules/Loading/R/get_timeidx.R") source("modules/Loading/R/check_latlon.R") - source('modules/Loading/R/mask_tas_tos.R') -## TODO: Source new s2dv_cube version -## TODO: Eliminate dim_var dimension (merge_across_dims?) load_tas_tos <- function(recipe) { @@ -52,16 +49,8 @@ load_tas_tos <- function(recipe) { time_freq=store.freq) } - ## TODO: Examine this verifications part, verify if it's necessary - # stream <- verifications$stream - # sdates <- verifications$fcst.sdate - - ## TODO: define fcst.name - ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name - # get esarchive datasets dict: - ## TODO: Adapt to 'filesystem' option in recipe - archive <- read_yaml("conf/archive.yml")$esarchive + archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable[1]]) @@ -73,13 +62,6 @@ load_tas_tos <- function(recipe) { fcst.nmember <- exp_descrip$nmember$fcst hcst.nmember <- exp_descrip$nmember$hcst - ## TODO: it is necessary? - ##if ("accum" %in% names(reference_descrip)) { - ## accum <- unlist(reference_descrip$accum[store.freq][[1]]) - ##} else { - ## accum <- FALSE - ##} - var_dir_obs <- reference_descrip[[store.freq]][variable] var_dir_exp <- exp_descrip[[store.freq]][variable] @@ -373,7 +355,7 @@ load_tas_tos <- function(recipe) { # Combine obs tas and tos data #------------------------------------------------------------------- - + ## TODO: Ask about this list if(!recipe$Analysis$Datasets$Reference$name %in% c('HadCRUT4','HadCRUT5','BEST','GISTEMPv4')){ obs <- mask_tas_tos(input_data = obs, region = c(lons.min, lons.max,lats.min, lats.max), @@ -396,9 +378,12 @@ load_tas_tos <- function(recipe) { if(recipe$Analysis$Regrid$type == 'to_system'){ aux <- CDORemap(data_array = obs$data, ## Not regridding to desired grid when latitudes are ordered descending - lons = obs$coords$longitude, lats = obs$coords$latitude, - grid = regrid_params$obs.gridtype, method = recipe$Analysis$Regrid$method, - avoid_writes = TRUE, crop = c(lons.min, lons.max,lats.min, lats.max), + lons = obs$coords$longitude, + lats = obs$coords$latitude, + grid = regrid_params$obs.gridtype, + method = recipe$Analysis$Regrid$method, + avoid_writes = TRUE, + crop = c(lons.min, lons.max,lats.min, lats.max), force_remap = TRUE) obs$data <- aux$data_array @@ -427,8 +412,10 @@ load_tas_tos <- function(recipe) { if (!is.null(recipe$Analysis$Time$fcst_year)) { aux <- CDORemap(data_array = fcst$data, - lons = fcst$coords$longitude, lats = fcst$coords$latitude, - grid = regrid_params$fcst.gridtype, method = recipe$Analysis$Regrid$method, + lons = fcst$coords$longitude, + lats = fcst$coords$latitude, + grid = regrid_params$fcst.gridtype, + method = recipe$Analysis$Regrid$method, avoid_writes = TRUE, crop = TRUE, force_remap = TRUE) @@ -459,8 +446,10 @@ load_tas_tos <- function(recipe) { if (!is.null(recipe$Analysis$Time$fcst_year)) { aux <- CDORemap(data_array = fcst$data, - lons = fcst$coords$longitude, lats = fcst$coords$latitude, - grid = regrid_params$fcst.gridtype, method = recipe$Analysis$Regrid$method, + lons = fcst$coords$longitude, + lats = fcst$coords$latitude, + grid = regrid_params$fcst.gridtype, + method = recipe$Analysis$Regrid$method, avoid_writes = TRUE, crop = TRUE, force_remap = TRUE) @@ -473,8 +462,10 @@ load_tas_tos <- function(recipe) { } aux <- CDORemap(data_array = obs$data, - lons = obs$coords$longitude, lats = obs$coords$latitude, - grid = regrid_params$obs.gridtype, method = recipe$Analysis$Regrid$method, + lons = obs$coords$longitude, + lats = obs$coords$latitude, + grid = regrid_params$obs.gridtype, + method = recipe$Analysis$Regrid$method, avoid_writes = TRUE, crop = TRUE, force_remap = TRUE) @@ -518,19 +509,8 @@ load_tas_tos <- function(recipe) { } } - - # Compute anomalies if requested - # Print a summary of the loaded data for the user, for each object - if (recipe$Run$logger$threshold <= 2) { - data_summary(hcst, recipe) - data_summary(obs, recipe) - if (!is.null(fcst)) { - data_summary(fcst, recipe) - } - } info(recipe$Run$logger, "##### DATA LOADING COMPLETED SUCCESSFULLY #####") return(list(hcst = hcst, fcst = fcst, obs = obs)) - } -- GitLab From 82231bcb55c29ec66a49ccd6bd79b83a6b9e82ca Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Wed, 15 Nov 2023 10:51:49 +0100 Subject: [PATCH 5/7] Fix tas-tos test recipe, update load_tas_tos() --- conf/archive.yml | 360 ++++++------------ modules/Loading/R/load_tas_tos.R | 11 +- .../recipe-seasonal_monthly_1_tas-tos.yml | 34 +- 3 files changed, 130 insertions(+), 275 deletions(-) diff --git a/conf/archive.yml b/conf/archive.yml index 4ea8c4c9..cca68c22 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -5,36 +5,43 @@ esarchive: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/ecmwf/system5c3s/" - daily_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", - "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/", - "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", - "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", - "tdps":"_f6h/", "hurs":"_f6h/"} - monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", - "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/", - "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", - "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", - "tdps":"_f6h/", "tos":"_f6h/", "sic":"_f24h/"} + daily_mean: {"tas":"daily_mean/tas_f6h/", "rsds":"daily/rsds_s0-24h/", + "prlr":"daily/prlr_s0-24h/", "tasmin":"daily/tasmin/", + "tasmax":"daily/tasmax/", "sfcWind":"daily_mean/sfcWind_f6h/", + "ta300":"daily_mean/ta300_f12h/", "ta500":"daily_mean/ta500_f12h/", + "ta850":"daily_mean/ta850_f12h/", "g300":"daily_mean/g300_f12h/", + "g500":"daily_mean/g500_f12h/", "g850":"daily_mean/g850_f12h/", + "tdps":"daily_mean/tdps_f6h/", "hurs":"daily_mean/hurs_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "rsds":"monthly_mean/rsds_s0-24h/", + "prlr":"monthly_mean/prlr_s0-24h/", + "sfcWind":"monthly_mean/sfcWind_f6h/", + "tasmin":"monthly_mean/tasmin_f24h/", + "tasmax":"monthly_mean/tasmax_f24h/", + "ta300":"monthly_mean/ta300_f12h/", "ta500":"monthly_mean/ta500_f12h/", + "ta850":"monthly_mean/ta850_f12h/", "g300":"monthly_mean/g300_f12h/", + "g500":"monthly_mean/g500_f12h/", "g850":"monthly_mean/g500_f12h/", + "tdps":"monthly_mean/tdps_f6h/", "psl":"monthly_mean/psl_f6h/", + "tos":"monthly_mean/tos_f6h/", "sic":"monthly_mean/sic_f24h/"} nmember: fcst: 51 hcst: 25 calendar: "proleptic_gregorian" time_stamp_lag: "0" reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" - land_sea_mask: "/esarchive/exp/ecmwf/system5c3s/constant/lsm/lsm.nc" # $var$ + land_sea_mask: "/esarchive/exp/ecmwf/system5c3s/constant/lsm/lsm.nc" ECMWF-SEAS5.1: name: "ECMWF SEAS5 (v5.1)" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/ecmwf/system51c3s/" - daily_mean: {"tas":"_f6h/", "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", - "tdps":"_f6h/"} - monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", "prlr":"_s0-24h/", - "sfcWind":"_f6h/", "tasmin":"_f24h/", "tasmax":"_f24h/", - "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", - "tdps":"_f6h/"} + daily_mean: {"tas":"daily_mean/tas_f6h/", "prlr":"daily/prlr_s0-24h/", + "sfcWind":"daily_mean/sfcWind_f6h/", + "uas":"daily_mean/uas_f6h/", "vas":"daily_mean/vas_f6h/", + "psl":"daily_mean/psl_f6h/", "tdps":"daily_mean/tdps_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "rsds":"monthly_mean/rsds_s0-24h/", + "prlr":"monthly_mean/prlr_s0-24h/", "sfcWind":"monthly_mean/sfcWind_f6h/", + "tasmin":"monthly_mean/tasmin_f24h/", "tasmax":"monthly_mean/tasmax_f24h/", + "uas":"monthly_mean/uas_f6h/", "vas":"monthly_mean/vas_f6h/", + "psl":"monthly_mean/psl_f6h/", "tdps":"monthly_mean/tdps_f6h/"} nmember: fcst: 51 hcst: 25 @@ -45,9 +52,9 @@ esarchive: name: "Meteo-France System 7" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/meteofrance/system7c3s/" - monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/", - "tasmax":"_f6h/", "tasmin": "_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "g500":"monthly_mean/g500_f12h/", + "prlr":"monthly_mean/prlr_f24h/", "sfcWind": "monthly_mean/sfcWind_f6h/", + "tasmax":"monthly_mean/tasmax_f6h/", "tasmin": "monthly_mean/tasmin_f6h/"} nmember: fcst: 51 hcst: 25 @@ -58,9 +65,9 @@ esarchive: name: "DWD GCFS 2.1" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/dwd/system21_m1/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f24h/", - "g500":"_f12h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", + "g500":"monthly_mean/g500_f12h/", "sfcWind":"monthly_mean/sfcWind_f6h/", + "tasmin":"monthly_mean/tasmin_f24h/", "tasmax":"monthly_mean/tasmax_f24h/"} nmember: fcst: 50 hcst: 30 @@ -71,9 +78,9 @@ esarchive: name: "CMCC-SPS3.5" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/cmcc/system35c3s/" - monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/", - "tasmax":"_f24h/", "tasmin":"_f24h"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f24h/", + "g500":"monthly_mean/g500_f12h/", "sfcWind":"monthly_mean/sfcWind_f6h/", + "tasmin":"monthly_mean/tasmin_f24h/", "tasmax":"monthly_mean/tasmax_f24h/"} nmember: fcst: 50 hcst: 40 @@ -84,8 +91,8 @@ esarchive: name: "JMA System 2" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/jma/system2c3s/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", + "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} nmember: fcst: 10 hcst: 10 @@ -96,8 +103,8 @@ esarchive: name: "ECCC CanCM4i" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/eccc/eccc1/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", + "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} nmember: fcst: 10 hcst: 10 @@ -108,8 +115,8 @@ esarchive: name: "UK MetOffice GloSea 6 (v6.0)" institution: "European Centre for Medium-Range Weather Forecasts" src: "exp/ukmo/glosea6_system600-c3s/" - monthly_mean: {"tas":"_f6h/", "tasmin":"_f24h/", - "tasmax":"_f24h/", "prlr":"_f24h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "tasmin":"monthly_mean/tasmin_f24h/", + "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f24h/"} nmember: fcst: 62 hcst: 28 @@ -120,8 +127,8 @@ esarchive: name: "NCEP CFSv2" institution: "NOAA NCEP" #? src: "exp/ncep/cfs-v2/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/", "prlr":"monthly_mean/prlr_f6h/", + "tasmax":"monthly_mean/tasmax_f6h/", "tasmin":"monthly_mean/tasmin_f6h/"} nmember: fcst: 20 hcst: 20 @@ -133,34 +140,34 @@ esarchive: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/era5/" - daily_mean: {"tas":"_f1h-r1440x721cds/", - "rsds":"_f1h-r1440x721cds/", - "prlr":"_f1h-r1440x721cds/", - "g300":"_f1h-r1440x721cds/", - "g500":"_f1h-r1440x721cds/", - "g850":"_f1h-r1440x721cds/", - "sfcWind":"_f1h-r1440x721cds/", - "tasmax":"_f1h-r1440x721cds/", - "tasmin":"_f1h-r1440x721cds/", - "ta300":"_f1h-r1440x721cds/", - "ta500":"_f1h-r1440x721cds/", - "ta850":"_f1h-r1440x721cds/", - "hurs":"_f1h-r1440x721cds/"} - monthly_mean: {"tas":"_f1h-r1440x721cds/", - "prlr":"_f1h-r1440x721cds/", - "rsds":"_f1h-r1440x721cds/", - "g300":"_f1h-r1440x721cds/", - "g500":"_f1h-r1440x721cds/", - "g850":"_f1h-r1440x721cds/", - "sfcWind":"_f1h-r1440x721cds/", - "tasmax":"_f1h-r1440x721cds/", - "tasmin":"_f1h-r1440x721cds/", - "ta300":"_f1h-r1440x721cds/", - "ta500":"_f1h-r1440x721cds/", - "ta850":"_f1h-r1440x721cds/", - "tos":"_f1h-r1440x721cds", - "sic":"_f1h-r1440x721cds", - } + daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", + "rsds":"daily_mean/rsds_f1h-r1440x721cds/", + "prlr":"daily_mean/prlr_f1h-r1440x721cds/", + "g300":"daily_mean/g300_f1h-r1440x721cds/", + "g500":"daily_mean/g500_f1h-r1440x721cds/", + "g850":"daily_mean/g850_f1h-r1440x721cds/", + "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", + "tasmax":"daily/tasmax_f1h-r1440x721cds/", + "tasmin":"daily/tasmin_f1h-r1440x721cds/", + "ta300":"daily_mean/ta300_f1h-r1440x721cds/", + "ta500":"daily_mean/ta500_f1h-r1440x721cds/", + "ta850":"daily_mean/ta850_f1h-r1440x721cds/", + "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} + monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", + "psl":"monthly_mean/psl_f1h-r1440x721cds/", + "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", + "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", + "g300":"monthly_mean/g300_f1h-r1440x721cds/", + "g500":"monthly_mean/g500_f1h-r1440x721cds/", + "g850":"monthly_mean/g850_f1h-r1440x721cds/", + "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", + "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", + "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", + "ta300":"montly_mean/ta300_f1h-r1440x721cds/", + "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", + "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", + "tos":"monthly_mean/tos_f1h-r1440x721cds/", + "sic":"monthly_mean/sic_f1h-r1440x721cds/"} calendar: "standard" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" @@ -168,230 +175,81 @@ esarchive: name: "ERA5-Land" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"_f1h/", "rsds":"_f1h/", - "prlr":"_f1h/", "sfcWind":"_f1h/"} - monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/", - "tasmax":"_f24h/", "prlr":"_f1h/", - "sfcWind":"_f1h/", "rsds":"_f1h/", - "tdps":"_f1h/"} + daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", + "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", + "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} + monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", + "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", + "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", + "tdps":"monthly_mean/tdps_f1h/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" UERRA: name: "ECMWF UERRA" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"_f6h/"} - monthly_mean: {"tas":"_f6h/"} + daily_mean: {"tas":"daily_mean/tas_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" CERRA: name: "ECMWF CERRA" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/cerra/" - daily_mean: {"hur":"_f3h-r2631x1113/", "ps":"_f3h-r2631x1113/", "sfcWind":"_f3h-r2631x1113/", - "tas":"_f3h-r2631x1113/", "winddir":"_f3h-r2631x1113/"} - monthly_mean: {"hur":"_f3h-r2631x1113/", "ps":"_f3h-r2631x1113/", "sfcWind":"_f3h-r2631x1113/", - "tas":"_f3h-r2631x1113/", "winddir":"_f3h-r2631x1113/","tasmin":"_f24h-r2631x1113/","tasmax":"_f24h-r2631x1113/"} + daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", + "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", + "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} + monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", + "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", + "tas":"monthly_mean/tas_f3h-r2631x1113/", + "winddir":"monthly_mean/winddir_f3h-r2631x1113/", + "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", + "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" CERRA-Land: name: "ECMWF CERRA-Land" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/cerraland/" - daily_mean: {"prlr":"_f6h-r2631x1113/"} - monthly_mean: {"prlr":"_f6h-r2631x1113/"} + daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} + monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" + HadCRUT5: + name: "HadCRUT5" + institution: "Met Office" + src: "obs/ukmo/hadcrut_v5.0_analysis/" + monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" BEST: name: "BEST" institution: "European Centre for Medium-Range Weather Forecasts" src: "obs/berkeleyearth/berkeleyearth/" - daily_mean: {"tas":"/"} - monthly_mean: {"tas":"/"} + daily_mean: {"tas":"daily_mean/tas/"} + monthly_mean: {"tas":"monthly_mean/tas/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" - - - mars: - src: "/esarchive/" + src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/ecmwf/system5c3s/" - daily_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", - "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/", - "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", - "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", - "tdps":"_f6h/", "hurs":"_f6h/"} - monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", - "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/", - "ta300":"_f12h/", "ta500":"_f12h/", "ta850":"_f12h/", - "g300":"_f12h/", "g500":"_f12h/", "g850":"_f12h/", - "tdps":"_f6h/"} - nmember: - fcst: 51 - hcst: 25 - calendar: "proleptic_gregorian" - time_stamp_lag: "0" - reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" - ECMWF-SEAS5.1: - name: "ECMWF SEAS5 (v5.1)" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/ecmwf/system51c3s/" - daily_mean: {"tas":"_f6h/", "prlr":"_s0-24h/", "sfcWind":"_f6h/", - "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", - "tdps":"_f6h/"} - monthly_mean: {"tas":"_f6h/", "rsds":"_s0-24h/", "prlr":"_s0-24h/", - "sfcWind":"_f6h/", "tasmin":"_f24h/", "tasmax":"_f24h/", - "uas":"_f6h/", "vas":"_f6h/", "psl":"_f6h/", - "tdps":"_f6h/"} - nmember: - fcst: 51 - hcst: 25 - calendar: "proleptic_gregorian" - time_stamp_lag: "0" - reference_grid: "conf/grid_description/griddes_system51c3s.txt" - Meteo-France-System7: - name: "Meteo-France System 7" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/meteofrance/system7c3s/" - monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/", - "tasmax":"_f6h/", "tasmin": "_f6h/"} + src: "GRIB_system5_tas_CORRECTED/" + monthly_mean: {"tas":""} nmember: fcst: 51 - hcst: 25 - time_stamp_lag: "+1" - calendar: "proleptic_gregorian" - reference_grid: "conf/grid_description/griddes_system7c3s.txt" - DWD-GCFS2.1: - name: "DWD GCFS 2.1" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/dwd/system21_m1/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f24h/", - "g500":"_f12h/", "sfcWind":"_f6h/", - "tasmin":"_f24h/", "tasmax":"_f24h/"} - nmember: - fcst: 50 - hcst: 30 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_system21_m1.txt" - CMCC-SPS3.5: - name: "CMCC-SPS3.5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/cmcc/system35c3s/" - monthly_mean: {"tas":"_f6h/", "g500":"_f12h/", - "prlr":"_f24h/", "sfcWind": "_f6h/", - "tasmax":"_f24h/", "tasmin":"_f24h"} - nmember: - fcst: 50 - hcst: 40 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_system35c3s.txt" - JMA-CPS2: - name: "JMA System 2" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/jma/system2c3s/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} - nmember: - fcst: 10 - hcst: 10 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_system2c3s.txt" - ECCC-CanCM4i: - name: "ECCC CanCM4i" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/eccc/eccc1/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} - nmember: - fcst: 10 - hcst: 10 - calendar: "proleptic_gregorian" - time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_eccc1.txt" - UK-MetOffice-Glosea600: - name: "UK MetOffice GloSea 6 (v6.0)" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "exp/ukmo/glosea6_system600-c3s/" - monthly_mean: {"tas":"_f6h/", "tasmin":"_f24h/", - "tasmax":"_f24h/", "prlr":"_f24h/"} - nmember: - fcst: 62 - hcst: 28 + hcst: 51 calendar: "proleptic_gregorian" time_stamp_lag: "+1" - reference_grid: "conf/grid_description/griddes_ukmo600.txt" - NCEP-CFSv2: - name: "NCEP CFSv2" - institution: "NOAA NCEP" #? - src: "exp/ncep/cfs-v2/" - monthly_mean: {"tas":"_f6h/", "prlr":"_f6h/", - "tasmax":"_f6h/", "tasmin":"_f6h/"} - nmember: - fcst: 20 - hcst: 20 - calendar: "gregorian" - time_stamp_lag: "0" - reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" Reference: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - daily_mean: {"tas":"_f1h-r1440x721cds/", - "rsds":"_f1h-r1440x721cds/", - "prlr":"_f1h-r1440x721cds/", - "g300":"_f1h-r1440x721cds/", - "g500":"_f1h-r1440x721cds/", - "g850":"_f1h-r1440x721cds/", - "sfcWind":"_f1h-r1440x721cds/", - "tasmax":"_f1h-r1440x721cds/", - "tasmin":"_f1h-r1440x721cds/", - "ta300":"_f1h-r1440x721cds/", - "ta500":"_f1h-r1440x721cds/", - "ta850":"_f1h-r1440x721cds/", - "hurs":"_f1h-r1440x721cds/"} - monthly_mean: {"tas":"_f1h-r1440x721cds/", - "prlr":"_f1h-r1440x721cds/", - "rsds":"_f1h-r1440x721cds/", - "g300":"_f1h-r1440x721cds/", - "g500":"_f1h-r1440x721cds/", - "g850":"_f1h-r1440x721cds/", - "sfcWind":"_f1h-r1440x721cds/", - "tasmax":"_f1h-r1440x721cds/", - "tasmin":"_f1h-r1440x721cds/", - "ta300":"_f1h-r1440x721cds/", - "ta500":"_f1h-r1440x721cds/", - "ta850":"_f1h-r1440x721cds/"} + src: "GRIB_era5_tas/" + monthly_mean: {"tas":""} calendar: "standard" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - ERA5-Land: - name: "ERA5-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"_f1h/", "rsds":"_f1h/", - "prlr":"_f1h/", "sfcWind":"_f1h/"} - monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/", - "tasmax":"_f24h/", "prlr":"_f1h/", - "sfcWind":"_f1h/", "rsds":"_f1h/", - "tdps":"_f1h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - UERRA: - name: "ECMWF UERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"_f6h/"} - monthly_mean: {"tas":"_f6h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R index 7ba7fa7c..5e6b8837 100644 --- a/modules/Loading/R/load_tas_tos.R +++ b/modules/Loading/R/load_tas_tos.R @@ -66,16 +66,13 @@ load_tas_tos <- function(recipe) { var_dir_exp <- exp_descrip[[store.freq]][variable] # ----------- - obs.path <- paste0(archive$src, - obs.dir, store.freq, "/$var$", "$var_dir$", - "/$var$_$file_date$.nc") + obs.path <- paste0(archive$src, obs.dir, "$var_dir$", + "$var$_$file_date$.nc") - hcst.path <- paste0(archive$src, - hcst.dir, store.freq, "/$var$", "$var_dir$", + hcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") - fcst.path <- paste0(archive$src, - hcst.dir, store.freq, "/$var$", "$var_dir$", + fcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", "/$var$_$file_date$.nc") # Define regrid parameters: diff --git a/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml b/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml index 768b8e71..17855ac2 100644 --- a/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml +++ b/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml @@ -1,5 +1,5 @@ Description: - Author: V. Agudetse + Author: N. Milders Analysis: Horizon: Seasonal @@ -15,7 +15,7 @@ Analysis: name: ERA5 Time: sdate: '1101' - fcst_year: '2020' + fcst_year: # '2020' hcst_start: '1993' hcst_end: '1996' ftime_min: 1 @@ -33,21 +33,21 @@ Analysis: compute: no cross_validation: save: 'none' - Calibration: - method: mse_min - save: 'all' - Skill: - metric: RPSS CRPSS EnsCorr Corr_individual_members Enscorr_specs - save: 'all' - Probabilities: - percentiles: [[1/3, 2/3], [1/10, 9/10]] - save: 'all' - Indicators: - index: no - Visualization: - plots: skill_metrics most_likely_terciles forecast_ensemble_mean - multi_panel: yes - projection: cylindrical_equidistant + Calibration: + method: mse_min + save: 'all' + Skill: + metric: RPSS CRPSS EnsCorr Corr_individual_members Enscorr_specs + save: 'all' + Probabilities: + percentiles: [[1/3, 2/3], [1/10, 9/10]] + save: 'all' + Indicators: + index: no + Visualization: + plots: skill_metrics most_likely_terciles forecast_ensemble_mean + multi_panel: yes + projection: cylindrical_equidistant Output_format: S2S4E Run: Loglevel: INFO -- GitLab From f7310a4393c25e4b0c7f944d3c621a85557c13e5 Mon Sep 17 00:00:00 2001 From: Nadia Milders Date: Thu, 16 Nov 2023 15:16:23 +0100 Subject: [PATCH 6/7] fixed tas-tos unit test --- .../recipe-seasonal_monthly_1_tas-tos.yml | 4 +- .../testthat/test-seasonal_monthly_tas-tos.R | 173 +----------------- 2 files changed, 12 insertions(+), 165 deletions(-) diff --git a/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml b/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml index 17855ac2..41048db1 100644 --- a/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml +++ b/tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml @@ -14,8 +14,8 @@ Analysis: Reference: name: ERA5 Time: - sdate: '1101' - fcst_year: # '2020' + sdate: '0101' + fcst_year: '2018' hcst_start: '1993' hcst_end: '1996' ftime_min: 1 diff --git a/tests/testthat/test-seasonal_monthly_tas-tos.R b/tests/testthat/test-seasonal_monthly_tas-tos.R index c942cc79..1345f63d 100644 --- a/tests/testthat/test-seasonal_monthly_tas-tos.R +++ b/tests/testthat/test-seasonal_monthly_tas-tos.R @@ -1,10 +1,6 @@ context("Seasonal monthly data") -source("modules/Loading/Loading.R") -source("modules/Calibration/Calibration.R") -source("modules/Skill/Skill.R") -source("modules/Saving/Saving.R") -source("modules/Visualization/Visualization.R") +source("./modules/Loading/Loading.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) @@ -15,33 +11,6 @@ suppressWarnings({invisible(capture.output( data <- Loading(recipe) ))}) -# Calibrate data -suppressWarnings({invisible(capture.output( -calibrated_data <- Calibration(recipe, data) -))}) - -# Compute skill metrics -suppressWarnings({invisible(capture.output( -skill_metrics <- Skill(recipe, calibrated_data) -))}) - -suppressWarnings({invisible(capture.output( -probs <- Probabilities(recipe, calibrated_data) -))}) - -# Saving -suppressWarnings({invisible(capture.output( -Saving(recipe = recipe, data = calibrated_data, - skill_metrics = skill_metrics, probabilities = probs) -))}) - -# Plotting -suppressWarnings({invisible(capture.output( -Visualization(recipe = recipe, data = calibrated_data, - skill_metrics = skill_metrics, probabilities = probs, - significance = T) -))}) -outdir <- get_dir(recipe = recipe, variable = data$hcst$attrs$Variable$varName) # ------- TESTS -------- @@ -81,11 +50,11 @@ names(data$obs) ) expect_equal( dim(data$hcst$data), -c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 4, time = 3, latitude = 3, longitude = 3, ensemble = 25) +c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 4, time = 3, latitude = 4, longitude = 4, ensemble = 25) ) expect_equal( dim(data$fcst$data), -c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 1, time = 3, latitude = 3, longitude = 3, ensemble = 51) +c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 1, time = 3, latitude = 4, longitude = 4, ensemble = 51) ) expect_equal( dim(data$hcst$attrs$Dates), @@ -93,156 +62,34 @@ c(sday = 1, sweek = 1, syear = 4, time = 3) ) expect_equal( as.vector(drop(data$hcst$data)[1:2,1:2,1,2,3]), -c(293.9651, 295.9690, 290.6771, 290.7957), +c(285.5869, 287.8836, 285.9362, 289.0483), tolerance = 0.0001 ) expect_equal( mean(data$hcst$data), -290.8758, +290.1099, tolerance = 0.0001 ) expect_equal( range(data$hcst$data), -c(284.7413, 299.6219), +c(283.2845, 299.7845), tolerance = 0.0001 ) expect_equal( (data$hcst$attrs$Dates)[1], -as.POSIXct("1993-11-30 23:59:59", tz = 'UTC') +as.POSIXct("1993-01-31 18:00:00", tz = 'UTC') ) expect_equal( (data$hcst$attrs$Dates)[2], -as.POSIXct("1994-11-30 23:59:59", tz = 'UTC') +as.POSIXct("1994-01-31 18:00:00", tz = 'UTC') ) expect_equal( (data$hcst$attrs$Dates)[5], -as.POSIXct("1993-12-31 23:59:59", tz = 'UTC') +as.POSIXct("1993-02-28 18:00:00", tz = 'UTC') ) expect_equal( (data$obs$attrs$Dates)[10], -as.POSIXct("1995-01-15 12:00:00", tz = 'UTC') -) - -}) - -test_that("2. Calibration", { - -expect_equal( -is.list(calibrated_data), -TRUE -) -expect_equal( -names(calibrated_data), -c("hcst", "obs", "fcst") -) -expect_equal( -class(calibrated_data$hcst), -"s2dv_cube" -) -expect_equal( -class(calibrated_data$fcst), -"s2dv_cube" -) -expect_equal( -dim(calibrated_data$hcst$data), -c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 4, time = 3, latitude = 3, longitude = 3, ensemble = 25) -) -expect_equal( -dim(calibrated_data$fcst$data), -c(dat = 1, var = 1, sday = 1, sweek = 1, syear = 1, time = 3, latitude = 3, longitude = 3, ensemble = 51) -) -expect_equal( -mean(calibrated_data$fcst$data), -291.6433, -tolerance = 0.0001 -) -expect_equal( -mean(calibrated_data$hcst$data), -290.9006, -tolerance = 0.0001 -) -expect_equal( -as.vector(drop(calibrated_data$hcst$data)[1, , 2, 3, 4]), -c(291.8887, 287.0233, 289.8808), -tolerance = 0.0001 -) -expect_equal( -range(calibrated_data$fcst$data), -c(283.8926, 299.0644), -tolerance = 0.0001 -) - -}) - - -#====================================== -test_that("3. Metrics", { - -expect_equal( -is.list(skill_metrics), -TRUE -) -expect_equal( -names(skill_metrics), -c("rpss", "rpss_significance", "crpss", "crpss_significance", "enscorr", - "enscorr_significance", "corr_individual_members", "corr_individual_members_significance", - "enscorr_specs") -) -expect_equal( -class(skill_metrics$rpss), -"array" -) -expect_equal( -dim(skill_metrics$rpss), -c(var = 1, time = 3, latitude = 3, longitude = 3) -) -expect_equal( -dim(skill_metrics$rpss_significance), -dim(skill_metrics$rpss) -) -expect_equal( -as.vector(skill_metrics$rpss[, , 2, 3]), -c(-0.2918857, -1.4809143, -1.3842286), -tolerance = 0.0001 -) -expect_equal( -as.vector(skill_metrics$rpss_significance[, , 2, 3]), -rep(FALSE, 3) -) - -}) - -test_that("4. Saving", { -outputs <- paste0(recipe$Run$output_dir, "/outputs/") -expect_equal( -all(basename(list.files(outputs, recursive = T)) %in% -c("tas_19931101.nc", "tas_19941101.nc", "tas_19951101.nc", - "tas_19961101.nc", "tas_20201101.nc", "tas-corr_month11.nc", - "tas-obs_19931101.nc", "tas-obs_19941101.nc", "tas-obs_19951101.nc", - "tas-obs_19961101.nc", "tas-percentiles_month11.nc", "tas-probs_19931101.nc", - "tas-probs_19941101.nc", "tas-probs_19951101.nc", "tas-probs_19961101.nc", - "tas-probs_20201101.nc", "tas-skill_month11.nc")), -TRUE -) -expect_equal( -length(list.files(outputs, recursive = T)), -17 -) - -}) - -test_that("5. Visualization", { -plots <- paste0(recipe$Run$output_dir, "/plots/") -expect_equal( -all(basename(list.files(plots, recursive = T)) %in% -c("crpss-november.png", "enscorr_specs-november.png", "enscorr-november.png", - "forecast_ensemble_mean-20201101.png", "forecast_most_likely_tercile-20201101.png", - "rpss-november.png")), -TRUE -) -expect_equal( -length(list.files(plots, recursive = T)), -6 +as.POSIXct("1994-03-15 12:00:00", tz = 'UTC') ) }) -- GitLab From 2472a8af8bbfc53aa865835bb7fd1de8e0bca08a Mon Sep 17 00:00:00 2001 From: Victoria Agudetse Roures Date: Mon, 20 Nov 2023 10:48:11 +0100 Subject: [PATCH 7/7] Add recipe check for sic_threshold, check recipe *after* restructuring --- tools/check_recipe.R | 12 ++++++++++++ tools/prepare_outputs.R | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tools/check_recipe.R b/tools/check_recipe.R index e4c59a55..69a7dd4c 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -62,6 +62,18 @@ check_recipe <- function(recipe) { } else { archive <- NULL } + # Check variable parameters + variables <- tolower(strsplit(recipe$Analysis$Variables$name, ", | |,")[[1]]) + # Sea-ice threshold check + if (("tas-tos" %in% variables) && + (!is.null(recipe$Analysis$Variables$sic_threshold))) { + if (!is.numeric(recipe$Analysis$Variables$sic_threshold) || + dplyr::between(recipe$Analysis$Variables$sic_threshold, 0, 1)) { + error(recipe$Run$logger, + paste("The element Analysis:Variables:sic_threshold must be a", + "numeric value between 0 and 1.")) + } + } # Check system names if (!is.null(archive)) { if (!all(recipe$Analysis$Datasets$System$name %in% names(archive$System))) { diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index fa4d23ed..54e82095 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -89,6 +89,10 @@ prepare_outputs <- function(recipe_file, warn(recipe$Run$logger, "Filesystem not specified in the recipe. Setting it to 'esarchive'.") } + # Restructure the recipe to make the atomic recipe more readable + if (restructure) { + recipe <- restructure_recipe(recipe) + } # Run recipe checker if (disable_checks) { warn(recipe$Run$logger, @@ -96,9 +100,5 @@ prepare_outputs <- function(recipe_file, } else { check_recipe(recipe) } - # Restructure the recipe to make the atomic recipe more readable - if (restructure) { - recipe <- restructure_recipe(recipe) - } return(recipe) } -- GitLab