From 256c9a6a8ee89d99f03c7da52b2faca881714e02 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Wed, 21 Aug 2024 09:49:11 +0200 Subject: [PATCH 01/11] added archive_reference.yml --- conf/archive_reference.yml | 216 +++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 conf/archive_reference.yml diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml new file mode 100644 index 00000000..07772ae8 --- /dev/null +++ b/conf/archive_reference.yml @@ -0,0 +1,216 @@ +# ----- Reference from archiv.yml + + Reference: + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/era5/" + weekly_mean: {"tas":"weekly_mean/tas_f1h-r1440x721cds/", # from subseasonal + "prlr":"weekly_mean/prlr_f1h-r1440x721cds/"} # from subseasonal + daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", + "rsds":"daily_mean/rsds_f1h-r1440x721cds/", + "prlr":"daily_mean/prlr_f1h-r1440x721cds/", + "g300":"daily_mean/g300_f1h-r1440x721cds/", + "g500":"daily_mean/g500_f1h-r1440x721cds/", + "g850":"daily_mean/g850_f1h-r1440x721cds/", + "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", + "tasmax":"daily/tasmax_f1h-r1440x721cds/","daily/tasmax-r1440x721cds/", # second entry from decadal + "tasmin":"daily/tasmin_f1h-r1440x721cds/","daily/tasmin-r1440x721cds/", # second entry from decadal + "ta300":"daily_mean/ta300_f1h-r1440x721cds/", + "ta500":"daily_mean/ta500_f1h-r1440x721cds/", + "ta850":"daily_mean/ta850_f1h-r1440x721cds/", + "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} + monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", + "psl":"monthly_mean/psl_f1h-r1440x721cds/", + "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", + "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", + "g300":"monthly_mean/g300_f1h-r1440x721cds/", + "g500":"monthly_mean/g500_f1h-r1440x721cds/", + "g850":"monthly_mean/g850_f1h-r1440x721cds/", + "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", + "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", + "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", + "ta300":"montly_mean/ta300_f1h-r1440x721cds/", + "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", + "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", + "tos":"monthly_mean/tos_f1h-r1440x721cds/", + "sic":"monthly_mean/sic_f1h-r1440x721cds/"} + + calendar: "standard" # calendar in archive_decadal ERA5 is gregorian + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" + + ERA5-Land: + name: "ERA5-Land" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/era5land/" + daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", + "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", + "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} + monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", + "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", + "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", + "tdps":"monthly_mean/tdps_f1h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" + + UERRA: + name: "ECMWF UERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/uerra_mescan/" + daily_mean: {"tas":"daily_mean/tas_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" + + CERRA: + name: "ECMWF CERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/cerra/" + daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", + "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", + "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/", + "prlr":"daily_mean/prlr_f6h-r2631x1113/", # from second CERRA in archive + "tasmax":"daily_mean/tasmax-r2631x1113/", "tasmin":"daily_mean/tasmin-r2631x1113/"} # from second CERRA in archive + + monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", + "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", + "tas":"monthly_mean/tas_f3h-r2631x1113/", + "winddir":"monthly_mean/winddir_f3h-r2631x1113/", + "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", + "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/", + "prlr":"monthly_mean/prlr_f6h-r2631x1113/" # from second CERRA in archive + } + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" + + CERRA-Land: + name: "ECMWF CERRA-Land" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/cerraland/" + daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} + monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" + HadCRUT5: + name: "HadCRUT5" + institution: "Met Office" + src: "obs/ukmo/hadcrut_v5.0_analysis/" + monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" + BEST: + name: "BEST" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "obs/berkeleyearth/berkeleyearth/" + daily_mean: {"tas":"daily_mean/tas/"} + monthly_mean: {"tas":"monthly_mean/tas/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" + +# ------------------------- +mars: + src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" + + + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "GRIB_era5_tas/" + monthly_mean: {"tas":"", "prlr":""} + calendar: "standard" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" + + ERA5: + src: "ERA5/" # src different from line 113 + daily_mean: {"psl":"psl_f1h-r1440x721cds/"} + reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ERA5/daily_mean/psl_f1h-r1440x721cds/psl_201805.nc" + + CERRA: + src: "CERRA/" + reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/CERRA/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" + + + + +# ----- Reference from archive_decade.yml + + GHCNv4: + name: + institution: + src: "obs/noaa/ghcn_v4/" + monthly_mean: {"tas":"", "tasanomaly":""} + daily_mean: + calendar: "gregorian" + reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" + +# ---- +# missing info in script + NCEP1: + +# ---- + JRA-55: + name: "JRA-55" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/jma/jra55/" + monthly_mean: {"tas":"monthly_mean/tas_f6h", "psl":"monthly_mean/psl_f6h", + "tos":"", "pr":"monthly_mean/pr_s0-3h", + "prlr":"monthly_mean/prlr_s0-3h"} + daily_mean: {"tas":"daily_mean/tas_f6h", "psl":"daily_mean/psl_f6h", + "prlr":"daily_mean/prlr_s0-3h", "sfcWind":"daily_mean/sfcWind_f6h"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" + +# ---- + GISTEMPv4: + name: "GISTEMPv4" + institution: "NASA Goddard Institute for Space Studies" + src: "obs/noaa-nasa/ghcnersstgiss/" + monthly_mean: {"tasanomaly":""} + daily_mean: + calendar: "standard" + reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + HadCRUT4: + name: "HadCRUT4" + institution: "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" + src: "obs/ukmo/hadcrut_v4.6/" + monthly_mean: {"tasanomaly":""} + daily_mean: + calendar: "standard" + reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + HadSLP2: + name: "HadSLP2" + institution: + src: "obs/ukmo/hadslp_v2/" + monthly_mean: {"psl":""} + daily_mean: + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl/psl_200811.nc" + + +# change notes ------------------------------------------------------------------------------ + +# UERRA from archive_subseasonal.yml same as archive.yml, + # UERRA from archive_subseasonal.yml removed + +# CERRA from archive_subseaonal.yml same as archive.yml 2nd CERRA, + # CERRA from archive_subseaonal.yml removed + + +# CERRA-Land from archive_subseaonal.yml same as archive.yml, + # CERRA-Land from archive_subseaonal.yml removed + +# HadCRUT5 from archive_subseaonal.yml same as archive.yml, + # HadCRUT5 from archive_subseaonal.yml removed + +# BEST from archive_subseaonal.yml same as archive.yml, + # BEST from archive_subseaonal.yml removed + +# ERA5-Land from archive_subseaonal.yml same as archive.yml, + # ERA5-Land from archive_subseaonal.yml removed + +# ERA5 differenct lines retained, else deleted. -- GitLab From f0ec38ee7f37285da94ee7866bb6024e1849d663 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Wed, 21 Aug 2024 16:11:07 +0200 Subject: [PATCH 02/11] removed reference from conf/archive.yml,archive_subseasonal.yml,archive_decadal.yml --- conf/archive.yml | 132 ----------------------------------- conf/archive_decadal.yml | 94 ------------------------- conf/archive_reference.yml | 55 +++++++++++---- conf/archive_subseasonal.yml | 98 +------------------------- 4 files changed, 43 insertions(+), 336 deletions(-) diff --git a/conf/archive.yml b/conf/archive.yml index e49e9b01..e763c50d 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -136,102 +136,6 @@ esarchive: calendar: "gregorian" time_stamp_lag: "0" reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", - "rsds":"daily_mean/rsds_f1h-r1440x721cds/", - "prlr":"daily_mean/prlr_f1h-r1440x721cds/", - "g300":"daily_mean/g300_f1h-r1440x721cds/", - "g500":"daily_mean/g500_f1h-r1440x721cds/", - "g850":"daily_mean/g850_f1h-r1440x721cds/", - "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax_f1h-r1440x721cds/", - "tasmin":"daily/tasmin_f1h-r1440x721cds/", - "ta300":"daily_mean/ta300_f1h-r1440x721cds/", - "ta500":"daily_mean/ta500_f1h-r1440x721cds/", - "ta850":"daily_mean/ta850_f1h-r1440x721cds/", - "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", - "psl":"monthly_mean/psl_f1h-r1440x721cds/", - "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", - "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", - "g300":"monthly_mean/g300_f1h-r1440x721cds/", - "g500":"monthly_mean/g500_f1h-r1440x721cds/", - "g850":"monthly_mean/g850_f1h-r1440x721cds/", - "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", - "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", - "ta300":"montly_mean/ta300_f1h-r1440x721cds/", - "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", - "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", - "tos":"monthly_mean/tos_f1h-r1440x721cds/", - "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" - ERA5-Land: - name: "ERA5-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", - "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", - "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", - "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", - "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", - "tdps":"monthly_mean/tdps_f1h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - UERRA: - name: "ECMWF UERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"daily_mean/tas_f6h/"} - monthly_mean: {"tas":"monthly_mean/tas_f6h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" - CERRA: - name: "ECMWF CERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerra/" - daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", - "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", - "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", - "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", - "tas":"monthly_mean/tas_f3h-r2631x1113/", - "winddir":"monthly_mean/winddir_f3h-r2631x1113/", - "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", - "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" - CERRA-Land: - name: "ECMWF CERRA-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerraland/" - daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} - monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" - HadCRUT5: - name: "HadCRUT5" - institution: "Met Office" - src: "obs/ukmo/hadcrut_v5.0_analysis/" - monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" - BEST: - name: "BEST" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "obs/berkeleyearth/berkeleyearth/" - daily_mean: {"tas":"daily_mean/tas/"} - monthly_mean: {"tas":"monthly_mean/tas/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" - mars: src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" System: @@ -246,14 +150,6 @@ mars: calendar: "proleptic_gregorian" time_stamp_lag: "+1" reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "GRIB_era5_tas/" - monthly_mean: {"tas":""} - calendar: "standard" - reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" sample: src: @@ -269,14 +165,6 @@ sample: calendar: "proleptic_gregorian" time_stamp_lag: "0" reference_grid: "conf/grid_description/griddes_GRIB_system51_m1.txt" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "GRIB_era5_tas/" - monthly_mean: {"tas":"", "prlr":""} - calendar: "standard" - reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" IFCA: src: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" @@ -298,23 +186,3 @@ IFCA: calendar: "proleptic_gregorian" time_stamp_lag: "0" reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ECMWF-SEAS5/monthly_mean/tas_f6h/tas_20180501.nc" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "ERA5/" - daily_mean: {"psl":"psl_f1h-r1440x721cds/"} - calendar: "standard" - reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ERA5/daily_mean/psl_f1h-r1440x721cds/psl_201805.nc" - CERRA: - name: "ECMWF CERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "CERRA/" - daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "prlr":"daily_mean/prlr_f6h-r2631x1113/", - "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", "tas":"daily_mean/tas_f3h-r2631x1113/", - "tasmax":"daily_mean/tasmax-r2631x1113/", "tasmin":"daily_mean/tasmin-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "prlr":"monthly_mean/prlr_f6h-r2631x1113/", - "sfcWind":"monthly_mean/_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", - "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/","tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/CERRA/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index a2f340e2..160e3997 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -374,97 +374,3 @@ esarchive: reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" -# ===================================== - - Reference: - GHCNv4: - name: - institution: - src: "obs/noaa/ghcn_v4/" - monthly_mean: {"tas":"", "tasanomaly":""} - daily_mean: - calendar: "gregorian" - reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" -# ---- - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", - "rsds":"daily_mean/rsds_f1h-r1440x721cds/", - "prlr":"daily_mean/prlr_f1h-r1440x721cds/", - "g300":"daily_mean/g300_f1h-r1440x721cds/", - "g500":"daily_mean/g500_f1h-r1440x721cds/", - "g850":"daily_mean/g850_f1h-r1440x721cds/", - "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax-r1440x721cds/", - "tasmin":"daily/tasmin-r1440x721cds/", - "ta300":"daily_mean/ta300_f1h-r1440x721cds/", - "ta500":"daily_mean/ta500_f1h-r1440x721cds/", - "ta850":"daily_mean/ta850_f1h-r1440x721cds/", - "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", - "psl":"monthly_mean/psl_f1h-r1440x721cds/", - "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", - "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", - "g300":"monthly_mean/g300_f1h-r1440x721cds/", - "g500":"monthly_mean/g500_f1h-r1440x721cds/", - "g850":"monthly_mean/g850_f1h-r1440x721cds/", - "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", - "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", - "ta300":"montly_mean/ta300_f1h-r1440x721cds/", - "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", - "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", - "tos":"monthly_mean/tos_f1h-r1440x721cds/", - "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - -# ---- -# missing info in script - NCEP1: - -# ---- - JRA-55: - name: "JRA-55" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/jma/jra55/" - monthly_mean: {"tas":"monthly_mean/tas_f6h", "psl":"monthly_mean/psl_f6h", - "tos":"", "pr":"monthly_mean/pr_s0-3h", - "prlr":"monthly_mean/prlr_s0-3h"} - daily_mean: {"tas":"daily_mean/tas_f6h", "psl":"daily_mean/psl_f6h", - "prlr":"daily_mean/prlr_s0-3h", "sfcWind":"daily_mean/sfcWind_f6h"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" - -# ---- - GISTEMPv4: - name: "GISTEMPv4" - institution: "NASA Goddard Institute for Space Studies" - src: "obs/noaa-nasa/ghcnersstgiss/" - monthly_mean: {"tasanomaly":""} - daily_mean: - calendar: "standard" - reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly/tasanomaly_200811.nc" - -# ---- - HadCRUT4: - name: "HadCRUT4" - institution: "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" - src: "obs/ukmo/hadcrut_v4.6/" - monthly_mean: {"tasanomaly":""} - daily_mean: - calendar: "standard" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" - -# ---- - HadSLP2: - name: "HadSLP2" - institution: - src: "obs/ukmo/hadslp_v2/" - monthly_mean: {"psl":""} - daily_mean: - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl/psl_200811.nc" - diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml index 07772ae8..b2cde9d5 100644 --- a/conf/archive_reference.yml +++ b/conf/archive_reference.yml @@ -1,5 +1,5 @@ -# ----- Reference from archiv.yml - +esarchive: + src: "/esarchive/" Reference: ERA5: name: "ERA5" @@ -14,8 +14,12 @@ "g500":"daily_mean/g500_f1h-r1440x721cds/", "g850":"daily_mean/g850_f1h-r1440x721cds/", "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax_f1h-r1440x721cds/","daily/tasmax-r1440x721cds/", # second entry from decadal - "tasmin":"daily/tasmin_f1h-r1440x721cds/","daily/tasmin-r1440x721cds/", # second entry from decadal + "tasmax": + - "daily/tasmax_f1h-r1440x721cds/", + - "daily/tasmax-r1440x721cds/", # entry from decadal + "tasmin": + - "daily/tasmin_f1h-r1440x721cds/", + - "daily/tasmin-r1440x721cds/", # entry from decadal "ta300":"daily_mean/ta300_f1h-r1440x721cds/", "ta500":"daily_mean/ta500_f1h-r1440x721cds/", "ta850":"daily_mean/ta850_f1h-r1440x721cds/", @@ -69,17 +73,14 @@ src: "recon/ecmwf/cerra/" daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", - "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/", - "prlr":"daily_mean/prlr_f6h-r2631x1113/", # from second CERRA in archive - "tasmax":"daily_mean/tasmax-r2631x1113/", "tasmin":"daily_mean/tasmin-r2631x1113/"} # from second CERRA in archive - + "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/" + } monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", "winddir":"monthly_mean/winddir_f3h-r2631x1113/", "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", - "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/", - "prlr":"monthly_mean/prlr_f6h-r2631x1113/" # from second CERRA in archive + "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/" } calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" @@ -92,6 +93,7 @@ monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" + HadCRUT5: name: "HadCRUT5" institution: "Met Office" @@ -99,6 +101,7 @@ monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" + BEST: name: "BEST" institution: "European Centre for Medium-Range Weather Forecasts" @@ -111,8 +114,19 @@ # ------------------------- mars: src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" + Reference: + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "GRIB_era5_tas/" + monthly_mean: {"tas":""} + calendar: "standard" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" +sample: + src: + Reference: ERA5: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" @@ -121,13 +135,28 @@ mars: calendar: "standard" reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" + +IFCA: + src: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" + Reference: ERA5: - src: "ERA5/" # src different from line 113 + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "ERA5/" daily_mean: {"psl":"psl_f1h-r1440x721cds/"} + calendar: "standard" reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ERA5/daily_mean/psl_f1h-r1440x721cds/psl_201805.nc" - CERRA: - src: "CERRA/" + name: "ECMWF CERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "CERRA/" + daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "prlr":"daily_mean/prlr_f6h-r2631x1113/", + "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", "tas":"daily_mean/tas_f3h-r2631x1113/", + "tasmax":"daily_mean/tasmax-r2631x1113/", "tasmin":"daily_mean/tasmin-r2631x1113/"} + monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "prlr":"monthly_mean/prlr_f6h-r2631x1113/", + "sfcWind":"monthly_mean/_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", + "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/","tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} + calendar: "proleptic_gregorian" reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/CERRA/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" diff --git a/conf/archive_subseasonal.yml b/conf/archive_subseasonal.yml index c142851d..e2d6a3db 100644 --- a/conf/archive_subseasonal.yml +++ b/conf/archive_subseasonal.yml @@ -18,100 +18,4 @@ esarchive: calendar: "proleptic_gregorian" time_stamp_lag: "0" # Do we need it for subseasonal? reference_grid: "/esarchive/exp/ncep/cfs-v2/weekly_mean/s2s/tas_f24h/tas_20040624.nc" # is it the same as seasonal? - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - weekly_mean: {"tas":"weekly_mean/tas_f1h-r1440x721cds/", - "prlr":"weekly_mean/prlr_f1h-r1440x721cds/"} - daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", - "rsds":"daily_mean/rsds_f1h-r1440x721cds/", - "prlr":"daily_mean/prlr_f1h-r1440x721cds/", - "g300":"daily_mean/g300_f1h-r1440x721cds/", - "g500":"daily_mean/g500_f1h-r1440x721cds/", - "g850":"daily_mean/g850_f1h-r1440x721cds/", - "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax_f1h-r1440x721cds/", - "tasmin":"daily/tasmin_f1h-r1440x721cds/", - "ta300":"daily_mean/ta300_f1h-r1440x721cds/", - "ta500":"daily_mean/ta500_f1h-r1440x721cds/", - "ta850":"daily_mean/ta850_f1h-r1440x721cds/", - "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", - "psl":"monthly_mean/psl_f1h-r1440x721cds/", - "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", - "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", - "g300":"monthly_mean/g300_f1h-r1440x721cds/", - "g500":"monthly_mean/g500_f1h-r1440x721cds/", - "g850":"monthly_mean/g850_f1h-r1440x721cds/", - "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", - "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", - "ta300":"montly_mean/ta300_f1h-r1440x721cds/", - "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", - "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", - "tos":"monthly_mean/tos_f1h-r1440x721cds/", - "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" - ERA5-Land: - name: "ERA5-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", - "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", - "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", - "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", - "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", - "tdps":"monthly_mean/tdps_f1h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - UERRA: - name: "ECMWF UERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"daily_mean/tas_f6h/"} - monthly_mean: {"tas":"monthly_mean/tas_f6h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" - CERRA: - name: "ECMWF CERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerra/" - daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", - "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", - "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", - "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", - "tas":"monthly_mean/tas_f3h-r2631x1113/", - "winddir":"monthly_mean/winddir_f3h-r2631x1113/", - "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", - "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" - CERRA-Land: - name: "ECMWF CERRA-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerraland/" - daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} - monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" - HadCRUT5: - name: "HadCRUT5" - institution: "Met Office" - src: "obs/ukmo/hadcrut_v5.0_analysis/" - monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" - BEST: - name: "BEST" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "obs/berkeleyearth/berkeleyearth/" - daily_mean: {"tas":"daily_mean/tas/"} - monthly_mean: {"tas":"monthly_mean/tas/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" + -- GitLab From 7836713a628fe1d0f71b8a0eb9ae4b16843ddc12 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Wed, 21 Aug 2024 16:13:48 +0200 Subject: [PATCH 03/11] renamed archive.yml to archive_seasonal.yml --- conf/{archive.yml => archive_seasonal.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename conf/{archive.yml => archive_seasonal.yml} (100%) diff --git a/conf/archive.yml b/conf/archive_seasonal.yml similarity index 100% rename from conf/archive.yml rename to conf/archive_seasonal.yml -- GitLab From 0d31ee651e0fcd547d68ee343adce7b0c5a9604b Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Thu, 22 Aug 2024 12:07:01 +0200 Subject: [PATCH 04/11] updated archive_reference.yml to follow filesystem --- conf/archive_reference.yml | 154 +++++++++++++++---------------------- 1 file changed, 60 insertions(+), 94 deletions(-) diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml index b2cde9d5..77d57ad3 100644 --- a/conf/archive_reference.yml +++ b/conf/archive_reference.yml @@ -39,11 +39,9 @@ esarchive: "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", "tos":"monthly_mean/tos_f1h-r1440x721cds/", "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" # calendar in archive_decadal ERA5 is gregorian reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" - ERA5-Land: name: "ERA5-Land" institution: "European Centre for Medium-Range Weather Forecasts" @@ -57,7 +55,6 @@ esarchive: "tdps":"monthly_mean/tdps_f1h/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - UERRA: name: "ECMWF UERRA" institution: "European Centre for Medium-Range Weather Forecasts" @@ -66,25 +63,21 @@ esarchive: monthly_mean: {"tas":"monthly_mean/tas_f6h/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" - CERRA: name: "ECMWF CERRA" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/cerra/" daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", - "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/" - } + "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", "winddir":"monthly_mean/winddir_f3h-r2631x1113/", "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", - "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/" - } + "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" - CERRA-Land: name: "ECMWF CERRA-Land" institution: "European Centre for Medium-Range Weather Forecasts" @@ -93,7 +86,6 @@ esarchive: monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" - HadCRUT5: name: "HadCRUT5" institution: "Met Office" @@ -101,7 +93,6 @@ esarchive: monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" - BEST: name: "BEST" institution: "European Centre for Medium-Range Weather Forecasts" @@ -110,6 +101,64 @@ esarchive: monthly_mean: {"tas":"monthly_mean/tas/"} calendar: "proleptic_gregorian" reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" +# +# ----- Reference from archive_decade.yml + + GHCNv4: + name: + institution: + src: "obs/noaa/ghcn_v4/" + monthly_mean: {"tas":"", "tasanomaly":""} + daily_mean: + calendar: "gregorian" + reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" + +# ---- +# missing info in script + NCEP1: + +# ---- + JRA-55: + name: "JRA-55" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/jma/jra55/" + monthly_mean: {"tas":"monthly_mean/tas_f6h", "psl":"monthly_mean/psl_f6h", + "tos":"", "pr":"monthly_mean/pr_s0-3h", + "prlr":"monthly_mean/prlr_s0-3h"} + daily_mean: {"tas":"daily_mean/tas_f6h", "psl":"daily_mean/psl_f6h", + "prlr":"daily_mean/prlr_s0-3h", "sfcWind":"daily_mean/sfcWind_f6h"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" + +# ---- + GISTEMPv4: + name: "GISTEMPv4" + institution: "NASA Goddard Institute for Space Studies" + src: "obs/noaa-nasa/ghcnersstgiss/" + monthly_mean: {"tasanomaly":""} + daily_mean: + calendar: "standard" + reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + HadCRUT4: + name: "HadCRUT4" + institution: "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" + src: "obs/ukmo/hadcrut_v4.6/" + monthly_mean: {"tasanomaly":""} + daily_mean: + calendar: "standard" + reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + HadSLP2: + name: "HadSLP2" + institution: + src: "obs/ukmo/hadslp_v2/" + monthly_mean: {"psl":""} + daily_mean: + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl/psl_200811.nc" # ------------------------- mars: @@ -160,86 +209,3 @@ IFCA: reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/CERRA/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" - - -# ----- Reference from archive_decade.yml - - GHCNv4: - name: - institution: - src: "obs/noaa/ghcn_v4/" - monthly_mean: {"tas":"", "tasanomaly":""} - daily_mean: - calendar: "gregorian" - reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" - -# ---- -# missing info in script - NCEP1: - -# ---- - JRA-55: - name: "JRA-55" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/jma/jra55/" - monthly_mean: {"tas":"monthly_mean/tas_f6h", "psl":"monthly_mean/psl_f6h", - "tos":"", "pr":"monthly_mean/pr_s0-3h", - "prlr":"monthly_mean/prlr_s0-3h"} - daily_mean: {"tas":"daily_mean/tas_f6h", "psl":"daily_mean/psl_f6h", - "prlr":"daily_mean/prlr_s0-3h", "sfcWind":"daily_mean/sfcWind_f6h"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" - -# ---- - GISTEMPv4: - name: "GISTEMPv4" - institution: "NASA Goddard Institute for Space Studies" - src: "obs/noaa-nasa/ghcnersstgiss/" - monthly_mean: {"tasanomaly":""} - daily_mean: - calendar: "standard" - reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly/tasanomaly_200811.nc" - -# ---- - HadCRUT4: - name: "HadCRUT4" - institution: "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" - src: "obs/ukmo/hadcrut_v4.6/" - monthly_mean: {"tasanomaly":""} - daily_mean: - calendar: "standard" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" - -# ---- - HadSLP2: - name: "HadSLP2" - institution: - src: "obs/ukmo/hadslp_v2/" - monthly_mean: {"psl":""} - daily_mean: - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl/psl_200811.nc" - - -# change notes ------------------------------------------------------------------------------ - -# UERRA from archive_subseasonal.yml same as archive.yml, - # UERRA from archive_subseasonal.yml removed - -# CERRA from archive_subseaonal.yml same as archive.yml 2nd CERRA, - # CERRA from archive_subseaonal.yml removed - - -# CERRA-Land from archive_subseaonal.yml same as archive.yml, - # CERRA-Land from archive_subseaonal.yml removed - -# HadCRUT5 from archive_subseaonal.yml same as archive.yml, - # HadCRUT5 from archive_subseaonal.yml removed - -# BEST from archive_subseaonal.yml same as archive.yml, - # BEST from archive_subseaonal.yml removed - -# ERA5-Land from archive_subseaonal.yml same as archive.yml, - # ERA5-Land from archive_subseaonal.yml removed - -# ERA5 differenct lines retained, else deleted. -- GitLab From 611bb5c616a57fcc9c05c742eb7ec883100eee09 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Thu, 22 Aug 2024 12:10:52 +0200 Subject: [PATCH 05/11] updated archive_referenc.yml, as per meeting on 220824:11:00 --- conf/archive_reference.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml index 77d57ad3..88989ac5 100644 --- a/conf/archive_reference.yml +++ b/conf/archive_reference.yml @@ -5,8 +5,8 @@ esarchive: name: "ERA5" institution: "European Centre for Medium-Range Weather Forecasts" src: "recon/ecmwf/era5/" - weekly_mean: {"tas":"weekly_mean/tas_f1h-r1440x721cds/", # from subseasonal - "prlr":"weekly_mean/prlr_f1h-r1440x721cds/"} # from subseasonal + weekly_mean: {"tas":"weekly_mean/tas_f1h-r1440x721cds/", + "prlr":"weekly_mean/prlr_f1h-r1440x721cds/"} daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", "rsds":"daily_mean/rsds_f1h-r1440x721cds/", "prlr":"daily_mean/prlr_f1h-r1440x721cds/", @@ -14,12 +14,8 @@ esarchive: "g500":"daily_mean/g500_f1h-r1440x721cds/", "g850":"daily_mean/g850_f1h-r1440x721cds/", "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax": - - "daily/tasmax_f1h-r1440x721cds/", - - "daily/tasmax-r1440x721cds/", # entry from decadal - "tasmin": - - "daily/tasmin_f1h-r1440x721cds/", - - "daily/tasmin-r1440x721cds/", # entry from decadal + "tasmax":"daily/tasmax-r1440x721cds/", + "tasmin":"daily/tasmin-r1440x721cds/", "ta300":"daily_mean/ta300_f1h-r1440x721cds/", "ta500":"daily_mean/ta500_f1h-r1440x721cds/", "ta850":"daily_mean/ta850_f1h-r1440x721cds/", @@ -39,7 +35,7 @@ esarchive: "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", "tos":"monthly_mean/tos_f1h-r1440x721cds/", "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" # calendar in archive_decadal ERA5 is gregorian + calendar: "standard" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" ERA5-Land: @@ -102,7 +98,6 @@ esarchive: calendar: "proleptic_gregorian" reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" # -# ----- Reference from archive_decade.yml GHCNv4: name: -- GitLab From 6f33523b5cccca493ea880e8874d15afcc88cf54 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Thu, 22 Aug 2024 15:04:19 +0200 Subject: [PATCH 06/11] updated archive_reference.yml folllwing error from read_yaml --- conf/archive_reference.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml index 88989ac5..b456a8c5 100644 --- a/conf/archive_reference.yml +++ b/conf/archive_reference.yml @@ -35,7 +35,7 @@ esarchive: "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", "tos":"monthly_mean/tos_f1h-r1440x721cds/", "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" + calendar: "standard" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" ERA5-Land: @@ -66,7 +66,7 @@ esarchive: daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", + monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", "winddir":"monthly_mean/winddir_f3h-r2631x1113/", -- GitLab From 4625d582c40ad9edbf2ebc8d68b6ba3dbf98b990 Mon Sep 17 00:00:00 2001 From: vagudets Date: Thu, 22 Aug 2024 15:40:24 +0200 Subject: [PATCH 07/11] Replace repetitive lines with 'get_archive()' calls, adapt seasonal and decadal to new archive structure --- conf/archive_reference.yml | 16 ++++++++-------- modules/Downscaling/Downscaling.R | 2 +- modules/Loading/R/load_decadal.R | 2 +- modules/Loading/R/load_seasonal.R | 2 +- modules/Loading/R/load_tas_tos.R | 2 +- modules/Saving/R/save_forecast.R | 4 ++-- tests/testthat/test-seasonal_NAO.R | 1 - tests/testthat/test-seasonal_monthly.R | 1 - .../testthat/test-seasonal_monthly_statistics.R | 1 - tests/testthat/test-seasonal_monthly_tas-tos.R | 1 - tests/testthat/test-seasonal_monthly_timeagg.R | 1 - tools/check_recipe.R | 9 ++------- tools/get_archive.R | 4 +++- 13 files changed, 19 insertions(+), 27 deletions(-) diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml index 88989ac5..d67dc195 100644 --- a/conf/archive_reference.yml +++ b/conf/archive_reference.yml @@ -34,8 +34,8 @@ esarchive: "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", "tos":"monthly_mean/tos_f1h-r1440x721cds/", - "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" + "sic":"monthly_mean/sic_f1h-r1440x721cds/"} + calendar: "gregorian" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" ERA5-Land: @@ -66,7 +66,7 @@ esarchive: daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", + monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", "winddir":"monthly_mean/winddir_f3h-r2631x1113/", @@ -132,7 +132,7 @@ esarchive: src: "obs/noaa-nasa/ghcnersstgiss/" monthly_mean: {"tasanomaly":""} daily_mean: - calendar: "standard" + calendar: "gregorian" reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly/tasanomaly_200811.nc" # ---- @@ -142,7 +142,7 @@ esarchive: src: "obs/ukmo/hadcrut_v4.6/" monthly_mean: {"tasanomaly":""} daily_mean: - calendar: "standard" + calendar: "gregorian" reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" # ---- @@ -164,7 +164,7 @@ mars: institution: "European Centre for Medium-Range Weather Forecasts" src: "GRIB_era5_tas/" monthly_mean: {"tas":""} - calendar: "standard" + calendar: "gregorian" reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" @@ -176,7 +176,7 @@ sample: institution: "European Centre for Medium-Range Weather Forecasts" src: "GRIB_era5_tas/" monthly_mean: {"tas":"", "prlr":""} - calendar: "standard" + calendar: "gregorian" reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" @@ -188,7 +188,7 @@ IFCA: institution: "European Centre for Medium-Range Weather Forecasts" src: "ERA5/" daily_mean: {"psl":"psl_f1h-r1440x721cds/"} - calendar: "standard" + calendar: "gregorian" reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ERA5/daily_mean/psl_f1h-r1440x721cds/psl_201805.nc" CERRA: name: "ECMWF CERRA" diff --git a/modules/Downscaling/Downscaling.R b/modules/Downscaling/Downscaling.R index e95dfadd..07c22838 100644 --- a/modules/Downscaling/Downscaling.R +++ b/modules/Downscaling/Downscaling.R @@ -38,7 +38,7 @@ Downscaling <- function(recipe, data) { probs_cat <- sapply(unlist(recipe$Analysis$Workflow$Downscaling$probs_cat), function(frac) eval(parse(text = frac))) if (tolower(recipe$Analysis$Horizon) == "seasonal") { - archive <- read_yaml("conf/archive.yml") + archive <- read_yaml("conf/archive_seasonal.yml") } else if (tolower(recipe$Analysis$Horizon) == "subseasonal") { archive <- read_yaml("conf/archive_subseasonal.yml") } else if (tolower(recipe$Analysis$Horizon) == "decadal") { diff --git a/modules/Loading/R/load_decadal.R b/modules/Loading/R/load_decadal.R index bfd28979..3ea35008 100644 --- a/modules/Loading/R/load_decadal.R +++ b/modules/Loading/R/load_decadal.R @@ -13,7 +13,7 @@ source("modules/Loading/R/compare_exp_obs_grids.R") load_decadal <- function(recipe) { ## - archive <- read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]] + archive <- get_archive(recipe) # Print Start() info or not DEBUG <- TRUE diff --git a/modules/Loading/R/load_seasonal.R b/modules/Loading/R/load_seasonal.R index 4b8fa6ac..1c982538 100644 --- a/modules/Loading/R/load_seasonal.R +++ b/modules/Loading/R/load_seasonal.R @@ -47,7 +47,7 @@ load_seasonal <- function(recipe) { ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name # get datasets dict: - archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + archive <- get_archive(recipe) exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable[1]]) diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R index 18c000ad..328dddf0 100644 --- a/modules/Loading/R/load_tas_tos.R +++ b/modules/Loading/R/load_tas_tos.R @@ -50,7 +50,7 @@ load_tas_tos <- function(recipe) { } # get esarchive datasets dict: - archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + archive <- get_archive(recipe) exp_descrip <- archive$System[[exp.name]] reference_descrip <- archive$Reference[[ref.name]] diff --git a/modules/Saving/R/save_forecast.R b/modules/Saving/R/save_forecast.R index 6e3bc906..b2d74fc9 100644 --- a/modules/Saving/R/save_forecast.R +++ b/modules/Saving/R/save_forecast.R @@ -16,8 +16,8 @@ save_forecast <- function(recipe, global_attributes <- .get_global_attributes(recipe, archive) fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq - if (global_attributes$system == 'Multimodel'){ - if (fcst.horizon == 'decadal'){ + if (global_attributes$system == 'Multimodel') { + if (fcst.horizon == 'decadal') { calendar <- archive$System[[recipe$Analysis$Datasets$System$models[[1]]$name]]$calendar } else { calendar <- archive$System[[recipe$Analysis$Datasets$System$models[[1]]]]$calendar diff --git a/tests/testthat/test-seasonal_NAO.R b/tests/testthat/test-seasonal_NAO.R index f70eefc7..12c406eb 100644 --- a/tests/testthat/test-seasonal_NAO.R +++ b/tests/testthat/test-seasonal_NAO.R @@ -8,7 +8,6 @@ source("modules/Indices/Indices.R") recipe_file <- "tests/recipes/recipe-seasonal_NAO.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly.R b/tests/testthat/test-seasonal_monthly.R index 18bde4c0..198c947a 100644 --- a/tests/testthat/test-seasonal_monthly.R +++ b/tests/testthat/test-seasonal_monthly.R @@ -8,7 +8,6 @@ source("modules/Visualization/Visualization.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly_statistics.R b/tests/testthat/test-seasonal_monthly_statistics.R index cae76111..74647a21 100644 --- a/tests/testthat/test-seasonal_monthly_statistics.R +++ b/tests/testthat/test-seasonal_monthly_statistics.R @@ -7,7 +7,6 @@ source("modules/Visualization/Visualization.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_statistics.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly_tas-tos.R b/tests/testthat/test-seasonal_monthly_tas-tos.R index 1345f63d..4078d6e7 100644 --- a/tests/testthat/test-seasonal_monthly_tas-tos.R +++ b/tests/testthat/test-seasonal_monthly_tas-tos.R @@ -4,7 +4,6 @@ source("./modules/Loading/Loading.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly_timeagg.R b/tests/testthat/test-seasonal_monthly_timeagg.R index 21977d9c..ffa389c4 100644 --- a/tests/testthat/test-seasonal_monthly_timeagg.R +++ b/tests/testthat/test-seasonal_monthly_timeagg.R @@ -9,7 +9,6 @@ source("modules/Aggregation/Aggregation.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_timeagg.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 01134476..c9641787 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -17,9 +17,6 @@ check_recipe <- function(recipe) { PARAMS <- c("Horizon", "Time", "Variables", "Region", "Regrid", "Workflow", "Datasets") HORIZONS <- c("subseasonal", "seasonal", "decadal") - ARCHIVE_SEASONAL <- "conf/archive.yml" - ARCHIVE_DECADAL <- "conf/archive_decadal.yml" - ARCHIVE_SUBSEASONAL <- "conf/archive_subseasonal.yml" # Define error status variable error_status <- F @@ -45,9 +42,8 @@ check_recipe <- function(recipe) { error_status <- T } # Check time settings + archive <- get_archive(recipe) if (recipe$Analysis$Horizon == "seasonal") { - ## TODO: Specify filesystem - archive <- read_yaml(ARCHIVE_SEASONAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_SEASONAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -56,7 +52,6 @@ check_recipe <- function(recipe) { error_status <- TRUE } } else if (recipe$Analysis$Horizon == "decadal") { - archive <- read_yaml(ARCHIVE_DECADAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_DECADAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -65,7 +60,6 @@ check_recipe <- function(recipe) { error_status <- TRUE } } else if (recipe$Analysis$Horizon == "subseasonal") { - archive <- read_yaml(ARCHIVE_SUBSEASONAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_SUBSEASONAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -76,6 +70,7 @@ check_recipe <- function(recipe) { } else { archive <- NULL } + # Check variable parameters if ("name" %in% names(recipe$Analysis$Variables)) { recipe_variables <- recipe$Analysis$Variables$name diff --git a/tools/get_archive.R b/tools/get_archive.R index 56309ccb..8a1ea9b1 100644 --- a/tools/get_archive.R +++ b/tools/get_archive.R @@ -1,7 +1,7 @@ get_archive <- function(recipe) { if (tolower(recipe$Analysis$Horizon) == "seasonal") { archive <- - read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]] + read_yaml(paste0("conf/archive_seasonal.yml"))[[recipe$Run$filesystem]] } else if (tolower(recipe$Analysis$Horizon) == "subseasonal") { archive <- read_yaml(paste0("conf/archive_subseasonal.yml"))[[recipe$Run$filesystem]] @@ -9,6 +9,8 @@ get_archive <- function(recipe) { archive <- read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]] } + archive_ref <- read_yaml(paste0("conf/archive_reference.yml"))[[recipe$Run$filesystem]] + archive$Reference <- archive_ref$Reference ## TODO: Add dictionary filesystem dependency? # dict <- read_yaml("conf/variable-dictionary.yml") return(archive) -- GitLab From 40e1e2063c260abd98747845c9319c2c40d9cd70 Mon Sep 17 00:00:00 2001 From: THEERTHA KARIYATHAN Date: Thu, 22 Aug 2024 16:35:39 +0200 Subject: [PATCH 08/11] modified load_subseaonal.R with get_archive() --- modules/Loading/R/load_subseasonal.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/Loading/R/load_subseasonal.R b/modules/Loading/R/load_subseasonal.R index 793b46ec..4cac3d2d 100644 --- a/modules/Loading/R/load_subseasonal.R +++ b/modules/Loading/R/load_subseasonal.R @@ -33,7 +33,7 @@ load_subseasonal <- function(recipe) { ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name # get datasets dict: - archive <- read_yaml("conf/archive_subseasonal.yml")[[recipe$Run$filesystem]] + archive <-get_archive(recipe) # read_yaml("conf/archive_subseasonal.yml")[[recipe$Run$filesystem]] exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable[1]]) -- GitLab From 89bcf74729188aa4b7152fcd1d23f769f9a4e352 Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 23 Aug 2024 10:00:35 +0200 Subject: [PATCH 09/11] Replace read_yaml() with get_archive() in Scorecards and other functions' --- modules/Loading/R/load_GRIB.R | 4 ++-- modules/Multimodel/dims_multimodel.R | 2 +- modules/Multimodel/load_multimodel.R | 2 +- modules/Scorecards/R/tmp/ScorecardsMulti.R | 2 +- modules/Scorecards/R/tmp/ScorecardsSingle.R | 2 +- modules/Scorecards/R/tmp/ScorecardsSystemDiff.R | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/Loading/R/load_GRIB.R b/modules/Loading/R/load_GRIB.R index 0dd5f919..d8116d08 100644 --- a/modules/Loading/R/load_GRIB.R +++ b/modules/Loading/R/load_GRIB.R @@ -22,14 +22,14 @@ load_GRIB <- function(recipe) { lons.max <- recipe$Analysis$Region$lonmax # can only be 360 ref.name <- recipe$Analysis$Datasets$Reference$name exp.name <- recipe$Analysis$Datasets$System$name - variable <- recipe$Analysis$Variables$name #'tas' + variable <- recipe$Analysis$Variables$name store.freq <- recipe$Analysis$Variables$freq regrid.method <- recipe$Analysis$Regrid$method regrid.type <- recipe$Analysis$Regrid$type # get MARS datasets dict: - archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + archive <- get_archive(recipe) exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable]) reference_descrip <- archive$Reference[[ref.name]] diff --git a/modules/Multimodel/dims_multimodel.R b/modules/Multimodel/dims_multimodel.R index b132f1b4..9126350f 100644 --- a/modules/Multimodel/dims_multimodel.R +++ b/modules/Multimodel/dims_multimodel.R @@ -5,7 +5,7 @@ source("modules/Loading/R/check_latlon.R") dims_multimodel <- function(recipe) { - archive <- read_yaml("conf/archive.yml")$esarchive + archive <- get_archive() ref.name <- recipe$Analysis$Datasets$Reference$name if (tolower(recipe$Analysis$Horizon) == 'seasonal') { exp.name <- recipe$Analysis$Datasets$System$models diff --git a/modules/Multimodel/load_multimodel.R b/modules/Multimodel/load_multimodel.R index 529d2e5d..778aae6e 100644 --- a/modules/Multimodel/load_multimodel.R +++ b/modules/Multimodel/load_multimodel.R @@ -5,7 +5,7 @@ source("modules/Loading/R/check_latlon.R") load_multimodel <- function(recipe) { - archive <- read_yaml("conf/archive.yml")$esarchive + archive <- get_archive(recipe) ref.name <- recipe$Analysis$Datasets$Reference$name if (tolower(recipe$Analysis$Horizon) == 'seasonal') { exp.name <- recipe$Analysis$Datasets$System$models diff --git a/modules/Scorecards/R/tmp/ScorecardsMulti.R b/modules/Scorecards/R/tmp/ScorecardsMulti.R index 7278eb16..01d561e0 100644 --- a/modules/Scorecards/R/tmp/ScorecardsMulti.R +++ b/modules/Scorecards/R/tmp/ScorecardsMulti.R @@ -131,7 +131,7 @@ ScorecardsMulti <- function(data, sign, system, reference, var, start.year, } else { filesystem <- recipe$Run$filesystem } - sys_dict <- read_yaml("conf/archive.yml")[[filesystem]] + sys_dict <- get_archive(recipe) var_dict <- read_yaml("conf/variable-dictionary.yml")$vars ## Get scorecards table display names from configuration files diff --git a/modules/Scorecards/R/tmp/ScorecardsSingle.R b/modules/Scorecards/R/tmp/ScorecardsSingle.R index 9fd44545..61a6ee70 100644 --- a/modules/Scorecards/R/tmp/ScorecardsSingle.R +++ b/modules/Scorecards/R/tmp/ScorecardsSingle.R @@ -143,7 +143,7 @@ ScorecardsSingle <- function(data, sign, system, reference, var, start.year, } else { filesystem <- recipe$Run$filesystem } - sys_dict <- read_yaml("conf/archive.yml")[[filesystem]] + sys_dict <- get_archive(recipe) var_dict <- read_yaml("conf/variable-dictionary.yml")$vars ## Get scorecards table display names from configuration files diff --git a/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R b/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R index 88e9de15..c785bd5d 100644 --- a/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R +++ b/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R @@ -116,7 +116,7 @@ ScorecardsSystemDiff <- function(data, ftime_dim = 'time') ## Load configuration files - sys_dict <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + sys_dict <- get_archive(recipe) var_dict <- read_yaml("conf/variable-dictionary.yml")$vars ## Get scorecards table display names from configuration files -- GitLab From 117b5ff2f65378b80423f675bea566d9f2604d4f Mon Sep 17 00:00:00 2001 From: vagudets Date: Fri, 23 Aug 2024 11:32:31 +0200 Subject: [PATCH 10/11] Separate 'src' for exp and obs --- conf/archive_decadal.yml | 2 +- conf/archive_reference.yml | 8 ++++---- conf/archive_seasonal.yml | 8 ++++---- conf/archive_subseasonal.yml | 2 +- modules/Loading/R/helper_loading_decadal.R | 4 ++-- modules/Loading/R/load_GRIB.R | 6 +++--- modules/Loading/R/load_decadal.R | 2 +- modules/Loading/R/load_seasonal.R | 6 +++--- modules/Loading/R/load_subseasonal.R | 6 +++--- modules/Loading/R/load_tas_tos.R | 6 +++--- tools/get_archive.R | 1 + 11 files changed, 26 insertions(+), 25 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 160e3997..54b23ef2 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -1,5 +1,5 @@ esarchive: - src: "/esarchive/" + src_sys: "/esarchive/" System: # ---- EC-Earth3-i1: diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml index d67dc195..41073eb9 100644 --- a/conf/archive_reference.yml +++ b/conf/archive_reference.yml @@ -1,5 +1,5 @@ esarchive: - src: "/esarchive/" + src_ref: "/esarchive/" Reference: ERA5: name: "ERA5" @@ -157,7 +157,7 @@ esarchive: # ------------------------- mars: - src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" + src_ref: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" Reference: ERA5: name: "ERA5" @@ -169,7 +169,7 @@ mars: sample: - src: + src_ref: Reference: ERA5: name: "ERA5" @@ -181,7 +181,7 @@ sample: IFCA: - src: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" + src_ref: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" Reference: ERA5: name: "ERA5" diff --git a/conf/archive_seasonal.yml b/conf/archive_seasonal.yml index e763c50d..125f1804 100644 --- a/conf/archive_seasonal.yml +++ b/conf/archive_seasonal.yml @@ -1,5 +1,5 @@ esarchive: - src: "/esarchive/" + src_sys: "/esarchive/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" @@ -137,7 +137,7 @@ esarchive: time_stamp_lag: "0" reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" mars: - src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" + src_sys: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" @@ -152,7 +152,7 @@ mars: reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" sample: - src: + src_sys: System: ECMWF-SEAS5.1: name: "ECMWF SEAS5" @@ -167,7 +167,7 @@ sample: reference_grid: "conf/grid_description/griddes_GRIB_system51_m1.txt" IFCA: - src: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" + src_sys: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" diff --git a/conf/archive_subseasonal.yml b/conf/archive_subseasonal.yml index e2d6a3db..a48fe0fa 100644 --- a/conf/archive_subseasonal.yml +++ b/conf/archive_subseasonal.yml @@ -1,5 +1,5 @@ esarchive: - src: "/esarchive/" + src_sys: "/esarchive/" System: NCEP-CFSv2: name: "NCEP CFSv2" diff --git a/modules/Loading/R/helper_loading_decadal.R b/modules/Loading/R/helper_loading_decadal.R index 9b71c94b..65058a6b 100644 --- a/modules/Loading/R/helper_loading_decadal.R +++ b/modules/Loading/R/helper_loading_decadal.R @@ -111,12 +111,12 @@ get_dcpp_path <- function(archive, exp.name, table, grid, version, sdates) { if (length(table) == 1) { # only one variable, or the variables share a common path pattern - fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$startR, + fcst.path <- file.path(archive$src_sys, archive$System[[exp.name]]$src$startR, '$ensemble$', table, '$var$', grid, 'v*') fcst.files <- paste0('$var$_', table, '_*_$dcpp$_s$syear$-$ensemble$_', grid, '_$chunk$.nc') } else { # path pattern depends on the variable - fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$startR, + fcst.path <- file.path(archive$src_sys, archive$System[[exp.name]]$src$startR, '$ensemble$', '$table$', '$var$', '$grid$', '$version$') fcst.files <- paste0('$var$_', '$table$', '_*_$dcpp$_s$syear$-$ensemble$_', '$grid$', '_$chunk$.nc') } diff --git a/modules/Loading/R/load_GRIB.R b/modules/Loading/R/load_GRIB.R index d8116d08..9252561a 100644 --- a/modules/Loading/R/load_GRIB.R +++ b/modules/Loading/R/load_GRIB.R @@ -54,7 +54,7 @@ load_GRIB <- function(recipe) { # The correct files #exp_path <- "/esarchive/scratch/aho/tmp/GRIB/GRIB_system5_tas_CORRECTED/" - hcst.path <- paste0(archive$src, hcst.dir) + hcst.path <- paste0(archive$src_sys, hcst.dir) hcst.year <- paste0(as.numeric(hcst.inityear):as.numeric(hcst.endyear)) hcst.files <- paste0(hcst.path, variable, '_', hcst.year, hcst.sdate, '.grb') @@ -79,7 +79,7 @@ load_GRIB <- function(recipe) { # Load forecast #------------------------------------------------------------------- if (!is.null(fcst.year)) { - fcst.path <- paste0(archive$src, hcst.dir) + fcst.path <- paste0(archive$src_sys, hcst.dir) fcst.files <- paste0(fcst.path, variable, '_', fcst.year, hcst.sdate, '.grb') fcst <- GrbLoad(dat = as.list(fcst.files), time_step = hcst.ftime, has.memb = fcst.nmember, syear_time_dim = NULL, regrid = regrid_list) @@ -94,7 +94,7 @@ load_GRIB <- function(recipe) { # Load reference #------------------------------------------------------------------- #obs_path <- "/esarchive/scratch/aho/tmp/GRIB/GRIB_era5_tas/" - obs.path <- paste0(archive$src, obs.dir) + obs.path <- paste0(archive$src_ref, obs.dir) # Use hcst time attr to load obs hcst_times <- attr(hcst, 'time') hcst_times_strings <- format(hcst_times, '%Y%m') diff --git a/modules/Loading/R/load_decadal.R b/modules/Loading/R/load_decadal.R index 3ea35008..21df4223 100644 --- a/modules/Loading/R/load_decadal.R +++ b/modules/Loading/R/load_decadal.R @@ -213,7 +213,7 @@ load_decadal <- function(recipe) { #------------------------------------------- # Step 3. Load the reference #------------------------------------------- - obs.path <- file.path(archive$src, archive$Reference[[ref.name]]$src, + obs.path <- file.path(archive$src_ref, archive$Reference[[ref.name]]$src, "$var_dir$", "$var$_$file_date$.nc") var_dir_obs <- archive$Reference[[ref.name]][[store.freq]][variable] # list(tas = "_f1h-r1440x721cds", tos = "_f1h-r1440x721cds") diff --git a/modules/Loading/R/load_seasonal.R b/modules/Loading/R/load_seasonal.R index 1c982538..bbee8d4d 100644 --- a/modules/Loading/R/load_seasonal.R +++ b/modules/Loading/R/load_seasonal.R @@ -75,13 +75,13 @@ load_seasonal <- function(recipe) { var_dir_exp <- exp_descrip[[frequency]][variable] # ----------- - obs.path <- paste0(archive$src, obs.dir, "$var_dir$", + obs.path <- paste0(archive$src_ref, obs.dir, "$var_dir$", "$var$_$file_date$.nc") - hcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + hcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") - fcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + fcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") # Define regrid parameters: diff --git a/modules/Loading/R/load_subseasonal.R b/modules/Loading/R/load_subseasonal.R index 4cac3d2d..9176ad7e 100644 --- a/modules/Loading/R/load_subseasonal.R +++ b/modules/Loading/R/load_subseasonal.R @@ -64,13 +64,13 @@ load_subseasonal <- function(recipe) { var_dir_exp <- exp_descrip[[frequency]][variable] # ----------- - obs.path <- paste0(archive$src, obs.dir, "$var_dir$", + obs.path <- paste0(archive$src_ref, obs.dir, "$var_dir$", "$var$_$file_date$.nc") - hcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + hcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") - fcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + fcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") # Define regrid parameters: diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R index 328dddf0..4710920b 100644 --- a/modules/Loading/R/load_tas_tos.R +++ b/modules/Loading/R/load_tas_tos.R @@ -70,13 +70,13 @@ load_tas_tos <- function(recipe) { var_dir_exp <- exp_descrip[[frequency]][variable] # ----------- - obs.path <- paste0(archive$src, obs.dir, "$var_dir$", + obs.path <- paste0(archive$src_ref, obs.dir, "$var_dir$", "$var$_$file_date$.nc") - hcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + hcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") - fcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + fcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "/$var$_$file_date$.nc") # Define regrid parameters: diff --git a/tools/get_archive.R b/tools/get_archive.R index 8a1ea9b1..edcc057f 100644 --- a/tools/get_archive.R +++ b/tools/get_archive.R @@ -11,6 +11,7 @@ get_archive <- function(recipe) { } archive_ref <- read_yaml(paste0("conf/archive_reference.yml"))[[recipe$Run$filesystem]] archive$Reference <- archive_ref$Reference + archive$src_ref <- archive_ref$src_ref ## TODO: Add dictionary filesystem dependency? # dict <- read_yaml("conf/variable-dictionary.yml") return(archive) -- GitLab From 6330cd439a4ab71f4666344a87ed475a57242180 Mon Sep 17 00:00:00 2001 From: vagudets Date: Mon, 26 Aug 2024 15:18:54 +0200 Subject: [PATCH 11/11] Change default values in use case to make testing easier --- .../ex1_2_autosubmit_scorecards/ex1_2-recipe.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml index 68a8dc62..7201ea16 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml @@ -80,21 +80,21 @@ Run: Loglevel: INFO Terminal: yes filesystem: esarchive - output_dir: /esarchive/scratch/aho/auto-s2s-outputs/ - code_dir: /esarchive/scratch/aho/git/auto-s2s/ + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ + code_dir: /home/Earth/vagudets/git/auto-s2s/ autosubmit: yes # fill only if using autosubmit auto_conf: - script: /esarchive/scratch/aho/git/auto-s2s/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script - expid: a6pc # replace with your EXPID - hpc_user: bsc032734 # replace with your hpc username + script: use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script + expid: a6wq # replace with your EXPID + hpc_user: bsc032762 # replace with your hpc username wallclock: 03:00 # hh:mm - processors_per_job: 8 + processors_per_job: 16 platform: nord3v2 custom_directives: ['#SBATCH --exclusive'] email_notifications: yes # enable/disable email notifications. Change it if you want to. - email_address: an.ho@bsc.es # replace with your email address + email_address: victoria.agudetse@bsc.es # replace with your email address notify_completed: yes # notify me by email when a job finishes - notify_failed: yes # notify me by email when a job fails + notify_failed: no # notify me by email when a job fails -- GitLab