diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index a2f340e296082897e0d1d6827d4be73e8ebe6a56..54b23ef252dbf85eea1d343129038acd6e84d545 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -1,5 +1,5 @@ esarchive: - src: "/esarchive/" + src_sys: "/esarchive/" System: # ---- EC-Earth3-i1: @@ -374,97 +374,3 @@ esarchive: reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" -# ===================================== - - Reference: - GHCNv4: - name: - institution: - src: "obs/noaa/ghcn_v4/" - monthly_mean: {"tas":"", "tasanomaly":""} - daily_mean: - calendar: "gregorian" - reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" -# ---- - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", - "rsds":"daily_mean/rsds_f1h-r1440x721cds/", - "prlr":"daily_mean/prlr_f1h-r1440x721cds/", - "g300":"daily_mean/g300_f1h-r1440x721cds/", - "g500":"daily_mean/g500_f1h-r1440x721cds/", - "g850":"daily_mean/g850_f1h-r1440x721cds/", - "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax-r1440x721cds/", - "tasmin":"daily/tasmin-r1440x721cds/", - "ta300":"daily_mean/ta300_f1h-r1440x721cds/", - "ta500":"daily_mean/ta500_f1h-r1440x721cds/", - "ta850":"daily_mean/ta850_f1h-r1440x721cds/", - "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", - "psl":"monthly_mean/psl_f1h-r1440x721cds/", - "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", - "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", - "g300":"monthly_mean/g300_f1h-r1440x721cds/", - "g500":"monthly_mean/g500_f1h-r1440x721cds/", - "g850":"monthly_mean/g850_f1h-r1440x721cds/", - "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", - "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", - "ta300":"montly_mean/ta300_f1h-r1440x721cds/", - "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", - "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", - "tos":"monthly_mean/tos_f1h-r1440x721cds/", - "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - -# ---- -# missing info in script - NCEP1: - -# ---- - JRA-55: - name: "JRA-55" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/jma/jra55/" - monthly_mean: {"tas":"monthly_mean/tas_f6h", "psl":"monthly_mean/psl_f6h", - "tos":"", "pr":"monthly_mean/pr_s0-3h", - "prlr":"monthly_mean/prlr_s0-3h"} - daily_mean: {"tas":"daily_mean/tas_f6h", "psl":"daily_mean/psl_f6h", - "prlr":"daily_mean/prlr_s0-3h", "sfcWind":"daily_mean/sfcWind_f6h"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" - -# ---- - GISTEMPv4: - name: "GISTEMPv4" - institution: "NASA Goddard Institute for Space Studies" - src: "obs/noaa-nasa/ghcnersstgiss/" - monthly_mean: {"tasanomaly":""} - daily_mean: - calendar: "standard" - reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly/tasanomaly_200811.nc" - -# ---- - HadCRUT4: - name: "HadCRUT4" - institution: "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" - src: "obs/ukmo/hadcrut_v4.6/" - monthly_mean: {"tasanomaly":""} - daily_mean: - calendar: "standard" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" - -# ---- - HadSLP2: - name: "HadSLP2" - institution: - src: "obs/ukmo/hadslp_v2/" - monthly_mean: {"psl":""} - daily_mean: - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl/psl_200811.nc" - diff --git a/conf/archive_reference.yml b/conf/archive_reference.yml new file mode 100644 index 0000000000000000000000000000000000000000..41073eb9504304d74709bb8b5d4069bfa764f5e9 --- /dev/null +++ b/conf/archive_reference.yml @@ -0,0 +1,206 @@ +esarchive: + src_ref: "/esarchive/" + Reference: + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/era5/" + weekly_mean: {"tas":"weekly_mean/tas_f1h-r1440x721cds/", + "prlr":"weekly_mean/prlr_f1h-r1440x721cds/"} + daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", + "rsds":"daily_mean/rsds_f1h-r1440x721cds/", + "prlr":"daily_mean/prlr_f1h-r1440x721cds/", + "g300":"daily_mean/g300_f1h-r1440x721cds/", + "g500":"daily_mean/g500_f1h-r1440x721cds/", + "g850":"daily_mean/g850_f1h-r1440x721cds/", + "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", + "tasmax":"daily/tasmax-r1440x721cds/", + "tasmin":"daily/tasmin-r1440x721cds/", + "ta300":"daily_mean/ta300_f1h-r1440x721cds/", + "ta500":"daily_mean/ta500_f1h-r1440x721cds/", + "ta850":"daily_mean/ta850_f1h-r1440x721cds/", + "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} + monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", + "psl":"monthly_mean/psl_f1h-r1440x721cds/", + "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", + "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", + "g300":"monthly_mean/g300_f1h-r1440x721cds/", + "g500":"monthly_mean/g500_f1h-r1440x721cds/", + "g850":"monthly_mean/g850_f1h-r1440x721cds/", + "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", + "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", + "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", + "ta300":"montly_mean/ta300_f1h-r1440x721cds/", + "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", + "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", + "tos":"monthly_mean/tos_f1h-r1440x721cds/", + "sic":"monthly_mean/sic_f1h-r1440x721cds/"} + calendar: "gregorian" + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" + ERA5-Land: + name: "ERA5-Land" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/era5land/" + daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", + "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", + "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} + monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", + "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", + "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", + "tdps":"monthly_mean/tdps_f1h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" + UERRA: + name: "ECMWF UERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/uerra_mescan/" + daily_mean: {"tas":"daily_mean/tas_f6h/"} + monthly_mean: {"tas":"monthly_mean/tas_f6h/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" + CERRA: + name: "ECMWF CERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/cerra/" + daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", + "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", + "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} + monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", + "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", + "tas":"monthly_mean/tas_f3h-r2631x1113/", + "winddir":"monthly_mean/winddir_f3h-r2631x1113/", + "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", + "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" + CERRA-Land: + name: "ECMWF CERRA-Land" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/ecmwf/cerraland/" + daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} + monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" + HadCRUT5: + name: "HadCRUT5" + institution: "Met Office" + src: "obs/ukmo/hadcrut_v5.0_analysis/" + monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" + BEST: + name: "BEST" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "obs/berkeleyearth/berkeleyearth/" + daily_mean: {"tas":"daily_mean/tas/"} + monthly_mean: {"tas":"monthly_mean/tas/"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" +# + + GHCNv4: + name: + institution: + src: "obs/noaa/ghcn_v4/" + monthly_mean: {"tas":"", "tasanomaly":""} + daily_mean: + calendar: "gregorian" + reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" + +# ---- +# missing info in script + NCEP1: + +# ---- + JRA-55: + name: "JRA-55" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "recon/jma/jra55/" + monthly_mean: {"tas":"monthly_mean/tas_f6h", "psl":"monthly_mean/psl_f6h", + "tos":"", "pr":"monthly_mean/pr_s0-3h", + "prlr":"monthly_mean/prlr_s0-3h"} + daily_mean: {"tas":"daily_mean/tas_f6h", "psl":"daily_mean/psl_f6h", + "prlr":"daily_mean/prlr_s0-3h", "sfcWind":"daily_mean/sfcWind_f6h"} + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" + +# ---- + GISTEMPv4: + name: "GISTEMPv4" + institution: "NASA Goddard Institute for Space Studies" + src: "obs/noaa-nasa/ghcnersstgiss/" + monthly_mean: {"tasanomaly":""} + daily_mean: + calendar: "gregorian" + reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + HadCRUT4: + name: "HadCRUT4" + institution: "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" + src: "obs/ukmo/hadcrut_v4.6/" + monthly_mean: {"tasanomaly":""} + daily_mean: + calendar: "gregorian" + reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + HadSLP2: + name: "HadSLP2" + institution: + src: "obs/ukmo/hadslp_v2/" + monthly_mean: {"psl":""} + daily_mean: + calendar: "proleptic_gregorian" + reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl/psl_200811.nc" + +# ------------------------- +mars: + src_ref: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" + Reference: + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "GRIB_era5_tas/" + monthly_mean: {"tas":""} + calendar: "gregorian" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" + + +sample: + src_ref: + Reference: + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "GRIB_era5_tas/" + monthly_mean: {"tas":"", "prlr":""} + calendar: "gregorian" + reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" + + +IFCA: + src_ref: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" + Reference: + ERA5: + name: "ERA5" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "ERA5/" + daily_mean: {"psl":"psl_f1h-r1440x721cds/"} + calendar: "gregorian" + reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ERA5/daily_mean/psl_f1h-r1440x721cds/psl_201805.nc" + CERRA: + name: "ECMWF CERRA" + institution: "European Centre for Medium-Range Weather Forecasts" + src: "CERRA/" + daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "prlr":"daily_mean/prlr_f6h-r2631x1113/", + "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", "tas":"daily_mean/tas_f3h-r2631x1113/", + "tasmax":"daily_mean/tasmax-r2631x1113/", "tasmin":"daily_mean/tasmin-r2631x1113/"} + monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "prlr":"monthly_mean/prlr_f6h-r2631x1113/", + "sfcWind":"monthly_mean/_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", + "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/","tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} + calendar: "proleptic_gregorian" + reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/CERRA/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" + + diff --git a/conf/archive.yml b/conf/archive_seasonal.yml similarity index 53% rename from conf/archive.yml rename to conf/archive_seasonal.yml index e49e9b01785fdc9cd66beaed1e37bd8d74ac7032..125f180428d0e80dacdb291c548dff042dbdd7b2 100644 --- a/conf/archive.yml +++ b/conf/archive_seasonal.yml @@ -1,5 +1,5 @@ esarchive: - src: "/esarchive/" + src_sys: "/esarchive/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" @@ -136,104 +136,8 @@ esarchive: calendar: "gregorian" time_stamp_lag: "0" reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", - "rsds":"daily_mean/rsds_f1h-r1440x721cds/", - "prlr":"daily_mean/prlr_f1h-r1440x721cds/", - "g300":"daily_mean/g300_f1h-r1440x721cds/", - "g500":"daily_mean/g500_f1h-r1440x721cds/", - "g850":"daily_mean/g850_f1h-r1440x721cds/", - "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax_f1h-r1440x721cds/", - "tasmin":"daily/tasmin_f1h-r1440x721cds/", - "ta300":"daily_mean/ta300_f1h-r1440x721cds/", - "ta500":"daily_mean/ta500_f1h-r1440x721cds/", - "ta850":"daily_mean/ta850_f1h-r1440x721cds/", - "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", - "psl":"monthly_mean/psl_f1h-r1440x721cds/", - "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", - "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", - "g300":"monthly_mean/g300_f1h-r1440x721cds/", - "g500":"monthly_mean/g500_f1h-r1440x721cds/", - "g850":"monthly_mean/g850_f1h-r1440x721cds/", - "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", - "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", - "ta300":"montly_mean/ta300_f1h-r1440x721cds/", - "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", - "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", - "tos":"monthly_mean/tos_f1h-r1440x721cds/", - "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" - ERA5-Land: - name: "ERA5-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", - "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", - "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", - "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", - "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", - "tdps":"monthly_mean/tdps_f1h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - UERRA: - name: "ECMWF UERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"daily_mean/tas_f6h/"} - monthly_mean: {"tas":"monthly_mean/tas_f6h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" - CERRA: - name: "ECMWF CERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerra/" - daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", - "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", - "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", - "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", - "tas":"monthly_mean/tas_f3h-r2631x1113/", - "winddir":"monthly_mean/winddir_f3h-r2631x1113/", - "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", - "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" - CERRA-Land: - name: "ECMWF CERRA-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerraland/" - daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} - monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" - HadCRUT5: - name: "HadCRUT5" - institution: "Met Office" - src: "obs/ukmo/hadcrut_v5.0_analysis/" - monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" - BEST: - name: "BEST" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "obs/berkeleyearth/berkeleyearth/" - daily_mean: {"tas":"daily_mean/tas/"} - monthly_mean: {"tas":"monthly_mean/tas/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" - mars: - src: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" + src_sys: "/esarchive/scratch/aho/tmp/GRIB/" #"/mars/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" @@ -246,17 +150,9 @@ mars: calendar: "proleptic_gregorian" time_stamp_lag: "+1" reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "GRIB_era5_tas/" - monthly_mean: {"tas":""} - calendar: "standard" - reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" sample: - src: + src_sys: System: ECMWF-SEAS5.1: name: "ECMWF SEAS5" @@ -269,17 +165,9 @@ sample: calendar: "proleptic_gregorian" time_stamp_lag: "0" reference_grid: "conf/grid_description/griddes_GRIB_system51_m1.txt" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "GRIB_era5_tas/" - monthly_mean: {"tas":"", "prlr":""} - calendar: "standard" - reference_grid: "conf/grid_description/griddes_GRIB_system5_m1.txt" IFCA: - src: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" + src_sys: "/home/jovyan/IMPETUS4CHANGE/data/BSC/" System: ECMWF-SEAS5: name: "ECMWF SEAS5" @@ -298,23 +186,3 @@ IFCA: calendar: "proleptic_gregorian" time_stamp_lag: "0" reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ECMWF-SEAS5/monthly_mean/tas_f6h/tas_20180501.nc" - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "ERA5/" - daily_mean: {"psl":"psl_f1h-r1440x721cds/"} - calendar: "standard" - reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/ERA5/daily_mean/psl_f1h-r1440x721cds/psl_201805.nc" - CERRA: - name: "ECMWF CERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "CERRA/" - daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "prlr":"daily_mean/prlr_f6h-r2631x1113/", - "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", "tas":"daily_mean/tas_f3h-r2631x1113/", - "tasmax":"daily_mean/tasmax-r2631x1113/", "tasmin":"daily_mean/tasmin-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "prlr":"monthly_mean/prlr_f6h-r2631x1113/", - "sfcWind":"monthly_mean/_f3h-r2631x1113/", "tas":"monthly_mean/tas_f3h-r2631x1113/", - "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/","tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/home/jovyan/IMPETUS4CHANGE/data/BSC/CERRA/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" diff --git a/conf/archive_subseasonal.yml b/conf/archive_subseasonal.yml index c142851dcdef7ae21de59168a88f804e4534fb77..a48fe0fa9c66dfa3b1ac174665ede98966bf563c 100644 --- a/conf/archive_subseasonal.yml +++ b/conf/archive_subseasonal.yml @@ -1,5 +1,5 @@ esarchive: - src: "/esarchive/" + src_sys: "/esarchive/" System: NCEP-CFSv2: name: "NCEP CFSv2" @@ -18,100 +18,4 @@ esarchive: calendar: "proleptic_gregorian" time_stamp_lag: "0" # Do we need it for subseasonal? reference_grid: "/esarchive/exp/ncep/cfs-v2/weekly_mean/s2s/tas_f24h/tas_20040624.nc" # is it the same as seasonal? - Reference: - ERA5: - name: "ERA5" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5/" - weekly_mean: {"tas":"weekly_mean/tas_f1h-r1440x721cds/", - "prlr":"weekly_mean/prlr_f1h-r1440x721cds/"} - daily_mean: {"tas":"daily_mean/tas_f1h-r1440x721cds/", - "rsds":"daily_mean/rsds_f1h-r1440x721cds/", - "prlr":"daily_mean/prlr_f1h-r1440x721cds/", - "g300":"daily_mean/g300_f1h-r1440x721cds/", - "g500":"daily_mean/g500_f1h-r1440x721cds/", - "g850":"daily_mean/g850_f1h-r1440x721cds/", - "sfcWind":"daily_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"daily/tasmax_f1h-r1440x721cds/", - "tasmin":"daily/tasmin_f1h-r1440x721cds/", - "ta300":"daily_mean/ta300_f1h-r1440x721cds/", - "ta500":"daily_mean/ta500_f1h-r1440x721cds/", - "ta850":"daily_mean/ta850_f1h-r1440x721cds/", - "hurs":"daily_mean/hurs_f1h-r1440x721cds/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h-r1440x721cds/", - "psl":"monthly_mean/psl_f1h-r1440x721cds/", - "prlr":"monthly_mean/prlr_f1h-r1440x721cds/", - "rsds":"monthly_mean/rsds_f1h-r1440x721cds/", - "g300":"monthly_mean/g300_f1h-r1440x721cds/", - "g500":"monthly_mean/g500_f1h-r1440x721cds/", - "g850":"monthly_mean/g850_f1h-r1440x721cds/", - "sfcWind":"monthly_mean/sfcWind_f1h-r1440x721cds/", - "tasmax":"monthly_mean/tasmax_f1h-r1440x721cds/", - "tasmin":"monthly_mean/tasmin_f1h-r1440x721cds/", - "ta300":"montly_mean/ta300_f1h-r1440x721cds/", - "ta500":"monthly_mean/ta500_f1h-r1440x721cds/", - "ta850":"monthly_mean/ta850_f1h-r1440x721cds/", - "tos":"monthly_mean/tos_f1h-r1440x721cds/", - "sic":"monthly_mean/sic_f1h-r1440x721cds/"} - calendar: "standard" - reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" - land_sea_mask: "/esarchive/recon/ecmwf/era5/constant/lsm-r1440x721cds/sftof.nc" - ERA5-Land: - name: "ERA5-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/era5land/" - daily_mean: {"tas":"daily_mean/tas_f1h/", "rsds":"daily_mean/rsds_f1h/", - "prlr":"daily_mean/prlr_f1h/", "sfcWind":"daily_mean/sfcWind_f1h/", - "tasmin":"daily/tasmin/", "tasmax":"daily/tasmax/"} - monthly_mean: {"tas":"monthly_mean/tas_f1h/","tasmin":"monthly_mean/tasmin_f24h/", - "tasmax":"monthly_mean/tasmax_f24h/", "prlr":"monthly_mean/prlr_f1h/", - "sfcWind":"monthly_mean/sfcWind_f1h/", "rsds":"monthly_mean/rsds_f1h/", - "tdps":"monthly_mean/tdps_f1h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" - UERRA: - name: "ECMWF UERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/uerra_mescan/" - daily_mean: {"tas":"daily_mean/tas_f6h/"} - monthly_mean: {"tas":"monthly_mean/tas_f6h/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" - CERRA: - name: "ECMWF CERRA" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerra/" - daily_mean: {"hurs":"daily_mean/hurs_f3h-r2631x1113/", "ps":"daily_mean/ps_f3h-r2631x1113/", - "sfcWind":"daily_mean/sfcWind_f3h-r2631x1113/", - "tas":"daily_mean/tas_f3h-r2631x1113/", "winddir":"daily_mean/tas_f3h-r2631x1113/"} - monthly_mean: {"hurs":"monthly_mean/hurs_f3h-r2631x1113/", "ps":"monthly_mean/ps_f3h-r2631x1113/", - "sfcWind":"monthly_mean/sfcWind_f3h-r2631x1113/", - "tas":"monthly_mean/tas_f3h-r2631x1113/", - "winddir":"monthly_mean/winddir_f3h-r2631x1113/", - "tasmin":"monthly_mean/tasmin_f24h-r2631x1113/", - "tasmax":"monthly_mean/tasmax_f24h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerra/monthly_mean/tas_f3h-r2631x1113/tas_200506.nc" - CERRA-Land: - name: "ECMWF CERRA-Land" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "recon/ecmwf/cerraland/" - daily_mean: {"prlr":"daily_mean/prlr_f6h-r2631x1113/"} - monthly_mean: {"prlr":"monthly_mean/prlr_f6h-r2631x1113/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/recon/ecmwf/cerraland/monthly_mean/prlr_f6h-r2631x1113/prlr_200412.nc" - HadCRUT5: - name: "HadCRUT5" - institution: "Met Office" - src: "obs/ukmo/hadcrut_v5.0_analysis/" - monthly_mean: {"tasanomaly":"monthly_mean/tasanomaly/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/ukmo/hadcrut_v5.0_analysis/monthly_mean/tasanomaly/tasanomaly_202001.nc" - BEST: - name: "BEST" - institution: "European Centre for Medium-Range Weather Forecasts" - src: "obs/berkeleyearth/berkeleyearth/" - daily_mean: {"tas":"daily_mean/tas/"} - monthly_mean: {"tas":"monthly_mean/tas/"} - calendar: "proleptic_gregorian" - reference_grid: "/esarchive/obs/berkeleyearth/berkeleyearth/monthly_mean/tas/tas_201805.nc" + diff --git a/modules/Downscaling/Downscaling.R b/modules/Downscaling/Downscaling.R index e95dfadd874abd049c2053f48b595fadd9b1becd..07c228388c30fd6cab90b9cb8bfb59aa444eb7f2 100644 --- a/modules/Downscaling/Downscaling.R +++ b/modules/Downscaling/Downscaling.R @@ -38,7 +38,7 @@ Downscaling <- function(recipe, data) { probs_cat <- sapply(unlist(recipe$Analysis$Workflow$Downscaling$probs_cat), function(frac) eval(parse(text = frac))) if (tolower(recipe$Analysis$Horizon) == "seasonal") { - archive <- read_yaml("conf/archive.yml") + archive <- read_yaml("conf/archive_seasonal.yml") } else if (tolower(recipe$Analysis$Horizon) == "subseasonal") { archive <- read_yaml("conf/archive_subseasonal.yml") } else if (tolower(recipe$Analysis$Horizon) == "decadal") { diff --git a/modules/Loading/R/helper_loading_decadal.R b/modules/Loading/R/helper_loading_decadal.R index 9b71c94bf5326925c587576f8fb1414c10874144..65058a6bb502b55d3c75ebfdcd6817c130a425e3 100644 --- a/modules/Loading/R/helper_loading_decadal.R +++ b/modules/Loading/R/helper_loading_decadal.R @@ -111,12 +111,12 @@ get_dcpp_path <- function(archive, exp.name, table, grid, version, sdates) { if (length(table) == 1) { # only one variable, or the variables share a common path pattern - fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$startR, + fcst.path <- file.path(archive$src_sys, archive$System[[exp.name]]$src$startR, '$ensemble$', table, '$var$', grid, 'v*') fcst.files <- paste0('$var$_', table, '_*_$dcpp$_s$syear$-$ensemble$_', grid, '_$chunk$.nc') } else { # path pattern depends on the variable - fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$startR, + fcst.path <- file.path(archive$src_sys, archive$System[[exp.name]]$src$startR, '$ensemble$', '$table$', '$var$', '$grid$', '$version$') fcst.files <- paste0('$var$_', '$table$', '_*_$dcpp$_s$syear$-$ensemble$_', '$grid$', '_$chunk$.nc') } diff --git a/modules/Loading/R/load_GRIB.R b/modules/Loading/R/load_GRIB.R index 0dd5f9191c4275b66c5525c248e03b9c6e0a0e8b..9252561a9ced74f5f8280b54f2cfd6e885eeb3da 100644 --- a/modules/Loading/R/load_GRIB.R +++ b/modules/Loading/R/load_GRIB.R @@ -22,14 +22,14 @@ load_GRIB <- function(recipe) { lons.max <- recipe$Analysis$Region$lonmax # can only be 360 ref.name <- recipe$Analysis$Datasets$Reference$name exp.name <- recipe$Analysis$Datasets$System$name - variable <- recipe$Analysis$Variables$name #'tas' + variable <- recipe$Analysis$Variables$name store.freq <- recipe$Analysis$Variables$freq regrid.method <- recipe$Analysis$Regrid$method regrid.type <- recipe$Analysis$Regrid$type # get MARS datasets dict: - archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + archive <- get_archive(recipe) exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable]) reference_descrip <- archive$Reference[[ref.name]] @@ -54,7 +54,7 @@ load_GRIB <- function(recipe) { # The correct files #exp_path <- "/esarchive/scratch/aho/tmp/GRIB/GRIB_system5_tas_CORRECTED/" - hcst.path <- paste0(archive$src, hcst.dir) + hcst.path <- paste0(archive$src_sys, hcst.dir) hcst.year <- paste0(as.numeric(hcst.inityear):as.numeric(hcst.endyear)) hcst.files <- paste0(hcst.path, variable, '_', hcst.year, hcst.sdate, '.grb') @@ -79,7 +79,7 @@ load_GRIB <- function(recipe) { # Load forecast #------------------------------------------------------------------- if (!is.null(fcst.year)) { - fcst.path <- paste0(archive$src, hcst.dir) + fcst.path <- paste0(archive$src_sys, hcst.dir) fcst.files <- paste0(fcst.path, variable, '_', fcst.year, hcst.sdate, '.grb') fcst <- GrbLoad(dat = as.list(fcst.files), time_step = hcst.ftime, has.memb = fcst.nmember, syear_time_dim = NULL, regrid = regrid_list) @@ -94,7 +94,7 @@ load_GRIB <- function(recipe) { # Load reference #------------------------------------------------------------------- #obs_path <- "/esarchive/scratch/aho/tmp/GRIB/GRIB_era5_tas/" - obs.path <- paste0(archive$src, obs.dir) + obs.path <- paste0(archive$src_ref, obs.dir) # Use hcst time attr to load obs hcst_times <- attr(hcst, 'time') hcst_times_strings <- format(hcst_times, '%Y%m') diff --git a/modules/Loading/R/load_decadal.R b/modules/Loading/R/load_decadal.R index bfd289792914c3a92b5982f75df4d5fab53e65fe..21df4223e9953195bb98cadf6baaf1be7bee7ccd 100644 --- a/modules/Loading/R/load_decadal.R +++ b/modules/Loading/R/load_decadal.R @@ -13,7 +13,7 @@ source("modules/Loading/R/compare_exp_obs_grids.R") load_decadal <- function(recipe) { ## - archive <- read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]] + archive <- get_archive(recipe) # Print Start() info or not DEBUG <- TRUE @@ -213,7 +213,7 @@ load_decadal <- function(recipe) { #------------------------------------------- # Step 3. Load the reference #------------------------------------------- - obs.path <- file.path(archive$src, archive$Reference[[ref.name]]$src, + obs.path <- file.path(archive$src_ref, archive$Reference[[ref.name]]$src, "$var_dir$", "$var$_$file_date$.nc") var_dir_obs <- archive$Reference[[ref.name]][[store.freq]][variable] # list(tas = "_f1h-r1440x721cds", tos = "_f1h-r1440x721cds") diff --git a/modules/Loading/R/load_seasonal.R b/modules/Loading/R/load_seasonal.R index 4b8fa6aca80bd50d2189e4a817839419c1c5c965..bbee8d4d6a8615f0e0eac79576d8a1a5fdfd6fa4 100644 --- a/modules/Loading/R/load_seasonal.R +++ b/modules/Loading/R/load_seasonal.R @@ -47,7 +47,7 @@ load_seasonal <- function(recipe) { ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name # get datasets dict: - archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + archive <- get_archive(recipe) exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable[1]]) @@ -75,13 +75,13 @@ load_seasonal <- function(recipe) { var_dir_exp <- exp_descrip[[frequency]][variable] # ----------- - obs.path <- paste0(archive$src, obs.dir, "$var_dir$", + obs.path <- paste0(archive$src_ref, obs.dir, "$var_dir$", "$var$_$file_date$.nc") - hcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + hcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") - fcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + fcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") # Define regrid parameters: diff --git a/modules/Loading/R/load_subseasonal.R b/modules/Loading/R/load_subseasonal.R index 793b46ecd98f5c3d32762687c263f85c96c7aecd..9176ad7e70da79ac8917a55c507c7313a75e8170 100644 --- a/modules/Loading/R/load_subseasonal.R +++ b/modules/Loading/R/load_subseasonal.R @@ -33,7 +33,7 @@ load_subseasonal <- function(recipe) { ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name # get datasets dict: - archive <- read_yaml("conf/archive_subseasonal.yml")[[recipe$Run$filesystem]] + archive <-get_archive(recipe) # read_yaml("conf/archive_subseasonal.yml")[[recipe$Run$filesystem]] exp_descrip <- archive$System[[exp.name]] freq.hcst <- unlist(exp_descrip[[store.freq]][variable[1]]) @@ -64,13 +64,13 @@ load_subseasonal <- function(recipe) { var_dir_exp <- exp_descrip[[frequency]][variable] # ----------- - obs.path <- paste0(archive$src, obs.dir, "$var_dir$", + obs.path <- paste0(archive$src_ref, obs.dir, "$var_dir$", "$var$_$file_date$.nc") - hcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + hcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") - fcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + fcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") # Define regrid parameters: diff --git a/modules/Loading/R/load_tas_tos.R b/modules/Loading/R/load_tas_tos.R index 18c000ad1dfc9d5ba3c4bc8f3575077ef1215c8d..4710920ba902507291d350ebebc7c809064609c2 100644 --- a/modules/Loading/R/load_tas_tos.R +++ b/modules/Loading/R/load_tas_tos.R @@ -50,7 +50,7 @@ load_tas_tos <- function(recipe) { } # get esarchive datasets dict: - archive <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + archive <- get_archive(recipe) exp_descrip <- archive$System[[exp.name]] reference_descrip <- archive$Reference[[ref.name]] @@ -70,13 +70,13 @@ load_tas_tos <- function(recipe) { var_dir_exp <- exp_descrip[[frequency]][variable] # ----------- - obs.path <- paste0(archive$src, obs.dir, "$var_dir$", + obs.path <- paste0(archive$src_ref, obs.dir, "$var_dir$", "$var$_$file_date$.nc") - hcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + hcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "$var$_$file_date$.nc") - fcst.path <- paste0(archive$src, hcst.dir, "$var_dir$", + fcst.path <- paste0(archive$src_sys, hcst.dir, "$var_dir$", "/$var$_$file_date$.nc") # Define regrid parameters: diff --git a/modules/Multimodel/dims_multimodel.R b/modules/Multimodel/dims_multimodel.R index b132f1b4bb444ead67c23244ddf81fbaf40b0d47..9126350fd1af2fd70e280daa5b4ec28ce0b93adf 100644 --- a/modules/Multimodel/dims_multimodel.R +++ b/modules/Multimodel/dims_multimodel.R @@ -5,7 +5,7 @@ source("modules/Loading/R/check_latlon.R") dims_multimodel <- function(recipe) { - archive <- read_yaml("conf/archive.yml")$esarchive + archive <- get_archive() ref.name <- recipe$Analysis$Datasets$Reference$name if (tolower(recipe$Analysis$Horizon) == 'seasonal') { exp.name <- recipe$Analysis$Datasets$System$models diff --git a/modules/Multimodel/load_multimodel.R b/modules/Multimodel/load_multimodel.R index 529d2e5d4c725132c4836fcfeb090478e8b6589c..778aae6e6d08a517444e367139b1134033696128 100644 --- a/modules/Multimodel/load_multimodel.R +++ b/modules/Multimodel/load_multimodel.R @@ -5,7 +5,7 @@ source("modules/Loading/R/check_latlon.R") load_multimodel <- function(recipe) { - archive <- read_yaml("conf/archive.yml")$esarchive + archive <- get_archive(recipe) ref.name <- recipe$Analysis$Datasets$Reference$name if (tolower(recipe$Analysis$Horizon) == 'seasonal') { exp.name <- recipe$Analysis$Datasets$System$models diff --git a/modules/Saving/R/save_forecast.R b/modules/Saving/R/save_forecast.R index 6e3bc906d5490bf32ad9631370c37e205c4dc4fd..b2d74fc9936dfafc4fa4962f5b2d9948d4d7c676 100644 --- a/modules/Saving/R/save_forecast.R +++ b/modules/Saving/R/save_forecast.R @@ -16,8 +16,8 @@ save_forecast <- function(recipe, global_attributes <- .get_global_attributes(recipe, archive) fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq - if (global_attributes$system == 'Multimodel'){ - if (fcst.horizon == 'decadal'){ + if (global_attributes$system == 'Multimodel') { + if (fcst.horizon == 'decadal') { calendar <- archive$System[[recipe$Analysis$Datasets$System$models[[1]]$name]]$calendar } else { calendar <- archive$System[[recipe$Analysis$Datasets$System$models[[1]]]]$calendar diff --git a/modules/Scorecards/R/tmp/ScorecardsMulti.R b/modules/Scorecards/R/tmp/ScorecardsMulti.R index 7278eb16a2663ac9f723c68e375522439b046237..01d561e084bfa316de73c8f3de9c16f9d2aa45c6 100644 --- a/modules/Scorecards/R/tmp/ScorecardsMulti.R +++ b/modules/Scorecards/R/tmp/ScorecardsMulti.R @@ -131,7 +131,7 @@ ScorecardsMulti <- function(data, sign, system, reference, var, start.year, } else { filesystem <- recipe$Run$filesystem } - sys_dict <- read_yaml("conf/archive.yml")[[filesystem]] + sys_dict <- get_archive(recipe) var_dict <- read_yaml("conf/variable-dictionary.yml")$vars ## Get scorecards table display names from configuration files diff --git a/modules/Scorecards/R/tmp/ScorecardsSingle.R b/modules/Scorecards/R/tmp/ScorecardsSingle.R index 9fd445451dcd4bb54d3aeb7c7eab43075434e6eb..61a6ee70cd4b0cc07c0b515b03da4258b74de502 100644 --- a/modules/Scorecards/R/tmp/ScorecardsSingle.R +++ b/modules/Scorecards/R/tmp/ScorecardsSingle.R @@ -143,7 +143,7 @@ ScorecardsSingle <- function(data, sign, system, reference, var, start.year, } else { filesystem <- recipe$Run$filesystem } - sys_dict <- read_yaml("conf/archive.yml")[[filesystem]] + sys_dict <- get_archive(recipe) var_dict <- read_yaml("conf/variable-dictionary.yml")$vars ## Get scorecards table display names from configuration files diff --git a/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R b/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R index 88e9de1530b68be962773f9f836386947ebc0c79..c785bd5d2cb284f7dc4d6e42e3afefe4c14c28b5 100644 --- a/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R +++ b/modules/Scorecards/R/tmp/ScorecardsSystemDiff.R @@ -116,7 +116,7 @@ ScorecardsSystemDiff <- function(data, ftime_dim = 'time') ## Load configuration files - sys_dict <- read_yaml("conf/archive.yml")[[recipe$Run$filesystem]] + sys_dict <- get_archive(recipe) var_dict <- read_yaml("conf/variable-dictionary.yml")$vars ## Get scorecards table display names from configuration files diff --git a/tests/testthat/test-seasonal_NAO.R b/tests/testthat/test-seasonal_NAO.R index f70eefc755ef256c856ad9a123de336c577fedb1..12c406ebc87aec14a43a19f9f250241cbcd9b7c3 100644 --- a/tests/testthat/test-seasonal_NAO.R +++ b/tests/testthat/test-seasonal_NAO.R @@ -8,7 +8,6 @@ source("modules/Indices/Indices.R") recipe_file <- "tests/recipes/recipe-seasonal_NAO.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly.R b/tests/testthat/test-seasonal_monthly.R index 18bde4c01ec55b2fb29e618e588af74dc167c5f7..198c947ab7bee2ee75ce4086ac1ee8393db35935 100644 --- a/tests/testthat/test-seasonal_monthly.R +++ b/tests/testthat/test-seasonal_monthly.R @@ -8,7 +8,6 @@ source("modules/Visualization/Visualization.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly_statistics.R b/tests/testthat/test-seasonal_monthly_statistics.R index cae7611186539d4de6d5985d7a97bbf95d7a2e87..74647a211ffecb36949e1683e612b6fe2302aea1 100644 --- a/tests/testthat/test-seasonal_monthly_statistics.R +++ b/tests/testthat/test-seasonal_monthly_statistics.R @@ -7,7 +7,6 @@ source("modules/Visualization/Visualization.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_statistics.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly_tas-tos.R b/tests/testthat/test-seasonal_monthly_tas-tos.R index 1345f63dfb1190988413905221cbda99c2f37c74..4078d6e7486f72247b41d5411b71b9f632a379d8 100644 --- a/tests/testthat/test-seasonal_monthly_tas-tos.R +++ b/tests/testthat/test-seasonal_monthly_tas-tos.R @@ -4,7 +4,6 @@ source("./modules/Loading/Loading.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_tas-tos.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tests/testthat/test-seasonal_monthly_timeagg.R b/tests/testthat/test-seasonal_monthly_timeagg.R index 21977d9c131e709b2d7423b9890e5869e5f7384d..ffa389c49ac0af3112294724e0952230cce91bdf 100644 --- a/tests/testthat/test-seasonal_monthly_timeagg.R +++ b/tests/testthat/test-seasonal_monthly_timeagg.R @@ -9,7 +9,6 @@ source("modules/Aggregation/Aggregation.R") recipe_file <- "tests/recipes/recipe-seasonal_monthly_1_timeagg.yml" recipe <- prepare_outputs(recipe_file, disable_checks = F) -archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets suppressWarnings({invisible(capture.output( diff --git a/tools/check_recipe.R b/tools/check_recipe.R index 011344761ec1521f6bbebf3092ba94131f5fa01b..c9641787e99beec2f5e399c03a4c445967fca57e 100644 --- a/tools/check_recipe.R +++ b/tools/check_recipe.R @@ -17,9 +17,6 @@ check_recipe <- function(recipe) { PARAMS <- c("Horizon", "Time", "Variables", "Region", "Regrid", "Workflow", "Datasets") HORIZONS <- c("subseasonal", "seasonal", "decadal") - ARCHIVE_SEASONAL <- "conf/archive.yml" - ARCHIVE_DECADAL <- "conf/archive_decadal.yml" - ARCHIVE_SUBSEASONAL <- "conf/archive_subseasonal.yml" # Define error status variable error_status <- F @@ -45,9 +42,8 @@ check_recipe <- function(recipe) { error_status <- T } # Check time settings + archive <- get_archive(recipe) if (recipe$Analysis$Horizon == "seasonal") { - ## TODO: Specify filesystem - archive <- read_yaml(ARCHIVE_SEASONAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_SEASONAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -56,7 +52,6 @@ check_recipe <- function(recipe) { error_status <- TRUE } } else if (recipe$Analysis$Horizon == "decadal") { - archive <- read_yaml(ARCHIVE_DECADAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_DECADAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -65,7 +60,6 @@ check_recipe <- function(recipe) { error_status <- TRUE } } else if (recipe$Analysis$Horizon == "subseasonal") { - archive <- read_yaml(ARCHIVE_SUBSEASONAL)[[recipe$Run$filesystem]] if (!all(TIME_SETTINGS_SUBSEASONAL %in% names(recipe$Analysis$Time))) { error(recipe$Run$logger, paste0("The element 'Time' in the recipe must contain all of the ", @@ -76,6 +70,7 @@ check_recipe <- function(recipe) { } else { archive <- NULL } + # Check variable parameters if ("name" %in% names(recipe$Analysis$Variables)) { recipe_variables <- recipe$Analysis$Variables$name diff --git a/tools/get_archive.R b/tools/get_archive.R index 56309ccb1db11ca6018116b382bd52131e30aacc..edcc057f3333cffd6bfdee9365e79a27cd74978e 100644 --- a/tools/get_archive.R +++ b/tools/get_archive.R @@ -1,7 +1,7 @@ get_archive <- function(recipe) { if (tolower(recipe$Analysis$Horizon) == "seasonal") { archive <- - read_yaml(paste0("conf/archive.yml"))[[recipe$Run$filesystem]] + read_yaml(paste0("conf/archive_seasonal.yml"))[[recipe$Run$filesystem]] } else if (tolower(recipe$Analysis$Horizon) == "subseasonal") { archive <- read_yaml(paste0("conf/archive_subseasonal.yml"))[[recipe$Run$filesystem]] @@ -9,6 +9,9 @@ get_archive <- function(recipe) { archive <- read_yaml(paste0("conf/archive_decadal.yml"))[[recipe$Run$filesystem]] } + archive_ref <- read_yaml(paste0("conf/archive_reference.yml"))[[recipe$Run$filesystem]] + archive$Reference <- archive_ref$Reference + archive$src_ref <- archive_ref$src_ref ## TODO: Add dictionary filesystem dependency? # dict <- read_yaml("conf/variable-dictionary.yml") return(archive) diff --git a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml index 68a8dc627dbfdfffd3772aab55c2b1c80ba12881..7201ea16087551315a0361b357358974b93ad571 100644 --- a/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml +++ b/use_cases/ex1_2_autosubmit_scorecards/ex1_2-recipe.yml @@ -80,21 +80,21 @@ Run: Loglevel: INFO Terminal: yes filesystem: esarchive - output_dir: /esarchive/scratch/aho/auto-s2s-outputs/ - code_dir: /esarchive/scratch/aho/git/auto-s2s/ + output_dir: /esarchive/scratch/vagudets/auto-s2s-outputs/ + code_dir: /home/Earth/vagudets/git/auto-s2s/ autosubmit: yes # fill only if using autosubmit auto_conf: - script: /esarchive/scratch/aho/git/auto-s2s/use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script - expid: a6pc # replace with your EXPID - hpc_user: bsc032734 # replace with your hpc username + script: use_cases/ex1_2_autosubmit_scorecards/ex1_2-script.R # replace with the path to your script + expid: a6wq # replace with your EXPID + hpc_user: bsc032762 # replace with your hpc username wallclock: 03:00 # hh:mm - processors_per_job: 8 + processors_per_job: 16 platform: nord3v2 custom_directives: ['#SBATCH --exclusive'] email_notifications: yes # enable/disable email notifications. Change it if you want to. - email_address: an.ho@bsc.es # replace with your email address + email_address: victoria.agudetse@bsc.es # replace with your email address notify_completed: yes # notify me by email when a job finishes - notify_failed: yes # notify me by email when a job fails + notify_failed: no # notify me by email when a job fails