diff --git a/.gitignore b/.gitignore index 752f5cf97a2e597b4d8863bf8cfd55046423eb0f..8fec461d052296bfd1779cceec4e33b7b8893afa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ out-logs/ *.swp *.swo /modules/Calibration/test_victoria.R +modules/Loading/testing_recipes/recipe_decadal_calendartest.yml +modules/Loading/testing_recipes/recipe_decadal_daily_calendartest.yml diff --git a/conf/archive.yml b/conf/archive.yml index 492cde285ae9427ccf438dbe6d8557fe4189fc1e..ac65cb6d80e254e0ead46f63ea53b99c9fe110c0 100644 --- a/conf/archive.yml +++ b/conf/archive.yml @@ -14,6 +14,7 @@ archive: nmember: fcst: 51 hcst: 25 + calendar: "proleptic_gregorian" reference_grid: "/esarchive/exp/ecmwf/system5c3s/monthly_mean/tas_f6h/tas_20180501.nc" system7c3s: src: "exp/meteofrance/system7c3s/" @@ -23,6 +24,7 @@ archive: nmember: fcst: 51 hcst: 25 + calendar: "proleptic_gregorian" reference_grid: "conf/grid_description/griddes_system7c3s.txt" system21_m1: src: "exp/dwd/system21_m1/" @@ -32,6 +34,7 @@ archive: nmember: fcst: 50 hcst: 30 + calendar: "proleptic_gregorian" reference_grid: "conf/grid_description/griddes_system21_m1.txt" system35c3s: src: "exp/cmcc/system35c3s/" @@ -41,6 +44,7 @@ archive: nmember: fcst: 50 hcst: 40 + calendar: "proleptic_gregorian" reference_grid: "conf/grid_description/griddes_system35c3s.txt" system2c3s: src: "exp/jma/system2c3s/" @@ -49,6 +53,7 @@ archive: nmember: fcst: 10 hcst: 10 + calendar: "proleptic_gregorian" reference_grid: "conf/grid_description/griddes_system2c3s.txt" eccc1: src: "exp/eccc/eccc1/" @@ -57,6 +62,7 @@ archive: nmember: fcst: 10 hcst: 10 + calendar: "proleptic_gregorian" reference_grid: "conf/grid_description/griddes_eccc1.txt" glosea6_system600-c3s: src: "exp/ukmo/glosea6_system600-c3s/" @@ -65,6 +71,7 @@ archive: nmember: fcst: 62 hcst: 28 + calendar: "proleptic_gregorian" reference_grid: "conf/grid_description/griddes_ukmo600.txt" ncep-cfsv2: @@ -74,6 +81,7 @@ archive: nmember: fcst: 20 hcst: 20 + calendar: "gregorian" reference_grid: "conf/grid_description/griddes_ncep-cfsv2.txt" Reference: era5: @@ -90,6 +98,7 @@ archive: "sfcWind":"_f1h-r1440x721cds/", "tasmax":"_f1h-r1440x721cds/", "tasmin":"_f1h-r1440x721cds/"} + calendar: "standard" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" era5land: src: "recon/ecmwf/era5land/" @@ -98,11 +107,13 @@ archive: monthly_mean: {"tas":"_f1h/","tasmin":"_f24h/", "tasmax":"_f24h/", "prlr":"_f1h/", "sfcWind":"_f1h/", "rsds":"_f1h/"} + calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/era5land/daily_mean/tas_f1h/tas_201805.nc" uerra: src: "recon/ecmwf/uerra_mescan/" daily_mean: {"tas":"_f6h/"} monthly_mean: {"tas":"_f6h/"} + calendar: "proleptic_gregorian" reference_grid: "/esarchive/recon/ecmwf/uerra_mescan/daily_mean/tas_f6h/tas_201805.nc" diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 03e4142546529ab455bb7d1841e2ddc843514f9d..61e3f2b8eb5d781be670d619e2113c9e5619d229 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -14,7 +14,7 @@ archive: daily_mean: grid: {"clt":"gr", "hurs":"gr", "hursmin":"gr", "pr":"gr", "psl":"gr", "rsds":"gr", "sfcWind":"gr", "sfcWindmax":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} version: {"clt":"v20190713", "hurs":"v20190713", "hursmin":"v20190713", "pr":"v20190713", "psl":"v20190713", "rsds":"v20190713", "sfcWind":"v20190713", "sfcWindmax":"v20190713", "tas":"v20190713", "tasmax":"v20190713", "tasmin":"v20190713"} - calendar: "proleptic_gregorian" + calendar: "proleptic_gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20190713/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196011-196110.nc" #'r512x256' @@ -33,7 +33,7 @@ archive: daily_mean: grid: {"pr":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} version: {"pr":"v20200508", "tas":"v20200731", "tasmax":"v20200730", "tasmin":"v20200730"} - calendar: "proleptic_gregorian" + calendar: "proleptic_gregorian" #NOTE:There are many members but not all of them are available on ESGF (only r6-10 available). Then, we might have some variables for the rest of the members (r1-5 and r11-15), but not for all the variables. That's why i'm only using r6-10 member: r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 initial_month: 11 @@ -70,7 +70,7 @@ archive: daily_mean: grid: {"tas":"gr", "pr":"gr", "psl":"gr"} version: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910"} - calendar: "proleptic_gregorian" + calendar: "proleptic_gregorian" member: r1i4p1f1,r2i4p1f1,r3i4p1f1,r4i4p1f1,r5i4p1f1,r6i4p1f1,r7i4p1f1,r8i4p1f1,r9i4p1f1,r10i4p1f1 initial_month: 11 reference_grid: "/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i4p1f1/Amon/tas/gr/v20210910/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i4p1f1_gr_196011-196110.nc" @@ -89,7 +89,7 @@ archive: daily_mean: grid: {"tas":"gn"} version: {"tasmin":"v20200101", "tasmax":"v20200101", "pr":"v20200417"} - calendar: "360-day" + calendar: "360-day" member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Amon/tas/gr/v20200316/tas_Amon_HadGEM3_dcppA-hindcast_s2018-r1i1p1f2_gr_201811-202903.nc" #'r432x324' @@ -108,7 +108,7 @@ archive: daily_mean: grid: {"pr":"gn", "sfcWind":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn", "uas":"gn", "vas":"gn"} version: {"pr":"v20200114", "sfcWind":"v20200101", "tas":"v20200408", "tasmax":"v20200114", "tasmin":"v20200114", "uas":"v20200114", "vas":"v20200114"} - calendar: "365_day" + calendar: "365_day" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 initial_month: 1 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/BCC-CSM2-MR/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r8i1p1f1/Amon/tas/gn/v20200101/tas_Amon_BCC-CSM2-MR_dcppA-hindcast_s2008-r8i1p1f1_gn_200801-201712.nc" @@ -127,7 +127,7 @@ archive: daily_mean: grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} version: {"pr":"v20190429", "tas":"v20190429", "tasmax":"v20190429", "tasmin":"v20190429"} - calendar: "365_day" + calendar: "365_day" member: r1i1p2f1,r2i1p2f1,r3i1p2f1,r4i1p2f1,r5i1p2f1,r6i1p2f1,r7i1p2f1,r8i1p2f1, r9i1p2f1, r10i1p2f1, r11i1p2f1,r12i1p2f1,r13i1p2f1,r14i1p2f1,r15i1p2f1,r16i1p2f1,r17i1p2f1,r18i1p2f1, r19i1p2f1, r20i1p2f1,r21i1p2f1,r22i1p2f1,r23i1p2f1,r24i1p2f1,r25i1p2f1,r26i1p2f1,r27i1p2f1,r28i1p2f1, r29i1p2f1, r30i1p2f1, r31i1p2f1,r32i1p2f1,r33i1p2f1,r34i1p2f1,r35i1p2f1,r36i1p2f1,r37i1p2f1,r38i1p2f1, r39i1p2f1, r40i1p2f1 initial_month: 1 #next year Jan reference_grid: "/esarchive/exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/r1i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_dcppA-hindcast_s2008-r1i1p2f1_gn_200901-201812.nc" @@ -145,7 +145,7 @@ archive: daily_mean: grid: version: - calendar: + calendar: "365_day" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1, r9i1p1f1, r10i1p1f1, r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1,r17i1p1f1,r18i1p1f1, r19i1p1f1, r20i1p1f1,r21i1p1f1,r22i1p1f1,r23i1p1f1,r24i1p1f1,r25i1p1f1,r26i1p1f1,r27i1p1f1,r28i1p1f1, r29i1p1f1, r30i1p1f1, r31i1p1f1,r32i1p1f1,r33i1p1f1,r34i1p1f1,r35i1p1f1,r36i1p1f1,r37i1p1f1,r38i1p1f1, r39i1p1f1, r40i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/ncar/cesm-dple-dcppA-hindcast/cmorfiles/DCPP/NCAR/CESM1-1-CAM5-CMIP5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_CESM1-1-CAM5-CMIP5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -163,7 +163,7 @@ archive: daily_mean: grid: {"pr":"gn", "tasmax":"gn", "tasmin":"gn"} version: {"pr":"v20210909", "tasmax":"v20210909", "tasmin":"v20210909"} - calendar: "365_day" + calendar: "365_day" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20210312/tas_Amon_CMCC-CM2-SR5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -182,7 +182,7 @@ archive: daily_mean: grid: version: - calendar: + calendar: "365_day" member: r1i1p1f1,r2i1p1f1,r3i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/FGOALS-f3-L/DCPP/CAS/FGOALS-f3-L/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20220212/tas_Amon_FGOALS-f3-L_dcppA-hindcast_s1960-r1i1p1f1_gr_196011-197012.nc" @@ -199,7 +199,7 @@ archive: daily_mean: grid: {"pr":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} version: {"pr":"v20200108", "tas":"v20200504", "tasmax":"v20200504", "tasmin":"v20200504"} - calendar: "gregorian" + calendar: "gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 1 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/IPSL-CM6A-LR/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20200504/tas_Amon_IPSL-CM6A-LR_dcppA-hindcast_s2008-r1i1p1f1_gr_200901-201812.nc" @@ -216,7 +216,7 @@ archive: daily_mean: grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} version: {"pr":"v20191217", "tas":"v20200416", "tasmax":"v20200416", "tasmin":"v20200416"} - calendar: "standard" + calendar: "standard" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MIROC6/DCPP/MIROC/MIROC6/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200417/tas_Amon_MIROC6_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -233,7 +233,7 @@ archive: daily_mean: grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} version: {"pr":"v20210128", "tas":"v20200320", "tasmax":"v20200101", "tasmin":"v20200101"} - calendar: "standard" + calendar: "standard" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -250,7 +250,7 @@ archive: daily_mean: grid: version: - calendar: + calendar: "proleptic_gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1,r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-LR/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MPI-ESM1-2-LR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -267,7 +267,7 @@ archive: daily_mean: grid: {"pr":"gn", "tasmax":"gn", "tasmin":"gn"} version: {"pr":"v20210122", "tasmax":"v20210122", "tasmin":"v20210122"} - calendar: "proleptic_gregorian" + calendar: "proleptic_gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MRI-ESM2-0/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MRI-ESM2-0_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201312.nc" @@ -285,7 +285,7 @@ archive: daily_mean: grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} version: {"pr":"v20191005", "tas":"v20200320", "tasmax":"v20191005", "tasmin":"v20191005"} - calendar: "noleap" + calendar: "noleap" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 10 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_NorCPM1_dcppA-hindcast_s2008-r1i1p1f1_gn_200810-201812.nc" @@ -302,7 +302,7 @@ archive: daily_mean: grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} version: {"pr":"v20191005", "tas":"v20200101", "tasmax":"v20191005", "tasmin":"v20191005"} - calendar: "noleap" + calendar: "noleap" member: r1i2p1f1,r2i2p1f1,r3i2p1f1,r4i2p1f1,r5i2p1f1,r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 initial_month: 10 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" @@ -315,8 +315,8 @@ archive: src: "obs/noaa/ghcn_v4/" monthly_mean: {"tas":"", "tasanomaly":""} daily_mean: + calendar: "gregorian" reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" - # ---- ERA5: src: "recon/ecmwf/era5/" @@ -324,7 +324,7 @@ archive: daily_mean: {"tas":"_f1h-r1440x721cds/", "rsds":"_f1h-r1440x721cds/", "prlr":"_f1h-r1440x721cds/", "sfcWind":"_f1h-r1440x721cds/", "tos":"_f1h-r1440x721cds"} - calendar: "proleptic_gregorian" + calendar: "gregorian" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" # ---- @@ -344,6 +344,7 @@ archive: src: "obs/noaa-nasa/ghcnersstgiss/" monthly_mean: {"tasanomaly":""} daily_mean: + calendar: "standard" reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly_200811.nc" # ---- @@ -351,6 +352,7 @@ archive: src: "obs/ukmo/hadcrut_v4.6/" monthly_mean: {"tasanomaly":""} daily_mean: + calendar: "standard" reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" # ---- @@ -358,5 +360,6 @@ archive: src: "obs/ukmo/hadslp_v2/" monthly_mean: {"psl":""} daily_mean: + calendar: "proleptic_gregorian" reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl_200811.nc" diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R index 8f0c4626db0cb59456b9591d271547758875e8d5..aaf4d91692445ceec16baf5c8af19ec5d3e1057b 100644 --- a/modules/Loading/Loading.R +++ b/modules/Loading/Loading.R @@ -289,9 +289,9 @@ load_datasets <- function(recipe_file) { dim(obs) <- default_dims # Convert obs to s2dv_cube - - # Check for consistency between hcst and obs grid obs <- as.s2dv_cube(obs) + + # Check for consistency between hcst and obs grid if (!(recipe$Analysis$Regrid$type == 'none')) { if (!identical(as.vector(hcst$lat), as.vector(obs$lat))) { stop("hcst and obs don't share the same latitude.") @@ -301,6 +301,8 @@ load_datasets <- function(recipe_file) { } } + + # Print a summary of the loaded data for the user, for each object data_summary(hcst, store.freq) data_summary(obs, store.freq) diff --git a/modules/Loading/Loading_decadal.R b/modules/Loading/Loading_decadal.R index f188c198810478f136f9e9652c165b04fb50903b..7f99ac68b130a9fa2cd1073a645feac4872531ab 100644 --- a/modules/Loading/Loading_decadal.R +++ b/modules/Loading/Loading_decadal.R @@ -54,7 +54,7 @@ load_datasets <- function(recipe_file) { ftimemax = as.numeric(recipe$Analysis$Time$ftime_max), initial_month = archive$System[[exp.name]]$initial_month, sdates = sdates_hcst, - calendar = archive$System[[exp.name]][[store.freq]]$calendar) + calendar = archive$System[[exp.name]]$calendar) } #NOTE: May be used in the future diff --git a/modules/Saving/Saving.R b/modules/Saving/Saving.R index 03288a8e053d01ff72557cf90e114c5de3c088c7..a301a1e6eb8b7c1d96427d7a92404343efd8b562 100644 --- a/modules/Saving/Saving.R +++ b/modules/Saving/Saving.R @@ -2,18 +2,29 @@ source("modules/Saving/paths2save.R") -save_data <- function(recipe, data, +save_data <- function(recipe, archive, data, calibrated_data = NULL, skill_metrics = NULL, - probabilities = NULL, - archive = NULL) { + probabilities = NULL) { # Wrapper for the saving functions. # recipe: The auto-s2s recipe + # archive: The auto-s2s archive # data: output of load_datasets() # calibrated_data: output of calibrate_datasets() # skill_metrics: output of compute_skill_metrics() # probabilities: output of compute_probabilities() + if (is.null(recipe)) { + stop("The 'recipe' parameter is mandatory.") + } + if (is.null(archive)) { + stop("The 'archive' parameter is mandatory.") + } + if (is.null(data)) { + stop("The 'data' parameter is mandatory. It should be the output of", + "load_datasets().") + } + dict <- read_yaml("conf/variable-dictionary.yml") # Create output directory @@ -75,10 +86,10 @@ get_global_attributes <- function(recipe) { return(attrs) } -get_times <- function(store.freq, fcst.horizon, leadtimes, sdate) { +get_times <- function(store.freq, fcst.horizon, leadtimes, sdate, calendar) { # Generates time dimensions and the corresponding metadata. ## TODO: Add calendar - ## TODO: Subseasonal and decadal + ## TODO: Subseasonal switch(fcst.horizon, "seasonal" = {time <- leadtimes; ref <- 'hours since '; @@ -90,7 +101,8 @@ get_times <- function(store.freq, fcst.horizon, leadtimes, sdate) { dim(time) <- length(time) sdate <- as.Date(sdate, format = '%Y%m%d') # reformatting - metadata <- list(time = list(units = paste0(ref, sdate, 'T00:00:00'))) + metadata <- list(time = list(units = paste0(ref, sdate, 'T00:00:00'), + calendar = calendar)) attr(time, 'variables') <- metadata names(dim(time)) <- 'time' @@ -145,10 +157,16 @@ save_forecast <- function(data_cube, global_attributes <- get_global_attributes(recipe) fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq - + calendar <- archive$System[[global_attributes$system]]$calendar +# if (fcst.horizon == "seasonal") { +# calendar <- attr(data_cube$Variable, "variable")$dim$time$calendar +# } else { +# calendar <- attr(data_cube$Variable, "variable")$dim[[3]]$calendar +# } # Generate vector containing leadtimes - dates <- ClimProjDiags::Subset(as.Date(data_cube$Dates$start), 'syear', 1) + dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1), + cal = calendar) if (fcst.horizon == 'decadal') { # Method 1: Use the first date as init_date. But it may be better to use the real initialized date (ask users) # init_date <- as.Date(data_cube$Dates$start[1], format = '%Y%m%d') @@ -156,25 +174,27 @@ save_forecast <- function(data_cube, init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month if (type == 'hcst') { #PROBLEM for fcst!!!!!!!!!!!! - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, '-', - sprintf('%02d', init_month), '-01')) + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', + sprintf('%02d', init_month), '-01'), + cal = calendar) } else if (type == 'fcst') { - init_date <- as.Date(paste0(recipe$Analysis$Time$fcst_year[1], '-', - sprintf('%02d', init_month), '-01')) + init_date <- as.PCICt(paste0(recipe$Analysis$Time$fcst_year[1], '-', + sprintf('%02d', init_month), '-01'), + cal = calendar) } } else { if (type == 'hcst') { - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, - recipe$Analysis$Time$sdate), - format = '%Y%m%d') + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, + recipe$Analysis$Time$sdate), + format = '%Y%m%d', cal = calendar) } else if (type == 'fcst') { - init_date <- as.Date(paste0(recipe$Analysis$Time$fcst_year, - recipe$Analysis$Time$sdate), - format = '%Y%m%d') + init_date <- as.PCICt(paste0(recipe$Analysis$Time$fcst_year, + recipe$Analysis$Time$sdate), + format = '%Y%m%d', cal = calendar) } } - # Get time difference in months - leadtimes <- interval(init_date, dates) %/% hours(1) + # Get time difference in hours + leadtimes <- as.numeric(dates - init_date)/3600 syears <- seq(1:dim(data_cube$data)['syear'][[1]]) syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1]) # expect dim = [sday = 1, sweek = 1, syear, time] @@ -201,6 +221,7 @@ save_forecast <- function(data_cube, } else { dims <- c(lalo, 'ensemble', 'time') var.expname <- variable + var.sdname <- var.sdname var.units <- attr(data_cube$Variable, 'variable')$units } @@ -211,7 +232,6 @@ save_forecast <- function(data_cube, attr(fcst[[1]], 'variables') <- metadata names(dim(fcst[[1]])) <- dims # Add global attributes - global_attributes <- get_global_attributes(recipe) attr(fcst[[1]], 'global_attrs') <- global_attributes # Select start date @@ -220,6 +240,7 @@ save_forecast <- function(data_cube, # fcst.sdate <- format(as.Date(data_cube$Dates$start[i]), '%Y%m%d') # init_date is like "1990-11-01" + init_date <- as.POSIXct(init_date) fcst.sdate <- init_date + lubridate::years(syears_val[i] - lubridate::year(init_date)) fcst.sdate <- format(fcst.sdate, '%Y%m%d') @@ -228,7 +249,7 @@ save_forecast <- function(data_cube, } # Get time dimension values and metadata - times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate) + times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar) time <- times$time # Generate name of output file @@ -274,25 +295,28 @@ save_observations <- function(data_cube, global_attributes <- get_global_attributes(recipe) fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq + calendar <- archive$Reference[[global_attributes$reference]]$calendar # Generate vector containing leadtimes ## TODO: Move to a separate function? - dates <- ClimProjDiags::Subset(as.Date(data_cube$Dates$start), 'syear', 1) + dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1), + cal = calendar) if (fcst.horizon == 'decadal') { # Method 1: Use the first date as init_date. But it may be better to use the real initialized date (ask users) # init_date <- as.Date(data_cube$Dates$start[1], format = '%Y%m%d') # Method 2: use initial month init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, '-', - sprintf('%02d', init_month), '-01')) + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', + sprintf('%02d', init_month), '-01'), + cal = calendar) } else { - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, - recipe$Analysis$Time$sdate), - format = '%Y%m%d') + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, + recipe$Analysis$Time$sdate), + format = '%Y%m%d', cal = calendar) } - # Get time difference in months - leadtimes <- interval(init_date, dates) %/% hours(1) + # Get time difference in hours + leadtimes <- as.numeric(dates - init_date)/3600 syears <- seq(1:dim(data_cube$data)['syear'][[1]]) syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1]) # expect dim = [sday = 1, sweek = 1, syear, time] @@ -329,7 +353,6 @@ save_observations <- function(data_cube, attr(fcst[[1]], 'variables') <- metadata names(dim(fcst[[1]])) <- dims # Add global attributes - global_attributes <- get_global_attributes(recipe) attr(fcst[[1]], 'global_attrs') <- global_attributes # Select start date. The date is computed for each year, and adapted for @@ -339,6 +362,7 @@ save_observations <- function(data_cube, ## cases, different approaches are necessary. if (fcst.horizon == 'decadal') { # init_date is like "1990-11-01" + init_date <- as.POSIXct(init_date) fcst.sdate <- init_date + lubridate::years(syears_val[i] - lubridate::year(init_date)) } else { @@ -351,6 +375,7 @@ save_observations <- function(data_cube, } # Ensure the year is correct if the first leadtime goes to the next year + init_date <- as.POSIXct(init_date) if (lubridate::month(fcst.sdate) < lubridate::month(init_date)) { lubridate::year(fcst.sdate) <- lubridate::year(fcst.sdate) + 1 } @@ -359,7 +384,7 @@ save_observations <- function(data_cube, fcst.sdate <- format(fcst.sdate, format = '%Y%m%d') # Get time dimension values and metadata - times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate) + times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar) time <- times$time # Generate name of output file @@ -438,20 +463,25 @@ save_metrics <- function(skill, # Time indices and metadata fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq + calendar <- archive$System[[global_attributes$system]]$calendar + # Generate vector containing leadtimes - dates <- ClimProjDiags::Subset(as.Date(data_cube$Dates$start), 'syear', 1) + dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1), + cal = calendar) + if (fcst.horizon == 'decadal') { init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, '-', - sprintf('%02d', init_month), '-01')) + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', + sprintf('%02d', init_month), '-01'), + cal = calendar) } else { - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, - recipe$Analysis$Time$sdate), - format = '%Y%m%d') + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, + recipe$Analysis$Time$sdate), + format = '%Y%m%d', cal = calendar) } - # Get time difference in months - leadtimes <- interval(init_date, dates) %/% hours(1) + # Get time difference in hours + leadtimes <- as.numeric(dates - init_date)/3600 # Select start date # If a fcst is provided, use that as the ref. year. Otherwise use 1970. @@ -472,7 +502,7 @@ save_metrics <- function(skill, } } - times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate) + times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar) time <- times$time # Generate name of output file @@ -541,20 +571,24 @@ save_corr <- function(skill, # Time indices and metadata fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq + calendar <- archive$System[[global_attributes$system]]$calendar + # Generate vector containing leadtimes - dates <- ClimProjDiags::Subset(as.Date(data_cube$Dates$start), 'syear', 1) + dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1), + cal = calendar) if (fcst.horizon == 'decadal') { init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, '-', - sprintf('%02d', init_month), '-01')) + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', + sprintf('%02d', init_month), '-01'), + cal = calendar) } else { - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, - recipe$Analysis$Time$sdate), - format = '%Y%m%d') + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, + recipe$Analysis$Time$sdate), + format = '%Y%m%d', cal = calendar) } - # Get time difference in months - leadtimes <- interval(init_date, dates) %/% hours(1) + # Get time difference in hours + leadtimes <- as.numeric(dates - init_date)/3600 # Select start date # If a fcst is provided, use that as the ref. year. Otherwise use 1970. @@ -575,7 +609,7 @@ save_corr <- function(skill, } } - times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate) + times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar) time <- times$time # Generate name of output file @@ -637,20 +671,23 @@ save_percentiles <- function(percentiles, # Time indices and metadata fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq + calendar <- archive$System[[global_attributes$system]]$calendar # Generate vector containing leadtimes - dates <- ClimProjDiags::Subset(as.Date(data_cube$Dates$start), 'syear', 1) + dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1), + cal = calendar) if (fcst.horizon == 'decadal') { init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, '-', - sprintf('%02d', init_month), '-01')) + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', + sprintf('%02d', init_month), '-01'), + cal = calendar) } else { - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, - recipe$Analysis$Time$sdate), - format = '%Y%m%d') + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, + recipe$Analysis$Time$sdate), + format = '%Y%m%d', cal = calendar) } # Get time difference in hours - leadtimes <- interval(init_date, dates) %/% hours(1) + leadtimes <- as.numeric(dates - init_date)/3600 # Select start date # If a fcst is provided, use that as the ref. year. Otherwise use 1970. @@ -671,7 +708,7 @@ save_percentiles <- function(percentiles, } } - times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate) + times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar) time <- times$time # Generate name of output file @@ -717,22 +754,25 @@ save_probabilities <- function(probs, global_attributes <- get_global_attributes(recipe) fcst.horizon <- tolower(recipe$Analysis$Horizon) store.freq <- recipe$Analysis$Variables$freq - + calendar <- archive$System[[global_attributes$system]]$calendar + # Generate vector containing leadtimes ## TODO: Move to a separate function? - dates <- ClimProjDiags::Subset(as.Date(data_cube$Dates$start), 'syear', 1) + dates <- as.PCICt(ClimProjDiags::Subset(data_cube$Dates$start, 'syear', 1), + cal = calendar) if (fcst.horizon == 'decadal') { init_month <- archive$System[[recipe$Analysis$Datasets$System$name]]$initial_month - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, '-', - sprintf('%02d', init_month), '-01')) + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, '-', + sprintf('%02d', init_month), '-01'), + cal = calendar) } else { - init_date <- as.Date(paste0(recipe$Analysis$Time$hcst_start, - recipe$Analysis$Time$sdate), - format = '%Y%m%d') + init_date <- as.PCICt(paste0(recipe$Analysis$Time$hcst_start, + recipe$Analysis$Time$sdate), + format = '%Y%m%d', cal = calendar) } # Get time difference in hours - leadtimes <- interval(init_date, dates) %/% hours(1) + leadtimes <- as.numeric(dates - init_date)/3600 syears <- seq(1:dim(data_cube$data)['syear'][[1]]) syears_val <- lubridate::year(data_cube$Dates$start[1, 1, , 1]) # expect dim = [sday = 1, sweek = 1, syear, time] @@ -762,12 +802,12 @@ save_probabilities <- function(probs, } # Add global attributes - global_attributes <- get_global_attributes(recipe) attr(probs_syear[[1]], 'global_attrs') <- global_attributes # Select start date if (fcst.horizon == 'decadal') { # init_date is like "1990-11-01" + init_date <- as.POSIXct(init_date) fcst.sdate <- init_date + lubridate::years(syears_val[i] - lubridate::year(init_date)) fcst.sdate <- format(fcst.sdate, '%Y%m%d') } else { @@ -775,7 +815,7 @@ save_probabilities <- function(probs, } # Get time dimension values and metadata - times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate) + times <- get_times(store.freq, fcst.horizon, leadtimes, fcst.sdate, calendar) time <- times$time # Generate name of output file diff --git a/modules/test_decadal.R b/modules/test_decadal.R index 7923c0286a91a26f6fcc7fca6bcd8f2d3a489550..8077b7e200142a638568e5e30ff60ac71d209407 100644 --- a/modules/test_decadal.R +++ b/modules/test_decadal.R @@ -17,4 +17,4 @@ skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe) # Compute percentiles and probability bins probabilities <- compute_probabilities(calibrated_data$hcst, recipe) # Export all data to netCDF -save_data(recipe, data, calibrated_data, skill_metrics, probabilities, archive = archive) +save_data(recipe, archive, data, calibrated_data, skill_metrics, probabilities) diff --git a/modules/test_victoria.R b/modules/test_victoria.R index df425659132de063cf2403537253932cdac29d76..eb5933b3547a7d7e2ff10e8d251aee0bed0a538f 100644 --- a/modules/test_victoria.R +++ b/modules/test_victoria.R @@ -4,8 +4,9 @@ source("modules/Calibration/Calibration.R") source("modules/Skill/Skill.R") source("modules/Saving/Saving.R") -recipe_file <- "modules/Loading/testing_recipes/recipe_era5land.yml" +recipe_file <- "modules/Loading/testing_recipes/recipe_4.yml" recipe <- read_yaml(recipe_file) +archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive # Load datasets data <- load_datasets(recipe_file) @@ -16,4 +17,4 @@ skill_metrics <- compute_skill_metrics(calibrated_data$hcst, data$obs, recipe) # Compute percentiles and probability bins probabilities <- compute_probabilities(calibrated_data$hcst, recipe) # Export all data to netCDF -save_data(recipe, data, calibrated_data, skill_metrics, probabilities) +save_data(recipe, archive, data, calibrated_data, skill_metrics, probabilities) diff --git a/tools/libs.R b/tools/libs.R index 248ada450574cfbcaf6615068989e6884b0d0e46..2b29835927f2443e7f5e8f67cacc345db0436633 100644 --- a/tools/libs.R +++ b/tools/libs.R @@ -11,6 +11,7 @@ library(abind) library(easyNCDF) library(CSTools) library(lubridate) +library(PCICt) # # library(parallel) # library(pryr) # To check mem usage.