diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml new file mode 100644 index 0000000000000000000000000000000000000000..e41d46a035c8f4dd30da56544ca62c3343792e5c --- /dev/null +++ b/conf/archive_decadal.yml @@ -0,0 +1,349 @@ +archive: + src: "/esarchive/" + System: +# ---- + EC-Earth3-i1: + src: + hcst: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + fcst: + monthly_mean: + #NOTE: tos is under both Amon and Omon --> wait to be changed + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} + grid: {"tas":"gr"} + version: {"tas":"v20190713"} + daily_mean: + grid: {"clt":"gr", "hurs":"gr", "hursmin":"gr", "pr":"gr", "psl":"gr", "rsds":"gr", "sfcWind":"gr", "sfcWindmax":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} + version: {"clt":"v20190713", "hurs":"v20190713", "hursmin":"v20190713", "pr":"v20190713", "psl":"v20190713", "rsds":"v20190713", "sfcWind":"v20190713", "sfcWindmax":"v20190713", "tas":"v20190713", "tasmax":"v20190713", "tasmin":"v20190713"} + calendar: "proleptic_gregorian" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20190713/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196011-196110.nc" #'r512x256' + +# ---- + #NOTE: EC-Earth3-i2 the first file of each sdate has 2 time step only (Nov-Dec). + # The rest files are Jan to Dec. + EC-Earth3-i2: + src: + hcst: "exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon"} + grid: {"tas":"gr"} + version: {"tas":"v20200730"} + daily_mean: + grid: {"pr":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} + version: {"pr":"v20200508", "tas":"v20200731", "tasmax":"v20200730", "tasmin":"v20200730"} + calendar: "proleptic_gregorian" + #NOTE:There are many members but not all of them are available on ESGF (only r6-10 available). Then, we might have some variables for the rest of the members (r1-5 and r11-15), but not for all the variables. That's why i'm only using r6-10 + member: r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r6i2p1f1/Amon/tas/gr/v20200730/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r6i2p1f1_gr_196011-196012.nc" #'r512x256' + +# ---- + EC-Earth3-i4: + src: + hcst: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + fcst: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppB-forecast/" +# src: {"1960:2020": "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/", +# "2021:2021": "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppB-forecast/"} + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "clt":"Amon", "hfls":"Amon", + "hurs":"Amon", "huss":"Amon", "rsds":"Amon", "rsut":"Amon", "ta":"Amon", + "tasmax":"Amon", "tosa":"Amon", "ua":"Amon", "va":"Amon", "zg":"Amon", + "evspsbl":"Amon", "hfss":"Amon", "hursmin":"Amon", "rlut":"Amon", + "rsdt":"Amon", "sfcWind":"Amon", "tasmin":"Amon", "ts":"Amon", "uas":"Amon", + "vas":"Amon"} + grid: {"tas":"gr", "pr":"gr", "psl":"gr", "clt":"gr", "hfls":"gr", + "hurs":"gr", "huss":"gr", "rsds":"gr", "rsut":"gr", "ta":"gr", + "tasmax":"gr", "tosa":"gr", "ua":"gr", "va":"gr", "zg":"gr", + "evspsbl":"gr", "hfss":"gr", "hursmin":"gr", "rlut":"gr", + "rsdt":"gr", "sfcWind":"gr", "tasmin":"gr", "ts":"gr", "uas":"gr", + "vas":"gr"} + version: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910", "clt":"v20210910", + "hurs":"v20210910", "huss":"v20210910", "rsds":"v20210910", "rsut":"v20210910", "ta":"v20210910", + "tasmax":"v20210910", "tosa":"v20210910", "ua":"v20210910", "va":"v20210910", "zg":"v20210910", + "evspsbl":"v20210910", "hfss":"v20210910", "hursmin":"v20210910", "rlut":"v20210910", + "rsdt":"v20210910", "sfcWind":"v20210910", "tasmin":"v20210910", "ts":"v20210910", "uas":"v20210910", + "vas":"v20210910"} + daily_mean: + grid: {"tas":"gr", "pr":"gr", "psl":"gr"} + version: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910"} + calendar: "proleptic_gregorian" + member: r1i4p1f1,r2i4p1f1,r3i4p1f1,r4i4p1f1,r5i4p1f1,r6i4p1f1,r7i4p1f1,r8i4p1f1,r9i4p1f1,r10i4p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i4p1f1/Amon/tas/gr/v20210910/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i4p1f1_gr_196011-196110.nc" + +# ---- + HadGEM3-GC31-MM: + src: + hcst: "exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" + fcst: "exp/CMIP6/dcppB-forecast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppB-forecast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon", "tos":"Omon"} + grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr", "tos":"gr"} + #version depends on member and variable + version: {"tas":"v20200316", "psl":"v20200316", "pr":"v20200316", "ts":"v20200316", "tos":"v20200417"} + daily_mean: + grid: {"tas":"gn"} + version: {"tasmin":"v20200101", "tasmax":"v20200101", "pr":"v20200417"} + calendar: "360-day" + member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Amon/tas/gr/v20200316/tas_Amon_HadGEM3_dcppA-hindcast_s2018-r1i1p1f2_gr_201811-202903.nc" #'r432x324' + +# ---- + BCC-CSM2-MR: + src: + hcst: "exp/CMIP6/dcppA-hindcast/BCC-CSM2-MR/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn"} + #NOTE: It should be the lastest version with all the files inside + version: {"tas":"v20200101"} +# version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} + daily_mean: + grid: {"pr":"gn", "sfcWind":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn", "uas":"gn", "vas":"gn"} + version: {"pr":"v20200114", "sfcWind":"v20200101", "tas":"v20200408", "tasmax":"v20200114", "tasmin":"v20200114", "uas":"v20200114", "vas":"v20200114"} + calendar: "365_day" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 + initial_month: 1 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/BCC-CSM2-MR/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r8i1p1f1/Amon/tas/gn/v20200101/tas_Amon_BCC-CSM2-MR_dcppA-hindcast_s2008-r8i1p1f1_gn_200801-201712.nc" + +# ---- + CanESM5: + src: + hcst: "exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/" + fcst: "exp/canesm5/cmip6-dcppB-forecast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppB-forecast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} + + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} + version: {"tas":"v20190429", "pr":"v20190429", "psl":"v20190429", "tasmin":"v20190429", "tasmax":"v20190429"} + daily_mean: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20190429", "tas":"v20190429", "tasmax":"v20190429", "tasmin":"v20190429"} + calendar: "365_day" + member: r1i1p2f1,r2i1p2f1,r3i1p2f1,r4i1p2f1,r5i1p2f1,r6i1p2f1,r7i1p2f1,r8i1p2f1, r9i1p2f1, r10i1p2f1, r11i1p2f1,r12i1p2f1,r13i1p2f1,r14i1p2f1,r15i1p2f1,r16i1p2f1,r17i1p2f1,r18i1p2f1, r19i1p2f1, r20i1p2f1,r21i1p2f1,r22i1p2f1,r23i1p2f1,r24i1p2f1,r25i1p2f1,r26i1p2f1,r27i1p2f1,r28i1p2f1, r29i1p2f1, r30i1p2f1, r31i1p2f1,r32i1p2f1,r33i1p2f1,r34i1p2f1,r35i1p2f1,r36i1p2f1,r37i1p2f1,r38i1p2f1, r39i1p2f1, r40i1p2f1 + initial_month: 13 #next year Jan + reference_grid: "/esarchive/exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/r1i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_dcppA-hindcast_s2008-r1i1p2f1_gn_200901-201812.nc" + +# ---- + CESM1-1-CAM5-CMIP5: + src: + hcst: "exp/ncar/cesm-dple-dcppA-hindcast/cmorfiles/DCPP/NCAR/CESM1-1-CAM5-CMIP5/dcppA-hindcast" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon"} + grid: {"tas":"gn", "pr":"gn"} + version: {"tas":"v20200101", "pr":"v20200101"} + daily_mean: + grid: + version: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1, r9i1p1f1, r10i1p1f1, r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1,r17i1p1f1,r18i1p1f1, r19i1p1f1, r20i1p1f1,r21i1p1f1,r22i1p1f1,r23i1p1f1,r24i1p1f1,r25i1p1f1,r26i1p1f1,r27i1p1f1,r28i1p1f1, r29i1p1f1, r30i1p1f1, r31i1p1f1,r32i1p1f1,r33i1p1f1,r34i1p1f1,r35i1p1f1,r36i1p1f1,r37i1p1f1,r38i1p1f1, r39i1p1f1, r40i1p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/ncar/cesm-dple-dcppA-hindcast/cmorfiles/DCPP/NCAR/CESM1-1-CAM5-CMIP5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_CESM1-1-CAM5-CMIP5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + +# ---- + CMCC-CM2-SR5: + src: + hcst: "exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/" + fcst: "exp/CMIP6/dcppB-forecast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppB-forecast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "prc":"Amon", "ts":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "prc":"gn", "ts":"gn"} + version: {"tas":"v20210312", "pr":"v20210312", "psl":"v20210312", "prc":"v20200101", "ts":"v20200101"} + daily_mean: + grid: {"pr":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20210909", "tasmax":"v20210909", "tasmin":"v20210909"} + calendar: "365_day" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20210312/tas_Amon_CMCC-CM2-SR5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + +# ---- +#QUESTION: missing in spreadsheet + FGOALS-f3-L: + src: + monthly_mean: + table: + grid: + version: + daily_mean: + grid: + version: + calendar: + member: + initial_month: + reference_grid: + +# ---- + IPSL-CM6A-LR: + src: + hcst: "exp/CMIP6/dcppA-hindcast/IPSL-CM6A-LR/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "sfcWind":"Amon"} + grid: {"tas":"gr", "pr":"gr", "psl":"gr", "sfcWind":"gr"} + version: {"tas":"v20200504", "pr":"v20200504", "psl":"v20200504", "sfcWind":"v20200504"} + daily_mean: + grid: {"pr":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} + version: {"pr":"v20200108", "tas":"v20200504", "tasmax":"v20200504", "tasmin":"v20200504"} + calendar: "gregorian" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 1 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/IPSL-CM6A-LR/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20200504/tas_Amon_IPSL-CM6A-LR_dcppA-hindcast_s2008-r1i1p1f1_gr_200901-201812.nc" + +# ---- + MIROC6: + src: + hcst: "exp/CMIP6/dcppA-hindcast/MIROC6/DCPP/MIROC/MIROC6/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} + version: {"tas":"v20200417", "pr":["v20200416","v20200504"], "psl":"v20200504", "tasmin":"v20200417", "tasmax":"v20200504"} + daily_mean: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20191217", "tas":"v20200416", "tasmax":"v20200416", "tasmin":"v20200416"} + calendar: "standard" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MIROC6/DCPP/MIROC/MIROC6/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200417/tas_Amon_MIROC6_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + +# ---- + MPI-ESM1.2-HR: + src: + hcst: "exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} + version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320", "sfcWind":"v20200320"} + daily_mean: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20210128", "tas":"v20200320", "tasmax":"v20200101", "tasmin":"v20200101"} + calendar: "standard" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + +# ---- + MPI-ESM1.2-LR: + src: + hcst: "exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-LR/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "ts":"gn"} + version: {"tas":"v20200101", "pr":"v20200101", "psl":"v20200101", "ts":"v20200101"} + daily_mean: + grid: + version: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1,r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-LR/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MPI-ESM1-2-LR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + +# ---- + MRI-ESM2-0: + src: + hcst: "exp/CMIP6/dcppA-hindcast/MRI-ESM2-0/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn"} + version: {"tas":"v20200101", "pr":"v20200101", "psl":"v20200101"} + daily_mean: + grid: {"pr":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20210122", "tasmax":"v20210122", "tasmin":"v20210122"} + calendar: "proleptic_gregorian" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MRI-ESM2-0/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MRI-ESM2-0_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201312.nc" + +# ---- + #NOTE: NorCPM1-i1 and i2 are under the same directory + NorCPM1-i1: + src: + hcst: "exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/" + fcst: + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn"} + version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320"} + daily_mean: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20191005", "tas":"v20200320", "tasmax":"v20191005", "tasmin":"v20191005"} + calendar: "noleap" + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 10 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_NorCPM1_dcppA-hindcast_s2008-r1i1p1f1_gn_200810-201812.nc" + +# ---- + NorCPM1-i2: + src: "exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/" + monthly_mean: + table: {"pr":"Amon", "psl":"Amon"} + grid: {"pr":"gn", "psl":"gn"} + version: {"pr":"v20200101", "psl":"v20200101"} + daily_mean: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20191005", "tas":"v20200101", "tasmax":"v20191005", "tasmin":"v20191005"} + calendar: "noleap" + member: r1i2p1f1,r2i2p1f1,r3i2p1f1,r4i2p1f1,r5i2p1f1,r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 + initial_month: 10 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" + + +# ===================================== + + Reference: + GHCNv4: + src: "obs/noaa/ghcn_v4/" + monthly_mean: {"tas":"", "tasanomaly":""} + daily_mean: + reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" + +# ---- + ERA5: + src: "recon/ecmwf/era5/" + monthly_mean: {"tas":"_f1h-r1440x721cds", "prlr":"_f1h-r1440x721cds", "psl":"_f1h-r1440x721cds"} + daily_mean: {"tas":"_f1h-r1440x721cds/", "rsds":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", "sfcWind":"_f1h-r1440x721cds/"} + calendar: "360-day" + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" + +# ---- +# missing info in script + NCEP1: + +# ---- + JRA-55: + src: "recon/jma/jra55/" + monthly_mean: {"tas":"_f6h", "psl":"_f6h", "tos":"", "pr":"_s0-3h", "prlr":"_s0-3h"} + daily_mean: {"tas":"_f6h", "psl":"_f6h", "prlr":"_s0-3h", "sfcWind":"_f6h"} + reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" + +# ---- + GISTEMPv4: + src: "obs/noaa-nasa/ghcnersstgiss/" + monthly_mean: {"tasanomaly":""} + daily_mean: + reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly_200811.nc" + +# ---- + HadCRUT4: + src: "obs/ukmo/hadcrut_v4.6/" + monthly_mean: {"tasanomaly":""} + daily_mean: + reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + HadSLP2: + src: "obs/ukmo/hadslp_v2/" + monthly_mean: {"psl":""} + daily_mean: + reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl_200811.nc" + diff --git a/modules/Loading/Loading.R b/modules/Loading/Loading.R old mode 100755 new mode 100644 diff --git a/modules/data_load/Loading_decadal.R b/modules/data_load/Loading_decadal.R new file mode 100644 index 0000000000000000000000000000000000000000..97b18c01fbab373a280a185d5fb6b916ef8c6686 --- /dev/null +++ b/modules/data_load/Loading_decadal.R @@ -0,0 +1,326 @@ +# Loading module: +# 1. archive.yml +# 2. recipe.yml +# 3. Load_decadal.R (V) +setwd('/esarchive/scratch/aho/git/auto-s2s/') + +## TODO: remove paths to personal scratchs +source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") +source("/esarchive/scratch/vagudets/repos/cstools/R/s2dv_cube.R") +# Load required libraries/funs +source("modules/data_load/dates2load.R") +source("tools/libs.R") + +#==================================================================== + +# recipe_file <- "modules/data_load/testing_recipes/recipe_decadal.yml" +# recipe_file <- "modules/data_load/testing_recipes/recipe_decadal_daily.yml" + +load_datasets <- function(recipe_file) { + + recipe <- read_yaml(recipe_file) + recipe$filename <- recipe_file + archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive + + #------------------------- + # Read from recipe: + #------------------------- + exp.name <- recipe$Analysis$Datasets$System$name #'HadGEM3' + ref.name <- recipe$Analysis$Datasets$Reference$name #'era5' + member <- strsplit(recipe$Analysis$Datasets$System$member, ',')[[1]] #c("r1i1p1f2", "r2i1p1f2") + variable <- recipe$Analysis$Variables$name #'tas' + store.freq <- recipe$Analysis$Variables$freq #monthly_mean + lats.min <- as.numeric(recipe$Analysis$Region$latmin) #0 + lats.max <- as.numeric(recipe$Analysis$Region$latmax) #10 + lons.min <- as.numeric(recipe$Analysis$Region$lonmin) #0 + lons.max <- as.numeric(recipe$Analysis$Region$lonmax) #10 + + # change to: sdates <- dates2load(recipe, logger) + sdates_hcst <- as.numeric(recipe$Analysis$Time$hcst_start):as.numeric(recipe$Analysis$Time$hcst_end) #1960:2015 + sdates_fcst <- as.numeric(recipe$Analysis$Time$fcst) + + time <- as.numeric(recipe$Analysis$Time$leadtimemin):as.numeric(recipe$Analysis$Time$leadtimemax) + #TODO: daily data + # If daily data & time is "month", need a function to calculate the indices. + # E.g., time = 1:4 means Dec to March, then indices will be c(31:(31+30+31+28+31)) for non-leap yr and c(31:(31+30+31+28), (31+30+31+28+2):(31+30+31+28+2+30)) for leap yr. + if (store.freq == "daily_mean") { + # for 1993-1994, Dec to March + time <- c(31:(31+31+31+28+30)) + } + +#NOTE: May be used in the future +# season <- recipe$Analysis$Time$season + + #------------------------- + # Read from archive: + #------------------------- + if (store.freq == "monthly_mean") { + table <- archive$System[[exp.name]][[store.freq]]$table[[variable]] #'Amon' + } else { + table <- 'day' + } + grid <- archive$System[[exp.name]][[store.freq]]$grid[[variable]] + version <- archive$System[[exp.name]][[store.freq]]$version[[variable]] + if (identical(member, 'all')) { + member <- strsplit(archive$System[[exp.name]]$member, ',')[[1]] + } + + #------------------------- + # derived from above: + #------------------------- + # Check lat and lon and decide CircularSort + circularsort <- check_latlon(latmin = lats.min, latmax = lats.max, lonmin = lons.min, lonmax = lons.max) + + # generate transform params for system and ref + regrid_params <- get_regrid_params(recipe, archive) + + # Only if the time length in each chunk may differ that we need largest_dims_length to be TRUE. Otherwise, set FALSE to increase efficiency. + need_largest_dims_length <- ifelse(exp.name == 'EC-Earth3-i2', TRUE, FALSE) + + #time for daily data. If recipe asks for time step, we don't need to calculate here +# fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') + + +#------------------------------------------- +# Step 1: Load the hcst +#------------------------------------------- + #monthly and daily + hcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$hcst, + '$ensemble$', table, '$var$', grid, version) + hcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') + + # monthly & daily + hcst <- Start(dat = file.path(hcst.path, hcst.files), + var = variable, + ensemble = member, + syear = paste0(sdates_hcst), +#split_multiselected_dims = T, + chunk = 'all', + chunk_depends = 'syear', + time = indices(time), + time_across = 'chunk', + merge_across_dims = TRUE, + largest_dims_length = need_largest_dims_length, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + transform = regrid_params$fcst.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$fcst.gridtype, #nc file + method = regrid_params$fcst.gridmethod), + transform_vars = c('latitude', 'longitude'), + synonims = list(longitude = c('lon', 'longitude'), + latitude = c('lat', 'latitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = c('syear', 'chunk')), + retrieve = T) + + # change syear to c(sday, sweek, syear) + dim(hcst) <- c(dim(hcst)[1:3], sday = 1, sweek = 1, dim(hcst)[4:7]) + tmp_time_attr <- attr(hcst, 'Variables')$common$time + if (!identical(dim(tmp_time_attr), dim(hcst)[6:7])) { + stop("hcst has problem in matching data and time attr dimension.") + } + dim(attr(hcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(hcst)[6:7]) + + # Change class from startR_array to s2dv_cube + suppressWarnings( + hcst <- as.s2dv_cube(hcst) + ) + +#------------------------------------------- +# Step 2: Load the fcst +#------------------------------------------- + if (!is.null(recipe$Analysis$Time$fcst)) { + + #monthly and daily + fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$fcst, + '$ensemble$', table, '$var$', grid, version) + fcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') + + # monthly & daily + fcst <- Start(dat = file.path(fcst.path, fcst.files), + var = variable, + ensemble = member, + syear = paste0(sdates_fcst), +#split_multiselected_dims = T, + chunk = 'all', + chunk_depends = 'syear', + time = indices(time), + time_across = 'chunk', + merge_across_dims = TRUE, + largest_dims_length = need_largest_dims_length, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + transform = regrid_params$fcst.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$fcst.gridtype, #nc file + method = regrid_params$fcst.gridmethod), + transform_vars = c('latitude', 'longitude'), + synonims = list(longitude = c('lon', 'longitude'), + latitude = c('lat', 'latitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = c('syear', 'chunk')), + retrieve = T) + + # change syear to c(sday, sweek, syear) + dim(fcst) <- c(dim(fcst)[1:3], sday = 1, sweek = 1, dim(fcst)[4:7]) + tmp_time_attr <- attr(fcst, 'Variables')$common$time + if (!identical(dim(tmp_time_attr), dim(fcst)[6:7])) { + stop("fcst has problem in matching data and time attr dimension.") + } + dim(attr(fcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(fcst)[6:7]) + + # Change class from startR_array to s2dv_cube + suppressWarnings( + fcst <- as.s2dv_cube(fcst) + ) + + if (!identical(dim(hcst$data)[-6], dim(fcst$data)[-6])) { + stop("hcst and fcst do not share the same dimension structure.") + } + + } + +#------------------------------------------- +# Step 3. Load the reference +#------------------------------------------- + obs.path <- file.path(archive$src, archive$Reference[[ref.name]]$src, store.freq, + paste0(variable, archive$Reference[[ref.name]][[store.freq]][[variable]])) + obs.files <- paste0('$var$_$file_date$.nc') + + # Get from startR_cube +# dates <- attr(hcst, 'Variables')$common$time + # Get from s2dv_cube + dates <- hcst$Dates$start + dates_file <- sapply(dates, format, '%Y%m') + dim(dates_file) <- dim(dates) + + if (store.freq == "daily_mean") { +#//////////////// +# Method 1: use hcst time attr as obs time selector +#//////////////// + + # Set hour to 12:00 to ensure correct date retrieval for daily data + lubridate::hour(dates) <- 12 + lubridate::minute(dates) <- 00 + # Restore correct dimensions + dim(dates) <- dim(dates_file) + + obs <- Start(dat = file.path(obs.path, obs.files), + var = variable, + file_date = unique(format(dates, '%Y%m')), + time = dates, # [sday, sweek, syear, time] + time_across = 'file_date', + merge_across_dims = TRUE, + split_multiselected_dims = TRUE, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + transform = regrid_params$obs.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$obs.gridtype, #nc file + method = regrid_params$obs.gridmethod), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = 'file_date'), + retrieve = TRUE) + + } else if (store.freq == "monthly_mean") { +#//////////////// +# Method 2: reshape hcst time attr's date into an array with time dim then as obs date selector +#//////////////// + + obs <- Start(dat = file.path(obs.path, obs.files), + var = variable, + file_date = dates_file, #dates_arr, # [sday, sweek, syear, time] + split_multiselected_dims = TRUE, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + transform = regrid_params$obs.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$obs.gridtype, #nc file + method = regrid_params$obs.gridmethod), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = 'file_date'), + retrieve = TRUE) + } + + +#dim(attr(obs, 'Variables')$common$time) +# sday sweek syear time +# 1 1 2 14 + + +# # TODO: Reorder obs dims to match hcst dims? +# # Adds ensemble dim to obs (for consistency with hcst/fcst) +# default_dims <- c(dat = 1, var = 1, sweek = 1, +# sday = 1, syear = 1, time = 1, +# latitude = 1, longitude = 1, ensemble = 1) +# default_dims[names(dim(obs))] <- dim(obs) +# dim(obs) <- default_dims + + if (!identical(dim(obs), dim(hcst$data)[-3])) { + stop("obs and hcst dimensions do not match.") + } + # Add ensemble dim to obs + dim(obs) <- c(dim(obs)[1:2], ensemble = 1, dim(obs)[3:8]) + + # Change class from startR_array to s2dv_cube + suppressWarnings( + obs <- as.s2dv_cube(obs) + ) + + +#------------------------------------------- +# Step 4. Print the summary of data +#------------------------------------------- + + # Print a summary of the loaded data for the user, for each object + data_summary(hcst, store.freq) + data_summary(obs, store.freq) + if (!is.null(fcst)) { + data_summary(fcst, store.freq) + } + + +#------------------------------------------- +# Step 5. Verify the consistance btwn hcst and obs +#------------------------------------------- + # dimension + if (any(!names(dim(obs$data)) %in% names(dim(hcst$data)))) { + stop("hcst and obs don't share the same dimension names.") + } else { + ens_ind <- which(names(dim(obs$data)) == 'ensemble') + match_ind <- match(names(dim(obs$data))[-ens_ind], names(dim(hcst$data))) + if (!all(dim(hcst$data)[match_ind] == dim(obs$data)[-ens_ind])) stop("hcst and obs don't share the same dimension length.") + } + + # time attribute + if (!identical(format(hcst$Dates$start, '%Y%m'), + format(obs$Dates$start, '%Y%m'))) + stop("hcst and obs don't share the same time.") + + # lat and lon attributes + if (!identical(as.vector(hcst$lat), + as.vector(obs$lat))) + stop("hcst and obs don't share the same latitude.") + if (!identical(as.vector(hcst$lon), + as.vector(obs$lon))) + stop("hcst and obs don't share the same longitude.") + + + + return(list(hcst = hcst, fcst = fcst, obs = obs)) +} diff --git a/modules/data_load/testing_recipes/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml new file mode 100644 index 0000000000000000000000000000000000000000..b7b6a10978c4e2f58cfc14bbfd9be04b7522ff8d --- /dev/null +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -0,0 +1,46 @@ +Description: + Author: An-Chi Ho + '': split version +Analysis: + Horizon: Decadal + Variables: + name: tas + freq: monthly_mean + Datasets: + System: + name: EC-Earth3-i4 #CanESM5 + member: 'all' #r1i1p1f1,r2i1p1f1,r3i1p1f1 + Multimodel: no + Reference: + name: ERA5 #JRA-55 + Time: + sdate: + fcst: 2021 +# fcst_sday: '1101' + hcst_start: 1990 #'1993' + hcst_end: 1992 #'2016' +# season: 'Annual' + leadtimemin: 1 + leadtimemax: 14 + Region: + latmin: 10 #-90 + latmax: 20 #90 + lonmin: 0 + lonmax: 15 #359.9 + Regrid: + method: bilinear + type: to_system #to_reference + Workflow: + Calibration: + method: bias + Skill: + metric: RPSS + Indicators: + index: FALSE + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/aho/git/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/aho/git/auto-s2s/ + diff --git a/modules/data_load/testing_recipes/recipe_decadal_daily.yml b/modules/data_load/testing_recipes/recipe_decadal_daily.yml new file mode 100644 index 0000000000000000000000000000000000000000..b314303595441b473def8d0bc5e869d114099c84 --- /dev/null +++ b/modules/data_load/testing_recipes/recipe_decadal_daily.yml @@ -0,0 +1,46 @@ +Description: + Author: An-Chi Ho + '': split version +Analysis: + Horizon: Decadal + Variables: + name: tas + freq: daily_mean + Datasets: + System: + name: EC-Earth3-i4 #BCC-CSM2-MR #CanESM5 + member: r1i4p1f1,r2i4p1f1,r3i4p1f1 #'all' + Multimodel: no + Reference: + name: ERA5 + Time: + sdate: + fcst: 2021 +# fcst_sday: '1101' + hcst_start: 1993 + hcst_end: 1994 + season: 'Annual' + leadtimemin: 1 + leadtimemax: 4 + Region: + latmin: 10 #-90 + latmax: 20 #90 + lonmin: 0 + lonmax: 15 #359.9 + Regrid: + method: bilinear + type: to_system #to_reference + Workflow: + Calibration: + method: qmap + Skill: + metric: RPSS + Indicators: + index: FALSE + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/aho/git/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/aho/git/auto-s2s/ + diff --git a/tools/check_latlon.R b/tools/check_latlon.R new file mode 100644 index 0000000000000000000000000000000000000000..c317acd24fd4088353557369e64889cce2943ad8 --- /dev/null +++ b/tools/check_latlon.R @@ -0,0 +1,90 @@ +# Check if the lat and lon inputs abide the regulation. +# +# (x) ←|-------------|-------------|-------------|-------------|-------------| →(x) +# -360 -180 0 180 360 540 +# |___________________________|___________________________| +# [0, 360] [0, 360] +# +# |___________________________|___________________________| +# [-180, 180] [-180, 180] +# Premise: lonmin < lonmax +# [Case 1: lonmin >= 0] +# 1-0: lonmax <360 →[0, 360] +# 1-1:lonmin >= 180 & lonmax > 360 →[-180, 180] +# 1-2:lonmin < 180 & lonmax > 360 →[0, 360] + warning* +# +# [Case 2: lonmax < 0] +# 2-0: lonmin >= -180 & lonmax < 180 →[-180, 180] +# 2-1: lonmin < -180 & lonmax < 0 →[0, 360] +# 2-2: lonmin < -180 & lonmax > 0 →[-180, 180] + warning* +# 2-3 lonmax > 180 →[-180, 180] + warning* +# +# *warning: The region is not one integrated box. + +check_latlon <- function(latmin, latmax, lonmin, lonmax) { + + if (any(!sapply(c(latmin, latmax, lonmin, lonmax), is.numeric)) | + any(sapply(c(latmin, latmax, lonmin, lonmax), length) != 1)) + stop("latmin, latmax, lonmin, and lonmax must be a number.") + # lat + if (any(c(latmin, latmax) < -90) | any(c(latmin, latmax) > 90)) + stop("latmin must be within [-90, 90].") + # lon + if (lonmin > lonmax) + stop("lonmax must not be smaller than lonmin.") + # Adjust lon if it is way out of scope + while (lonmin > 360 & lonmax > 360) { + lonmin <- lonmin - 360 + lonmax <- lonmax - 360 + } + while (lonmin < -180 & lonmax < -180) { + lonmin <- lonmin + 360 + lonmax <- lonmax + 360 + } + + warning_discrete_lon <- FALSE + warning_unconsidered_case <- FALSE + ## case 1 + if (lonmin >= 0) { + if (lonmax < 360) { ## 1-0 + circularsort <- CircularSort(0, 360) + } else if (lonmin >= 180 & lonmax > 360) { ## 1-1 + circularsort <- CircularSort(-180, 180) + if (lonmax >= 540) + warning_discrete_lon <- TRUE + } else if (lonmin < 180 & lonmax > 360) { ## 1-2 + circularsort <- CircularSort(0, 360) + warning_discrete_lon <- TRUE + } else { + circularsort <- CircularSort(0, 360) + warning_unconsidered_case <- TRUE + } + + } + ## case 2 + else { + if (lonmin >= -180 & lonmax < 180) { ## 2-0 + circularsort <- CircularSort(-180, 180) + } else if (lonmin < -180 & lonmax < 0) { ## 2-1 + circularsort <- CircularSort(0, 360) + if (lonmin < -360) + warning_discrete_lon <- TRUE + } else if (lonmin < -180 & lonmax > 0) { ## 2-2 + circularsort <- CircularSort(-180, 180) + warning_discrete_lon <- TRUE + } else if (lonmax > 180) { ## 2-3 + circularsort <- CircularSort(-180, 180) + warning_discrete_lon <- TRUE + } else { + circularsort <- CircularSort(-180, 180) + warning_unconsidered_case <- TRUE + } + } + + if (warning_discrete_lon) + warning("The longitude of returned data won't be continuous. Check the metadata to see the longitude range.") + if (warning_unconsidered_case) + warning("The longitude case is not being considered yet. Please report this case to maintainers and check the longitude of returned data.") + + return(circularsort) +}