From c141ca9164d4f21bb4a9968a30a77d125effc61e Mon Sep 17 00:00:00 2001 From: aho Date: Fri, 4 Mar 2022 17:00:07 +0100 Subject: [PATCH 01/21] build the first scripts based on Hadgem3 --- conf/archive_decadal.yml | 49 +++++++++++ modules/data_load/load_decadal.R | 116 +++++++++++++++++++++++++++ modules/data_load/recipe_decadal.yml | 47 +++++++++++ 3 files changed, 212 insertions(+) create mode 100644 conf/archive_decadal.yml create mode 100644 modules/data_load/load_decadal.R create mode 100644 modules/data_load/recipe_decadal.yml diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml new file mode 100644 index 00000000..119d1a69 --- /dev/null +++ b/conf/archive_decadal.yml @@ -0,0 +1,49 @@ +archive: + src: "/esarchive/" + Decadal: +# ---- + EC-Earth3: #-i1: + src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/" + historical: + src: + table: + simulation: + src: "dcppA-hindcast/" + #NOTE: tos is under both Amon and Omon + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} +# tas: +# path: "$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" +# sdate: +# start: 1960 +# end: 2018 + nmember: 10 +# ---- + HadGEM3-GC31-MM: + src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" # dcppB also (depend on sdate) + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon"} + grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr"} + version: {"tas":"v20200316", "psl":"v20200316", "pr":"v20200316", "ts":"v20200316"} #depend on member and variable + daily: + grid: {"tas":"gn"} + verison: + calendar: "360-day" + nmember: 10 # Usage?? + member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 + initial_month: 11 + +# tas: +# monthly path: "$member$/Amon/$var$/gr/v20200316/$var$_Amon_*_s$sdate$_$member$_gr_$sdate$11-$aux$03.nc" +# daily path: $member$/day/$var$/gn/v20200101/$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$sdate$1101-$aux$1230.nc' + +# ---- + + References: + GHCNv4: + src: "/esarchive/obs/noaa/ghcn_v4/" + #QUESTION: tas or tasanomaly??? + monthly_mean: {"tasanomaly":""} + ERA5: + src: "/esarchive/recon/ecmwf/era5/" + monthly_mean: {"tas":"_f1h-r1440x721cds", "prlr":"_f1h-r1440x721cds", "psl":"_f1h-r1440x721cds"} + diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R new file mode 100644 index 00000000..a7ed1efa --- /dev/null +++ b/modules/data_load/load_decadal.R @@ -0,0 +1,116 @@ +# Loading module: +# 1. archive.yml +# 2. recipe.yml +# 3. Load_decadal.R (V) + +library(yaml) +library(startR) + + dict <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/conf/archive_decadal.yml") + recipe <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/modules/data_load/recipe_decadal.yml") + + # Read from recipe: + model <- recipe$Analysis$Datasets$System$name #'HadGEM3' + member <- strsplit(recipe$Analysis$Datasets$System$member, ',')[[1]] #c("r1i1p1f2", "r2i1p1f2") + var <- recipe$Analysis$Variables$name #'tas' + freq <- recipe$Analysis$Variables$freq #monthly_mean + lat_min <- recipe$Analysis$Region$latmin #0 + lat_max <- recipe$Analysis$Region$latmax #10 + lon_min <- recipe$Analysis$Region$lonmin #0 + lon_max <- recipe$Analysis$Region$lonmax #10 + sdates <- as.numeric(recipe$Analysis$Time$sdate$start):as.numeric(recipe$Analysis$Time$sdate$end) #1960:2015 + fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 + # only used for daily data + season <- recipe$Analysis$Time$season + + # Read from archive: + table <- dict$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' # depend on var + grid <- dict$archive$Decadal[[model]][[freq]]$grid[[var]] + version <- dict$archive$Decadal[[model]][[freq]]$version[[var]] + member <- ifelse(member == 'all', strsplit(dict$archive$Decadal[[model]]$member, ',')[[1]], member) + + # derived from above: + circularsort <- ifelse(lon_min >= 0, CircularSort(0, 360), CircularSort(-180, 180)) + if (lon_min >= 0) { + circularsort <- CircularSort(0, 360) + } else { + circularsort <- CircularSort(-180, 180) + } + +#------------------------------------------- +# Step 1: Load the exp +#------------------------------------------- +#monthly and daily + path <- file.path(dict$archive$src, dict$archive$Decadal[[model]]$src, + '$member$', table, '$var$', grid, version) + + files <- paste0('$var$_', table, '_', model, '_*_s$sdate$_$member$_', grid, '_$chunk$.nc') + +#monthly + fyears <- 1 + fmonths <- (1 + 12 * (fyears[1] - 1)) : (2 + 12 * fyears[length(fyears)]) +#daily +# fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') + +#monthly + exp <- Start(dat = file.path(path, files), + var = var, + member = member, + sdate = paste0(sdates), + chunk = 'all', + chunk_depends = 'sdate', + fmonth = indices(fmonths), + lat = values(list(lat_min, lat_max)), + lat_reorder = Sort(), + lon = values(list(lon_min, lon_max)), + lon_reorder = circularsort, + synonims = list(fmonth = c('fmonth', 'time'), + lon = c('lon', 'longitude'), + lat = c('lat', 'latitude')), + return_vars = list(lat = NULL, lon = NULL, + fmonth = c('sdate', 'chunk')), + retrieve = F) + +#daily + data <- Start(dat = path, + var = variable, + sdate = paste0(sdates), + aux = 'all', + aux_depends = 'sdate', + lat = values(list(lat_min, lat_max)), + lat_reorder = Sort(), + lon = values(list(lon_min, lon_max)), + lon_reorder = CircularSort(0, 360), + fday = indices(fdays), + member = members, + synonims = list(fday = c('fday', 'time'), + lon = c('lon', 'longitude'), + lat = c('lat', 'latitude')), + return_vars = list(lat = NULL, lon = NULL, + fday = c('sdate', 'aux')), + retrieve = F) + + + +#------------------------------------------- +# Step 2. Load the reference +#------------------------------------------- +path <- '/esarchive/obs/noaa/ghcn_v2/monthly_mean/$var$/$var$_$year$$month$.nc' + +reference <- Start(dat = path, + var = var, + year = paste0(years_hist), + month = 'all', + lat = values(list(lat_min, lat_max)), + lon = values(list(lon_min, lon_max)), + synonims = list(lat = c('lat','latitude'), + lon = c('lon','longitude'), + month = c('month','time')), + return_vars = list(lat = NULL, lon = NULL), + lat_reorder = Sort(decreasing = F), + lon_reorder = CircularSort(0, 360), + num_procs = 1, retrieve = FALSE) + +all(attributes(hist)$Variables$common$lat == attributes(exp)$Variables$common$lat) +all(attributes(reference)$Variables$common$lat == attributes(hist)$Variables$common$lat) + diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml new file mode 100644 index 00000000..0c72135f --- /dev/null +++ b/modules/data_load/recipe_decadal.yml @@ -0,0 +1,47 @@ +Description: + Author: An-Chi Ho + '': split version +Analysis: + Horizon: Decadal + Variables: + name: tas + freq: monthly_mean + Datasets: + System: + name: HadGEM3-GC31-MM + member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 #all + Multimodel: no + Reference: + name: era5 + Time: + sdate: + start: 1960 + end: 2018 + fyear: + start: 1 + end: 1 + season: 'Annual' #only for daily data + leadtimemin: 2 + leadtimemax: 4 + Region: + latmin: -90 + latmax: 90 + lonmin: 0 + lonmax: 359.9 + Regrid: + method: bilinear + type: to_system + Workflow: + Calibration: + method: SBC + Skill: + metric: RPSS + Indicators: + index: no + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/aho/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/aho/auto-s2s/ + -- GitLab From 42b87dc76e5e805ffb1551a09b1d513ca3272830 Mon Sep 17 00:00:00 2001 From: aho Date: Fri, 4 Mar 2022 19:03:35 +0100 Subject: [PATCH 02/21] Revise time step --- modules/data_load/load_decadal.R | 15 +++++++++------ modules/data_load/recipe_decadal.yml | 6 +++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index a7ed1efa..05343e10 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -19,7 +19,8 @@ library(startR) lon_min <- recipe$Analysis$Region$lonmin #0 lon_max <- recipe$Analysis$Region$lonmax #10 sdates <- as.numeric(recipe$Analysis$Time$sdate$start):as.numeric(recipe$Analysis$Time$sdate$end) #1960:2015 - fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 +# fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 + time <- as.numeric(recipe$Analysis$Time$leadtimemin):as.numeric(recipe$Analysis$Time$leadtimemax) # only used for daily data season <- recipe$Analysis$Time$season @@ -30,7 +31,7 @@ library(startR) member <- ifelse(member == 'all', strsplit(dict$archive$Decadal[[model]]$member, ',')[[1]], member) # derived from above: - circularsort <- ifelse(lon_min >= 0, CircularSort(0, 360), CircularSort(-180, 180)) +# circularsort <- ifelse(lon_min >= 0, CircularSort(0, 360), CircularSort(-180, 180)) if (lon_min >= 0) { circularsort <- CircularSort(0, 360) } else { @@ -44,11 +45,11 @@ library(startR) path <- file.path(dict$archive$src, dict$archive$Decadal[[model]]$src, '$member$', table, '$var$', grid, version) - files <- paste0('$var$_', table, '_', model, '_*_s$sdate$_$member$_', grid, '_$chunk$.nc') + files <- paste0('$var$_', table, '_*_*_s$sdate$_$member$_', grid, '_$chunk$.nc') #monthly - fyears <- 1 - fmonths <- (1 + 12 * (fyears[1] - 1)) : (2 + 12 * fyears[length(fyears)]) +# fyears <- 1 +# fmonths <- (1 + 12 * (fyears[1] - 1)) : (2 + 12 * fyears[length(fyears)]) #daily # fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') @@ -59,7 +60,9 @@ library(startR) sdate = paste0(sdates), chunk = 'all', chunk_depends = 'sdate', - fmonth = indices(fmonths), + fmonth = indices(time), + fmonth_across = 'chunk', + merge_across_dims = TRUE, lat = values(list(lat_min, lat_max)), lat_reorder = Sort(), lon = values(list(lon_min, lon_max)), diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index 0c72135f..23a4c9e3 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -16,13 +16,13 @@ Analysis: Time: sdate: start: 1960 - end: 2018 + end: 1961 #2018 fyear: start: 1 end: 1 season: 'Annual' #only for daily data - leadtimemin: 2 - leadtimemax: 4 + leadtimemin: 1 + leadtimemax: 14 Region: latmin: -90 latmax: 90 -- GitLab From bbdd33d8bbe7ba4406a4c474191fbb431a00bdcd Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 7 Mar 2022 13:06:07 +0100 Subject: [PATCH 03/21] Add ECEarth3-i1 and ECEarth3-i2 to archive and test the loading script --- conf/archive_decadal.yml | 59 ++++++++++++++++++---------- modules/data_load/load_decadal.R | 16 ++++++-- modules/data_load/recipe_decadal.yml | 8 ++-- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 119d1a69..bf8e1b76 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -2,39 +2,58 @@ archive: src: "/esarchive/" Decadal: # ---- - EC-Earth3: #-i1: - src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/" - historical: - src: - table: - simulation: - src: "dcppA-hindcast/" + EC-Earth3-i1: #NOTE: i1 and i2 should unify? + #NOTE: a1ua is old, a3w5 is new + src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? +#"$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" + monthly_mean: #NOTE: tos is under both Amon and Omon table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} -# tas: -# path: "$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" -# sdate: -# start: 1960 -# end: 2018 - nmember: 10 + grid: {"tas":"gr"} + version: {"tas":"v20190713"} + daily: + grid: {"tas":"gr"} + verison: {"tas":"v20190713"} + calendar: "proleptic_gregorian" + # i1 is for alua, i4 is for a3w5 + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: 11 + resolution: 'r512x256' + +# ---- + EC-Earth3-i2: + src: "exp/CMIP6/dcppA-hindcast/ec-earth3/cmip6-dcppA-hindcast_i2p1/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" +#$member$/Amon/$var$/gr/v20200101/$var$_Amon_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$01-$fyear$12.nc' + monthly_mean: + table: {"tas":"Amon"} + grid: {"tas":"gr"} + version: {"tas":"v20200730"} + daily: + grid: {"tas":"gr"} + verison: {"tas":"v20200731"} + calendar: "proleptic_gregorian" + #NOTE:There are many members but not all of them are available on ESGF (only r6-10 available). Then, we might have some variables for the rest of the members (r1-5 and r11-15), but not for all the variables. That's why i'm only using r6-10 + member: r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 + initial_month: 11 + resolution: 'r512x256' + # ---- HadGEM3-GC31-MM: - src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" # dcppB also (depend on sdate) + src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" # dcppB-forecast also? (depend on sdate) +# tas: +# monthly path: "$member$/Amon/$var$/gr/v20200316/$var$_Amon_*_s$sdate$_$member$_gr_$sdate$11-$aux$03.nc" +# daily path: $member$/day/$var$/gn/v20200101/$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$sdate$1101-$aux$1230.nc' monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon"} grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr"} version: {"tas":"v20200316", "psl":"v20200316", "pr":"v20200316", "ts":"v20200316"} #depend on member and variable daily: grid: {"tas":"gn"} - verison: + verison: {"tasmin":"v20200101", "tasmax":"v20200101", "pr":"v20200417"} calendar: "360-day" - nmember: 10 # Usage?? member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 initial_month: 11 - -# tas: -# monthly path: "$member$/Amon/$var$/gr/v20200316/$var$_Amon_*_s$sdate$_$member$_gr_$sdate$11-$aux$03.nc" -# daily path: $member$/day/$var$/gn/v20200101/$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$sdate$1101-$aux$1230.nc' + resolution: 'r432x324' # ---- diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 05343e10..7717fcc8 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -28,7 +28,9 @@ library(startR) table <- dict$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' # depend on var grid <- dict$archive$Decadal[[model]][[freq]]$grid[[var]] version <- dict$archive$Decadal[[model]][[freq]]$version[[var]] - member <- ifelse(member == 'all', strsplit(dict$archive$Decadal[[model]]$member, ',')[[1]], member) + if (member == 'all') { + member <- strsplit(dict$archive$Decadal[[model]]$member, ',')[[1]] + } # derived from above: # circularsort <- ifelse(lon_min >= 0, CircularSort(0, 360), CircularSort(-180, 180)) @@ -44,8 +46,7 @@ library(startR) #monthly and daily path <- file.path(dict$archive$src, dict$archive$Decadal[[model]]$src, '$member$', table, '$var$', grid, version) - - files <- paste0('$var$_', table, '_*_*_s$sdate$_$member$_', grid, '_$chunk$.nc') + files <- paste0('$var$_', table, '_*_*_s$sdate$-$member$_', grid, '_$chunk$.nc') #monthly # fyears <- 1 @@ -67,6 +68,11 @@ library(startR) lat_reorder = Sort(), lon = values(list(lon_min, lon_max)), lon_reorder = circularsort, + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', #need to change + method = recipe$Analysis$Regrid$method), + transform_vars = c('lat', 'lon'), synonims = list(fmonth = c('fmonth', 'time'), lon = c('lon', 'longitude'), lat = c('lat', 'latitude')), @@ -74,7 +80,9 @@ library(startR) fmonth = c('sdate', 'chunk')), retrieve = F) -#daily + + +#daily (not done yet) data <- Start(dat = path, var = variable, sdate = paste0(sdates), diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index 23a4c9e3..f417a44a 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -7,11 +7,11 @@ Analysis: name: tas freq: monthly_mean Datasets: - System: + System: #naming? name: HadGEM3-GC31-MM member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 #all Multimodel: no - Reference: + Reference: #needed? naming? name: era5 Time: sdate: @@ -42,6 +42,6 @@ Analysis: Run: Loglevel: INFO Terminal: yes - output_dir: /esarchive/scratch/aho/auto-s2s/out-logs/ - code_dir: /esarchive/scratch/aho/auto-s2s/ + output_dir: /esarchive/scratch/aho/git/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/aho/git/auto-s2s/ -- GitLab From 5b9a333f29a405910be5d778cbe537416e2045eb Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 7 Mar 2022 16:54:06 +0100 Subject: [PATCH 04/21] Modification for the meeting's conclusion --- conf/archive_decadal.yml | 10 +++++----- modules/data_load/load_decadal.R | 28 +++++++++++++++++----------- modules/data_load/recipe_decadal.yml | 14 +++++++------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index bf8e1b76..f4ab1a21 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -18,11 +18,11 @@ archive: # i1 is for alua, i4 is for a3w5 member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 - resolution: 'r512x256' + reference_grid: "/esarchive/exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20190713/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196011-196110.nc" #'r512x256' # ---- EC-Earth3-i2: - src: "exp/CMIP6/dcppA-hindcast/ec-earth3/cmip6-dcppA-hindcast_i2p1/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + src: "exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #$member$/Amon/$var$/gr/v20200101/$var$_Amon_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$01-$fyear$12.nc' monthly_mean: table: {"tas":"Amon"} @@ -35,7 +35,7 @@ archive: #NOTE:There are many members but not all of them are available on ESGF (only r6-10 available). Then, we might have some variables for the rest of the members (r1-5 and r11-15), but not for all the variables. That's why i'm only using r6-10 member: r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 initial_month: 11 - resolution: 'r512x256' + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r6i2p1f1/Amon/tas/gr/v20200730/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r6i2p1f1_gr_196011-196012.nc" #'r512x256' # ---- HadGEM3-GC31-MM: @@ -53,9 +53,9 @@ archive: calendar: "360-day" member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 initial_month: 11 - resolution: 'r432x324' + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Amon/ts/gr/v20200316/ts_Amon_HadGEM3_dcppA-hindcast_s1960-r1i1p1f2_gr_196011-197103.nc" #'r432x324' -# ---- +# ===================================== References: GHCNv4: diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 7717fcc8..c7f45f47 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -8,8 +8,9 @@ library(startR) dict <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/conf/archive_decadal.yml") recipe <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/modules/data_load/recipe_decadal.yml") - + #------------------------- # Read from recipe: + #------------------------- model <- recipe$Analysis$Datasets$System$name #'HadGEM3' member <- strsplit(recipe$Analysis$Datasets$System$member, ',')[[1]] #c("r1i1p1f2", "r2i1p1f2") var <- recipe$Analysis$Variables$name #'tas' @@ -21,39 +22,44 @@ library(startR) sdates <- as.numeric(recipe$Analysis$Time$sdate$start):as.numeric(recipe$Analysis$Time$sdate$end) #1960:2015 # fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 time <- as.numeric(recipe$Analysis$Time$leadtimemin):as.numeric(recipe$Analysis$Time$leadtimemax) + regrid <- recipe$Analysis$Regrid$type # to_system + # only used for daily data season <- recipe$Analysis$Time$season + #------------------------- # Read from archive: - table <- dict$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' # depend on var + #------------------------- + table <- dict$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' grid <- dict$archive$Decadal[[model]][[freq]]$grid[[var]] version <- dict$archive$Decadal[[model]][[freq]]$version[[var]] if (member == 'all') { member <- strsplit(dict$archive$Decadal[[model]]$member, ',')[[1]] } + #------------------------- # derived from above: -# circularsort <- ifelse(lon_min >= 0, CircularSort(0, 360), CircularSort(-180, 180)) + #------------------------- if (lon_min >= 0) { circularsort <- CircularSort(0, 360) } else { circularsort <- CircularSort(-180, 180) } + # generate transform params for system and ref + regrid_params <- get_regrid_params(recipe, archive) + + #time for daily data. If recipe asks for time step, we don't need to calculate here +# fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') + #------------------------------------------- # Step 1: Load the exp #------------------------------------------- -#monthly and daily + #monthly and daily path <- file.path(dict$archive$src, dict$archive$Decadal[[model]]$src, '$member$', table, '$var$', grid, version) files <- paste0('$var$_', table, '_*_*_s$sdate$-$member$_', grid, '_$chunk$.nc') -#monthly -# fyears <- 1 -# fmonths <- (1 + 12 * (fyears[1] - 1)) : (2 + 12 * fyears[length(fyears)]) -#daily -# fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') - #monthly exp <- Start(dat = file.path(path, files), var = var, @@ -70,7 +76,7 @@ library(startR) lon_reorder = circularsort, transform = CDORemapper, transform_extra_cells = 2, - transform_params = list(grid = 'r360x181', #need to change + transform_params = list(grid = regrid_params$fcst.gridtype, #nc file method = recipe$Analysis$Regrid$method), transform_vars = c('lat', 'lon'), synonims = list(fmonth = c('fmonth', 'time'), diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index f417a44a..6ec67891 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -7,19 +7,19 @@ Analysis: name: tas freq: monthly_mean Datasets: - System: #naming? + System: name: HadGEM3-GC31-MM - member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 #all + member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 #'all' Multimodel: no - Reference: #needed? naming? + Reference: name: era5 Time: sdate: start: 1960 end: 1961 #2018 - fyear: - start: 1 - end: 1 +# fyear: +# start: 1 +# end: 1 season: 'Annual' #only for daily data leadtimemin: 1 leadtimemax: 14 @@ -30,7 +30,7 @@ Analysis: lonmax: 359.9 Regrid: method: bilinear - type: to_system + type: to_system #to_reference Workflow: Calibration: method: SBC -- GitLab From 207735e02370efd4eff11600e62b0843209e582c Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 14 Mar 2022 09:19:37 +0100 Subject: [PATCH 05/21] minor change --- conf/archive_decadal.yml | 13 +++++++------ modules/data_load/load_decadal.R | 9 ++++++--- modules/data_load/recipe_decadal.yml | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index f4ab1a21..1c300415 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -2,12 +2,12 @@ archive: src: "/esarchive/" Decadal: # ---- - EC-Earth3-i1: #NOTE: i1 and i2 should unify? - #NOTE: a1ua is old, a3w5 is new + EC-Earth3-i1: #NOTE: i1 and i2 should unify? No because the paths are different + #NOTE: a1ua is old, a3w5 is new. Cual? src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? #"$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" monthly_mean: - #NOTE: tos is under both Amon and Omon + #NOTE: tos is under both Amon and Omon --> wait to be changed table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} grid: {"tas":"gr"} version: {"tas":"v20190713"} @@ -44,9 +44,10 @@ archive: # monthly path: "$member$/Amon/$var$/gr/v20200316/$var$_Amon_*_s$sdate$_$member$_gr_$sdate$11-$aux$03.nc" # daily path: $member$/day/$var$/gn/v20200101/$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$sdate$1101-$aux$1230.nc' monthly_mean: - table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon"} - grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr"} - version: {"tas":"v20200316", "psl":"v20200316", "pr":"v20200316", "ts":"v20200316"} #depend on member and variable + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon", "tos":"Omon"} + grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr", "tos":"gr"} + #version depends on member and variable + version: {"tas":"v20200316", "psl":"v20200316", "pr":"v20200316", "ts":"v20200316", "tos":"v20200417"} daily: grid: {"tas":"gn"} verison: {"tasmin":"v20200101", "tasmax":"v20200101", "pr":"v20200417"} diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index c7f45f47..d5678d2b 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -2,10 +2,12 @@ # 1. archive.yml # 2. recipe.yml # 3. Load_decadal.R (V) - +setwd('/esarchive/scratch/aho/git/auto-s2s/') library(yaml) library(startR) +source("modules/data_load/regrid.R") # will change to CSOperational +#QUESTION: name? dict/archive dict <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/conf/archive_decadal.yml") recipe <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/modules/data_load/recipe_decadal.yml") #------------------------- @@ -33,7 +35,7 @@ library(startR) table <- dict$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' grid <- dict$archive$Decadal[[model]][[freq]]$grid[[var]] version <- dict$archive$Decadal[[model]][[freq]]$version[[var]] - if (member == 'all') { + if (identical(member, 'all')) { member <- strsplit(dict$archive$Decadal[[model]]$member, ',')[[1]] } @@ -46,7 +48,8 @@ library(startR) circularsort <- CircularSort(-180, 180) } # generate transform params for system and ref - regrid_params <- get_regrid_params(recipe, archive) + #NOTE: Not functional now + regrid_params <- get_regrid_params(recipe, dict) #time for daily data. If recipe asks for time step, we don't need to calculate here # fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index 6ec67891..4cd6187e 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -8,7 +8,7 @@ Analysis: freq: monthly_mean Datasets: System: - name: HadGEM3-GC31-MM + name: HadGEM3-GC31-MM #EC-Earth3-i1 #EC-Earth3-i2 member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 #'all' Multimodel: no Reference: -- GitLab From b1831a3e5e5bb8da1fd34ee0a243d4bba523ee50 Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 14 Mar 2022 14:59:18 +0100 Subject: [PATCH 06/21] Add dataset in archive and refine the loading script --- conf/archive_decadal.yml | 234 ++++++++++++++++++++++++++- modules/data_load/load_decadal.R | 109 +++++++++---- modules/data_load/recipe_decadal.yml | 6 +- 3 files changed, 304 insertions(+), 45 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 1c300415..a776e586 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -2,8 +2,7 @@ archive: src: "/esarchive/" Decadal: # ---- - EC-Earth3-i1: #NOTE: i1 and i2 should unify? No because the paths are different - #NOTE: a1ua is old, a3w5 is new. Cual? + EC-Earth3-i1: src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? #"$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" monthly_mean: @@ -15,7 +14,6 @@ archive: grid: {"tas":"gr"} verison: {"tas":"v20190713"} calendar: "proleptic_gregorian" - # i1 is for alua, i4 is for a3w5 member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20190713/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196011-196110.nc" #'r512x256' @@ -38,6 +36,38 @@ archive: reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r6i2p1f1/Amon/tas/gr/v20200730/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r6i2p1f1_gr_196011-196012.nc" #'r512x256' # ---- + EC-Earth3-i4: + src: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? +#"$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "clt":"Amon", "hfls":"Amon", + "hurs":"Amon", "huss":"Amon", "rsds":"Amon", "rsut":"Amon", "ta":"Amon", + "tasmax":"Amon", "tosa":"Amon", "ua":"Amon", "va":"Amon", "zg":"Amon", + "evspsbl":"Amon", "hfss":"Amon", "hursmin":"Amon", "rlut":"Amon", + "rsdt":"Amon", "sfcWind":"Amon", "tasmin":"Amon", "ts":"Amon", "uas":"Amon", + "vas":"Amon"} + grid: {"tas":"gr", "pr":"gr", "psl":"gr", "clt":"gr", "hfls":"gr", + "hurs":"gr", "huss":"gr", "rsds":"gr", "rsut":"gr", "ta":"gr", + "tasmax":"gr", "tosa":"gr", "ua":"gr", "va":"gr", "zg":"gr", + "evspsbl":"gr", "hfss":"gr", "hursmin":"gr", "rlut":"gr", + "rsdt":"gr", "sfcWind":"gr", "tasmin":"gr", "ts":"gr", "uas":"gr", + "vas":"gr"} + version: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910", "clt":"v20210910", + "hurs":"v20210910", "huss":"v20210910", "rsds":"v20210910", "rsut":"v20210910", "ta":"v20210910", + "tasmax":"v20210910", "tosa":"v20210910", "ua":"v20210910", "va":"v20210910", "zg":"v20210910", + "evspsbl":"v20210910", "hfss":"v20210910", "hursmin":"v20210910", "rlut":"v20210910", + "rsdt":"v20210910", "sfcWind":"v20210910", "tasmin":"v20210910", "ts":"v20210910", "uas":"v20210910", + "vas":"v20210910"} +} + daily: + grid: {"tas":"gr", "pr":"gr", "psl":"gr"} + verison: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910"} + calendar: "proleptic_gregorian" + member: r1i4p1f1,r2i4p1f1,r3i4p1f1,r4i4p1f1,r5i4p1f1,r6i4p1f1,r7i4p1f1,r8i4p1f1,r9i4p1f1,r10i4p1f1 + initial_month: 11 + reference_grid: "/esarchive/exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i4p1f1/Amon/tas/gr/v20210910/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i4p1f1_gr_196011-196110.nc" + +# ---- HadGEM3-GC31-MM: src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" # dcppB-forecast also? (depend on sdate) # tas: @@ -54,16 +84,206 @@ archive: calendar: "360-day" member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Amon/ts/gr/v20200316/ts_Amon_HadGEM3_dcppA-hindcast_s1960-r1i1p1f2_gr_196011-197103.nc" #'r432x324' + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Amon/tas/gr/v20200316/tas_Amon_HadGEM3_dcppA-hindcast_s2018-r1i1p1f2_gr_201811-202903.nc" #'r432x324' + +# ---- + BCC-CSM2-MR: + src: "exp/CMIP6/dcppA-hindcast/bcc-csm2-mr/cmip6-dcppA-hindcast_i1p1/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn"} +#QUESTION: Cual??? + version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 + initial_month: + reference_grid: + +# ---- + CanESM5: + src: "exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} + + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} + version: {"tas":"v20190429", "pr":"v20190429", "psl":"v20190429", "tasmin":"v20190429", "tasmax":"v20190429"} + daily: + grid: + verison: + calendar: + member: r1i1p2f1,r2i1p2f1,r3i1p2f1,r4i1p2f1,r5i1p2f1,r6i1p2f1,r7i1p2f1,r8i1p2f1, r9i1p2f1, r10i1p2f1, r11i1p2f1,r12i1p2f1,r13i1p2f1,r14i1p2f1,r15i1p2f1,r16i1p2f1,r17i1p2f1,r18i1p2f1, r19i1p2f1, r20i1p2f1,r21i1p2f1,r22i1p2f1,r23i1p2f1,r24i1p2f1,r25i1p2f1,r26i1p2f1,r27i1p2f1,r28i1p2f1, r29i1p2f1, r30i1p2f1, r31i1p2f1,r32i1p2f1,r33i1p2f1,r34i1p2f1,r35i1p2f1,r36i1p2f1,r37i1p2f1,r38i1p2f1, r39i1p2f1, r40i1p2f1 + initial_month: + reference_grid: + +# ---- + CESM1-1-CAM5-CMIP5: +#QUESTION: ????? + src: "/esarchive/exp/ncar/cesm-dple/" + monthly_mean: + table: + grid: + version: + daily: + grid: + verison: + calendar: + member: + initial_month: + reference_grid: + +# ---- + CMCC-CM2-SR5: + src: "exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} + version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320", "tasmin":"v20200320", "tasmax":"v20200320"} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: + reference_grid: + +# ---- +#QUESTION: missing in spreadsheet + FGOALS-f3-L: + src: + monthly_mean: + table: + grid: + version: + daily: + grid: + verison: + calendar: + member: + initial_month: + reference_grid: + +# ---- + IPSL-CM6A-LR: + src: "exp/CMIP6/dcppA-hindcast/ipsl-cm6a-lr/cmip6-dcppA-hindcast_i1p1/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "sfcWind":"Amon"} + grid: {"tas":"gr", "pr":"gr", "psl":"gr", "sfcWind":"gr"} + version: {"tas":"v20200504", "pr":"v20200504", "psl":"v20200504", "sfcWind":"v20200504"} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: + reference_grid: + +# ---- + MIROC6: + src: "exp/CMIP6/dcppA-hindcast/miroc6/cmip6-dcppA-hindcast_i1p1/DCPP/MIROC/MIROC6/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} + version: {"tas":"v20200417", "pr":["v20200416","v20200504"], "psl":"v20200504", "tasmin":"v20200417", "tasmax":"v20200504"} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: + reference_grid: + +# ---- + MPI-ESM1.2-HR: + src: "exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} + version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320", "sfcWind":"v20200320"} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: + reference_grid: + +# ---- + MPI-ESM1.2-LR: + src: "exp/CMIP6/dcppA-hindcast/mpi-esm1-2-lr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "ts":"gn"} + version: {"tas":"v20200101", "pr":"v20200101", "psl":"v20200101", "ts":"v20200101"} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1,r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1 + initial_month: + reference_grid: + +# ---- + MRI-ESM2-0: + src: "exp/CMIP6/dcppA-hindcast/mri-esm2-0/cmip6-dcppA-hindcast_i1p1/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn"} + version: {"tas":"v20200101", "pr":"v20200101", "psl":"v20200101"} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: + reference_grid: + +# ---- + NorCPM1-i1: + src: "exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i1p1/DCPP/NCC/NorCPM1/dcppA-hindcast/" + monthly_mean: + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn"} + version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320"} + daily: + grid: + verison: + calendar: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 + initial_month: + reference_grid: + +# ---- + NorCPM1-i2: + src: "exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i2p1/DCPP/NCC/NorCPM1/dcppA-hindcast/" + monthly_mean: + table: + grid: + version: + daily: + grid: + verison: + calendar: + member: r1i2p1f1,r2i2p1f1,r3i2p1f1,r4i2p1f1,r5i2p1f1,r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 + initial_month: + reference_grid: + # ===================================== References: GHCNv4: - src: "/esarchive/obs/noaa/ghcn_v4/" - #QUESTION: tas or tasanomaly??? + src: "obs/noaa/ghcn_v4/" monthly_mean: {"tasanomaly":""} + daily: + reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" ERA5: - src: "/esarchive/recon/ecmwf/era5/" + src: "recon/ecmwf/era5/" +#QUESTION: tas_f1h or tas_f1h-r1440x721cds? monthly_mean: {"tas":"_f1h-r1440x721cds", "prlr":"_f1h-r1440x721cds", "psl":"_f1h-r1440x721cds"} + daily: + calendar: "360-day" + reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index d5678d2b..be323b18 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -14,13 +14,14 @@ source("modules/data_load/regrid.R") # will change to CSOperational # Read from recipe: #------------------------- model <- recipe$Analysis$Datasets$System$name #'HadGEM3' + reference <- recipe$Analysis$Datasets$Reference$name #'era5' member <- strsplit(recipe$Analysis$Datasets$System$member, ',')[[1]] #c("r1i1p1f2", "r2i1p1f2") var <- recipe$Analysis$Variables$name #'tas' freq <- recipe$Analysis$Variables$freq #monthly_mean - lat_min <- recipe$Analysis$Region$latmin #0 - lat_max <- recipe$Analysis$Region$latmax #10 - lon_min <- recipe$Analysis$Region$lonmin #0 - lon_max <- recipe$Analysis$Region$lonmax #10 + lat_min <- as.numeric(recipe$Analysis$Region$latmin) #0 + lat_max <- as.numeric(recipe$Analysis$Region$latmax) #10 + lon_min <- as.numeric(recipe$Analysis$Region$lonmin) #0 + lon_max <- as.numeric(recipe$Analysis$Region$lonmax) #10 sdates <- as.numeric(recipe$Analysis$Time$sdate$start):as.numeric(recipe$Analysis$Time$sdate$end) #1960:2015 # fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 time <- as.numeric(recipe$Analysis$Time$leadtimemin):as.numeric(recipe$Analysis$Time$leadtimemax) @@ -48,8 +49,7 @@ source("modules/data_load/regrid.R") # will change to CSOperational circularsort <- CircularSort(-180, 180) } # generate transform params for system and ref - #NOTE: Not functional now - regrid_params <- get_regrid_params(recipe, dict) + regrid_params <- get_regrid_params(recipe, dict$archive$Decadal) #time for daily data. If recipe asks for time step, we don't need to calculate here # fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') @@ -70,25 +70,23 @@ source("modules/data_load/regrid.R") # will change to CSOperational sdate = paste0(sdates), chunk = 'all', chunk_depends = 'sdate', - fmonth = indices(time), - fmonth_across = 'chunk', + time = indices(time), + time_across = 'chunk', merge_across_dims = TRUE, lat = values(list(lat_min, lat_max)), - lat_reorder = Sort(), + lat_reorder = Sort(decreasing = TRUE), lon = values(list(lon_min, lon_max)), lon_reorder = circularsort, - transform = CDORemapper, + transform = regrid_params$fcst.transform, transform_extra_cells = 2, transform_params = list(grid = regrid_params$fcst.gridtype, #nc file - method = recipe$Analysis$Regrid$method), + method = regrid_params$fcst.gridmethod), transform_vars = c('lat', 'lon'), - synonims = list(fmonth = c('fmonth', 'time'), - lon = c('lon', 'longitude'), + synonims = list(lon = c('lon', 'longitude'), lat = c('lat', 'latitude')), return_vars = list(lat = NULL, lon = NULL, - fmonth = c('sdate', 'chunk')), - retrieve = F) - + time = c('sdate', 'chunk')), + retrieve = T) #daily (not done yet) @@ -98,7 +96,7 @@ source("modules/data_load/regrid.R") # will change to CSOperational aux = 'all', aux_depends = 'sdate', lat = values(list(lat_min, lat_max)), - lat_reorder = Sort(), + lat_reorder = Sort(decreasing = TRUE), lon = values(list(lon_min, lon_max)), lon_reorder = CircularSort(0, 360), fday = indices(fdays), @@ -115,22 +113,63 @@ source("modules/data_load/regrid.R") # will change to CSOperational #------------------------------------------- # Step 2. Load the reference #------------------------------------------- -path <- '/esarchive/obs/noaa/ghcn_v2/monthly_mean/$var$/$var$_$year$$month$.nc' - -reference <- Start(dat = path, - var = var, - year = paste0(years_hist), - month = 'all', - lat = values(list(lat_min, lat_max)), - lon = values(list(lon_min, lon_max)), - synonims = list(lat = c('lat','latitude'), - lon = c('lon','longitude'), - month = c('month','time')), - return_vars = list(lat = NULL, lon = NULL), - lat_reorder = Sort(decreasing = F), - lon_reorder = CircularSort(0, 360), - num_procs = 1, retrieve = FALSE) - -all(attributes(hist)$Variables$common$lat == attributes(exp)$Variables$common$lat) -all(attributes(reference)$Variables$common$lat == attributes(hist)$Variables$common$lat) + path.obs <- file.path(dict$archive$src, dict$archive$Reference[[reference]]$src, freq, + paste0(var, dict$archive$Reference[[reference]][[freq]][[var]])) + files.obs <- paste0('$var$_$file_date$.nc') + + + dates <- attr(exp, 'Variables')$common$time + + obs <- Start(dat = file.path(path.obs, files.obs), + var = var, + file_date = unique(format(dates, '%Y%m')), + time = dates, # [sdate, time] + time_across = 'file_date', + merge_across_dims = TRUE, + split_multiselected_dims = TRUE, + lat = values(list(lat_min, lat_max)), + lat_reorder = Sort(decreasing = TRUE), + lon = values(list(lon_min, lon_max)), + lon_reorder = circularsort, + transform = regrid_params$obs.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$obs.gridtype, #nc file + method = regrid_params$obs.gridmethod), + transform_vars = c('lat', 'lon'), + synonims = list(lat = c('lat','latitude'), + lon = c('lon','longitude')), + return_vars = list(lat = NULL, lon = NULL, + time = 'file_date'), + retrieve = TRUE) + + +#------------------------------------------- +# Step 3. Verify the consistance btwn exp and obs +#------------------------------------------- + # dimension + if (any(!names(dim(obs)) %in% names(dim(exp)))) { + stop("exp and obs don't share the same dimension names.") + } else { + match_ind <- match(names(dim(obs)), names(dim(exp))) + if (!all(dim(exp)[match_ind] == dim(obs))) stop("exp and obs don't share the same dimension length.") + } + + # time attribute + if (!identical(format(attr(exp, 'Variables')$common$time, '%Y%m'), + format(attr(obs, 'Variables')$common$time, '%Y%m'))) + stop("exp and obs don't share the same time.") + + # lat and lon attributes + if (!identical(as.vector(attr(exp, 'Variables')$common$lat), + as.vector(attr(obs, 'Variables')$common$lat))) + stop("exp and obs don't share the same latitude.") + if (!identical(as.vector(attr(exp, 'Variables')$common$lon), + as.vector(attr(obs, 'Variables')$common$lon))) + stop("exp and obs don't share the same longitude.") + + +#------------------------------------------- +# Step 4. Convert from startR_array to s2dv_array +#------------------------------------------- + diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index 4cd6187e..ccb3037d 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -8,11 +8,11 @@ Analysis: freq: monthly_mean Datasets: System: - name: HadGEM3-GC31-MM #EC-Earth3-i1 #EC-Earth3-i2 - member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 #'all' + name: EC-Earth3-i2 #HadGEM3-GC31-MM #EC-Earth3-i1 #EC-Earth3-i2 + member: 'all' #r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 Multimodel: no Reference: - name: era5 + name: ERA5 Time: sdate: start: 1960 -- GitLab From e5d3480bf6fa18ffb18898996d5eef0084fbc132 Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 14 Mar 2022 17:20:03 +0100 Subject: [PATCH 07/21] Rename to convention --- conf/archive_decadal.yml | 70 ++++++++++---------- modules/data_load/load_decadal.R | 95 ++++++++++++++-------------- modules/data_load/recipe_decadal.yml | 2 +- 3 files changed, 83 insertions(+), 84 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index a776e586..8866848c 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -10,9 +10,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} grid: {"tas":"gr"} version: {"tas":"v20190713"} - daily: + daily_mean: grid: {"tas":"gr"} - verison: {"tas":"v20190713"} + version: {"tas":"v20190713"} calendar: "proleptic_gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 @@ -26,9 +26,9 @@ archive: table: {"tas":"Amon"} grid: {"tas":"gr"} version: {"tas":"v20200730"} - daily: + daily_mean: grid: {"tas":"gr"} - verison: {"tas":"v20200731"} + version: {"tas":"v20200731"} calendar: "proleptic_gregorian" #NOTE:There are many members but not all of them are available on ESGF (only r6-10 available). Then, we might have some variables for the rest of the members (r1-5 and r11-15), but not for all the variables. That's why i'm only using r6-10 member: r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 @@ -59,9 +59,9 @@ archive: "rsdt":"v20210910", "sfcWind":"v20210910", "tasmin":"v20210910", "ts":"v20210910", "uas":"v20210910", "vas":"v20210910"} } - daily: + daily_mean: grid: {"tas":"gr", "pr":"gr", "psl":"gr"} - verison: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910"} + version: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910"} calendar: "proleptic_gregorian" member: r1i4p1f1,r2i4p1f1,r3i4p1f1,r4i4p1f1,r5i4p1f1,r6i4p1f1,r7i4p1f1,r8i4p1f1,r9i4p1f1,r10i4p1f1 initial_month: 11 @@ -72,15 +72,15 @@ archive: src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" # dcppB-forecast also? (depend on sdate) # tas: # monthly path: "$member$/Amon/$var$/gr/v20200316/$var$_Amon_*_s$sdate$_$member$_gr_$sdate$11-$aux$03.nc" -# daily path: $member$/day/$var$/gn/v20200101/$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$sdate$1101-$aux$1230.nc' +# daily_mean path: $member$/day/$var$/gn/v20200101/$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$sdate$1101-$aux$1230.nc' monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon", "tos":"Omon"} grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr", "tos":"gr"} #version depends on member and variable version: {"tas":"v20200316", "psl":"v20200316", "pr":"v20200316", "ts":"v20200316", "tos":"v20200417"} - daily: + daily_mean: grid: {"tas":"gn"} - verison: {"tasmin":"v20200101", "tasmax":"v20200101", "pr":"v20200417"} + version: {"tasmin":"v20200101", "tasmax":"v20200101", "pr":"v20200417"} calendar: "360-day" member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 initial_month: 11 @@ -94,9 +94,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn"} #QUESTION: Cual??? version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 initial_month: @@ -110,9 +110,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} version: {"tas":"v20190429", "pr":"v20190429", "psl":"v20190429", "tasmin":"v20190429", "tasmax":"v20190429"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p2f1,r2i1p2f1,r3i1p2f1,r4i1p2f1,r5i1p2f1,r6i1p2f1,r7i1p2f1,r8i1p2f1, r9i1p2f1, r10i1p2f1, r11i1p2f1,r12i1p2f1,r13i1p2f1,r14i1p2f1,r15i1p2f1,r16i1p2f1,r17i1p2f1,r18i1p2f1, r19i1p2f1, r20i1p2f1,r21i1p2f1,r22i1p2f1,r23i1p2f1,r24i1p2f1,r25i1p2f1,r26i1p2f1,r27i1p2f1,r28i1p2f1, r29i1p2f1, r30i1p2f1, r31i1p2f1,r32i1p2f1,r33i1p2f1,r34i1p2f1,r35i1p2f1,r36i1p2f1,r37i1p2f1,r38i1p2f1, r39i1p2f1, r40i1p2f1 initial_month: @@ -126,9 +126,9 @@ archive: table: grid: version: - daily: + daily_mean: grid: - verison: + version: calendar: member: initial_month: @@ -141,9 +141,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320", "tasmin":"v20200320", "tasmax":"v20200320"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: @@ -157,9 +157,9 @@ archive: table: grid: version: - daily: + daily_mean: grid: - verison: + version: calendar: member: initial_month: @@ -172,9 +172,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "sfcWind":"Amon"} grid: {"tas":"gr", "pr":"gr", "psl":"gr", "sfcWind":"gr"} version: {"tas":"v20200504", "pr":"v20200504", "psl":"v20200504", "sfcWind":"v20200504"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: @@ -187,9 +187,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} version: {"tas":"v20200417", "pr":["v20200416","v20200504"], "psl":"v20200504", "tasmin":"v20200417", "tasmax":"v20200504"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: @@ -202,9 +202,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320", "sfcWind":"v20200320"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: @@ -217,9 +217,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "ts":"gn"} version: {"tas":"v20200101", "pr":"v20200101", "psl":"v20200101", "ts":"v20200101"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1,r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1 initial_month: @@ -232,9 +232,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} version: {"tas":"v20200101", "pr":"v20200101", "psl":"v20200101"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: @@ -247,9 +247,9 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320"} - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: @@ -262,9 +262,9 @@ archive: table: grid: version: - daily: + daily_mean: grid: - verison: + version: calendar: member: r1i2p1f1,r2i2p1f1,r3i2p1f1,r4i2p1f1,r5i2p1f1,r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 initial_month: @@ -277,13 +277,13 @@ archive: GHCNv4: src: "obs/noaa/ghcn_v4/" monthly_mean: {"tasanomaly":""} - daily: + daily_mean: reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" ERA5: src: "recon/ecmwf/era5/" #QUESTION: tas_f1h or tas_f1h-r1440x721cds? monthly_mean: {"tas":"_f1h-r1440x721cds", "prlr":"_f1h-r1440x721cds", "psl":"_f1h-r1440x721cds"} - daily: + daily_mean: calendar: "360-day" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index be323b18..1f4e62b3 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -7,8 +7,7 @@ library(yaml) library(startR) source("modules/data_load/regrid.R") # will change to CSOperational -#QUESTION: name? dict/archive - dict <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/conf/archive_decadal.yml") + archive <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/conf/archive_decadal.yml") recipe <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/modules/data_load/recipe_decadal.yml") #------------------------- # Read from recipe: @@ -33,11 +32,11 @@ source("modules/data_load/regrid.R") # will change to CSOperational #------------------------- # Read from archive: #------------------------- - table <- dict$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' - grid <- dict$archive$Decadal[[model]][[freq]]$grid[[var]] - version <- dict$archive$Decadal[[model]][[freq]]$version[[var]] + table <- archive$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' + grid <- archive$archive$Decadal[[model]][[freq]]$grid[[var]] + version <- archive$archive$Decadal[[model]][[freq]]$version[[var]] if (identical(member, 'all')) { - member <- strsplit(dict$archive$Decadal[[model]]$member, ',')[[1]] + member <- strsplit(archive$archive$Decadal[[model]]$member, ',')[[1]] } #------------------------- @@ -49,7 +48,7 @@ source("modules/data_load/regrid.R") # will change to CSOperational circularsort <- CircularSort(-180, 180) } # generate transform params for system and ref - regrid_params <- get_regrid_params(recipe, dict$archive$Decadal) + regrid_params <- get_regrid_params(recipe, archive$archive$Decadal) #time for daily data. If recipe asks for time step, we don't need to calculate here # fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') @@ -59,53 +58,53 @@ source("modules/data_load/regrid.R") # will change to CSOperational # Step 1: Load the exp #------------------------------------------- #monthly and daily - path <- file.path(dict$archive$src, dict$archive$Decadal[[model]]$src, - '$member$', table, '$var$', grid, version) - files <- paste0('$var$_', table, '_*_*_s$sdate$-$member$_', grid, '_$chunk$.nc') + path <- file.path(archive$archive$src, archive$archive$Decadal[[model]]$src, + '$ensemble$', table, '$var$', grid, version) + files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') #monthly exp <- Start(dat = file.path(path, files), var = var, - member = member, - sdate = paste0(sdates), + ensemble = member, + syear = paste0(sdates), chunk = 'all', - chunk_depends = 'sdate', + chunk_depends = 'syear', time = indices(time), time_across = 'chunk', merge_across_dims = TRUE, - lat = values(list(lat_min, lat_max)), - lat_reorder = Sort(decreasing = TRUE), - lon = values(list(lon_min, lon_max)), - lon_reorder = circularsort, + latitude = values(list(lat_min, lat_max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lon_min, lon_max)), + longitude_reorder = circularsort, transform = regrid_params$fcst.transform, transform_extra_cells = 2, transform_params = list(grid = regrid_params$fcst.gridtype, #nc file method = regrid_params$fcst.gridmethod), - transform_vars = c('lat', 'lon'), - synonims = list(lon = c('lon', 'longitude'), - lat = c('lat', 'latitude')), - return_vars = list(lat = NULL, lon = NULL, - time = c('sdate', 'chunk')), + transform_vars = c('latitude', 'longitude'), + synonims = list(longitude = c('lon', 'longitude'), + latitude = c('lat', 'latitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = c('syear', 'chunk')), retrieve = T) #daily (not done yet) data <- Start(dat = path, var = variable, - sdate = paste0(sdates), + syear = paste0(sdates), aux = 'all', - aux_depends = 'sdate', - lat = values(list(lat_min, lat_max)), - lat_reorder = Sort(decreasing = TRUE), - lon = values(list(lon_min, lon_max)), - lon_reorder = CircularSort(0, 360), + aux_depends = 'syear', + latitude = values(list(lat_min, lat_max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lon_min, lon_max)), + longitude_reorder = CircularSort(0, 360), fday = indices(fdays), - member = members, + ensemble = members, synonims = list(fday = c('fday', 'time'), - lon = c('lon', 'longitude'), - lat = c('lat', 'latitude')), - return_vars = list(lat = NULL, lon = NULL, - fday = c('sdate', 'aux')), + longitude = c('lon', 'longitude'), + latitude = c('lat', 'latitude')), + return_vars = list(latitude = NULL, longitude = NULL, + fday = c('syear', 'aux')), retrieve = F) @@ -113,8 +112,8 @@ source("modules/data_load/regrid.R") # will change to CSOperational #------------------------------------------- # Step 2. Load the reference #------------------------------------------- - path.obs <- file.path(dict$archive$src, dict$archive$Reference[[reference]]$src, freq, - paste0(var, dict$archive$Reference[[reference]][[freq]][[var]])) + path.obs <- file.path(archive$archive$src, archive$archive$Reference[[reference]]$src, freq, + paste0(var, archive$archive$Reference[[reference]][[freq]][[var]])) files.obs <- paste0('$var$_$file_date$.nc') @@ -123,22 +122,22 @@ source("modules/data_load/regrid.R") # will change to CSOperational obs <- Start(dat = file.path(path.obs, files.obs), var = var, file_date = unique(format(dates, '%Y%m')), - time = dates, # [sdate, time] + time = dates, # [syear, time] time_across = 'file_date', merge_across_dims = TRUE, split_multiselected_dims = TRUE, - lat = values(list(lat_min, lat_max)), - lat_reorder = Sort(decreasing = TRUE), - lon = values(list(lon_min, lon_max)), - lon_reorder = circularsort, + latitude = values(list(lat_min, lat_max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lon_min, lon_max)), + longitude_reorder = circularsort, transform = regrid_params$obs.transform, transform_extra_cells = 2, transform_params = list(grid = regrid_params$obs.gridtype, #nc file method = regrid_params$obs.gridmethod), - transform_vars = c('lat', 'lon'), - synonims = list(lat = c('lat','latitude'), - lon = c('lon','longitude')), - return_vars = list(lat = NULL, lon = NULL, + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude')), + return_vars = list(latitude = NULL, longitude = NULL, time = 'file_date'), retrieve = TRUE) @@ -160,11 +159,11 @@ source("modules/data_load/regrid.R") # will change to CSOperational stop("exp and obs don't share the same time.") # lat and lon attributes - if (!identical(as.vector(attr(exp, 'Variables')$common$lat), - as.vector(attr(obs, 'Variables')$common$lat))) + if (!identical(as.vector(attr(exp, 'Variables')$common$latitude), + as.vector(attr(obs, 'Variables')$common$latitude))) stop("exp and obs don't share the same latitude.") - if (!identical(as.vector(attr(exp, 'Variables')$common$lon), - as.vector(attr(obs, 'Variables')$common$lon))) + if (!identical(as.vector(attr(exp, 'Variables')$common$longitude), + as.vector(attr(obs, 'Variables')$common$longitude))) stop("exp and obs don't share the same longitude.") diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index ccb3037d..ab93c7f1 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -5,7 +5,7 @@ Analysis: Horizon: Decadal Variables: name: tas - freq: monthly_mean + freq: monthly_mean #daily_mean Datasets: System: name: EC-Earth3-i2 #HadGEM3-GC31-MM #EC-Earth3-i1 #EC-Earth3-i2 -- GitLab From 910742e651036d29cfecbe5f8a4743593d798ffc Mon Sep 17 00:00:00 2001 From: aho Date: Tue, 15 Mar 2022 16:41:20 +0100 Subject: [PATCH 08/21] Change the names and supplement archive --- conf/archive_decadal.yml | 104 ++++++++++++++++++--------- modules/data_load/load_decadal.R | 83 ++++++++++----------- modules/data_load/recipe_decadal.yml | 14 ++-- 3 files changed, 119 insertions(+), 82 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 8866848c..a43bb792 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -1,10 +1,9 @@ archive: src: "/esarchive/" - Decadal: + System: # ---- EC-Earth3-i1: src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? -#"$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" monthly_mean: #NOTE: tos is under both Amon and Omon --> wait to be changed table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} @@ -19,9 +18,9 @@ archive: reference_grid: "/esarchive/exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20190713/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196011-196110.nc" #'r512x256' # ---- +#NOTE: EC-Earth3-i2 the first file of each sdate has 2 time step only (Nov-Dec). The rest files are Jan to Dec. EC-Earth3-i2: src: "exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" -#$member$/Amon/$var$/gr/v20200101/$var$_Amon_EC-Earth3_dcppA-hindcast_s$sdate$-$member$_gr_$fyear$01-$fyear$12.nc' monthly_mean: table: {"tas":"Amon"} grid: {"tas":"gr"} @@ -38,7 +37,6 @@ archive: # ---- EC-Earth3-i4: src: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? -#"$member$/Amon/$var$/gr/v20190713/$var$_Amon_*_s$sdate$-$member$_gr_$fyear$.nc" monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "clt":"Amon", "hfls":"Amon", "hurs":"Amon", "huss":"Amon", "rsds":"Amon", "rsut":"Amon", "ta":"Amon", @@ -58,7 +56,6 @@ archive: "evspsbl":"v20210910", "hfss":"v20210910", "hursmin":"v20210910", "rlut":"v20210910", "rsdt":"v20210910", "sfcWind":"v20210910", "tasmin":"v20210910", "ts":"v20210910", "uas":"v20210910", "vas":"v20210910"} -} daily_mean: grid: {"tas":"gr", "pr":"gr", "psl":"gr"} version: {"tas":"v20210910", "pr":"v20210910", "psl":"v20210910"} @@ -70,9 +67,6 @@ archive: # ---- HadGEM3-GC31-MM: src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" # dcppB-forecast also? (depend on sdate) -# tas: -# monthly path: "$member$/Amon/$var$/gr/v20200316/$var$_Amon_*_s$sdate$_$member$_gr_$sdate$11-$aux$03.nc" -# daily_mean path: $member$/day/$var$/gn/v20200101/$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$sdate$1101-$aux$1230.nc' monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon", "tos":"Omon"} grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr", "tos":"gr"} @@ -92,7 +86,7 @@ archive: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} -#QUESTION: Cual??? +#QUESTION: Cual??? Diff members have diff versions version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} daily_mean: grid: @@ -100,7 +94,7 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 initial_month: - reference_grid: + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/bcc-csm2-mr/cmip6-dcppA-hindcast_i1p1/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r8i1p1f1/Amon/tas/gn/v20200101/tas_Amon_BCC-CSM2-MR_dcppA-hindcast_s2008-r8i1p1f1_gn_200801-201712.nc" # ---- CanESM5: @@ -115,8 +109,8 @@ archive: version: calendar: member: r1i1p2f1,r2i1p2f1,r3i1p2f1,r4i1p2f1,r5i1p2f1,r6i1p2f1,r7i1p2f1,r8i1p2f1, r9i1p2f1, r10i1p2f1, r11i1p2f1,r12i1p2f1,r13i1p2f1,r14i1p2f1,r15i1p2f1,r16i1p2f1,r17i1p2f1,r18i1p2f1, r19i1p2f1, r20i1p2f1,r21i1p2f1,r22i1p2f1,r23i1p2f1,r24i1p2f1,r25i1p2f1,r26i1p2f1,r27i1p2f1,r28i1p2f1, r29i1p2f1, r30i1p2f1, r31i1p2f1,r32i1p2f1,r33i1p2f1,r34i1p2f1,r35i1p2f1,r36i1p2f1,r37i1p2f1,r38i1p2f1, r39i1p2f1, r40i1p2f1 - initial_month: - reference_grid: + initial_month: 1 + reference_grid: "/esarchive/exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/r1i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_dcppA-hindcast_s2008-r1i1p2f1_gn_200901-201812.nc" # ---- CESM1-1-CAM5-CMIP5: @@ -131,10 +125,11 @@ archive: version: calendar: member: - initial_month: + initial_month: 11 reference_grid: # ---- +#QUESTION: WRONG? CMCC-CM2-SR5: src: "exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/" monthly_mean: @@ -146,13 +141,13 @@ archive: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 - initial_month: - reference_grid: + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- #QUESTION: missing in spreadsheet FGOALS-f3-L: - src: + src: monthly_mean: table: grid: @@ -177,8 +172,8 @@ archive: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 - initial_month: - reference_grid: + initial_month: 1 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/ipsl-cm6a-lr/cmip6-dcppA-hindcast_i1p1/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20200504/tas_Amon_IPSL-CM6A-LR_dcppA-hindcast_s2008-r1i1p1f1_gr_200901-201812.nc" # ---- MIROC6: @@ -192,8 +187,8 @@ archive: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 - initial_month: - reference_grid: + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/miroc6/cmip6-dcppA-hindcast_i1p1/DCPP/MIROC/MIROC6/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200417/tas_Amon_MIROC6_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- MPI-ESM1.2-HR: @@ -207,8 +202,8 @@ archive: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 - initial_month: - reference_grid: + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- MPI-ESM1.2-LR: @@ -222,8 +217,8 @@ archive: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1,r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1 - initial_month: - reference_grid: + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-lr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MPI-ESM1-2-LR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- MRI-ESM2-0: @@ -237,8 +232,8 @@ archive: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 - initial_month: - reference_grid: + initial_month: 11 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mri-esm2-0/cmip6-dcppA-hindcast_i1p1/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MRI-ESM2-0_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201312.nc" # ---- NorCPM1-i1: @@ -252,33 +247,35 @@ archive: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 - initial_month: - reference_grid: + initial_month: 10 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i1p1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_NorCPM1_dcppA-hindcast_s2008-r1i1p1f1_gn_200810-201812.nc" # ---- NorCPM1-i2: src: "exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i2p1/DCPP/NCC/NorCPM1/dcppA-hindcast/" monthly_mean: - table: - grid: - version: + table: {"pr":"Amon", "psl":"Amon"} + grid: {"pr":"gn", "psl":"gn"} + version: {"pr":"v20200101", "psl":"v20200101"} daily_mean: grid: version: calendar: member: r1i2p1f1,r2i2p1f1,r3i2p1f1,r4i2p1f1,r5i2p1f1,r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 - initial_month: - reference_grid: + initial_month: 10 + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i2p1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" # ===================================== - References: + Reference: GHCNv4: src: "obs/noaa/ghcn_v4/" monthly_mean: {"tasanomaly":""} daily_mean: reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" + +# ---- ERA5: src: "recon/ecmwf/era5/" #QUESTION: tas_f1h or tas_f1h-r1440x721cds? @@ -287,3 +284,42 @@ archive: calendar: "360-day" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" +# ---- +# missing info in script + NCEP1: + +# ---- + JRA-55: + src: "recon/jma/jra55/" + monthly_mean: {"tas":"_f6h", "psl":"_f6h", "tos":"", "pr":"_s0-3h", "prlr":"_s0-3h"} + daily_mean: + reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" + +# ---- + GISTEMPv4: + src: "obs/noaa-nasa/ghcnersstgiss/" + monthly_mean: {"tasanomaly":""} + daily_mean: + reference_grid: "/esarchive/obs/noaa-nasa/ghcnersstgiss/monthly_mean/tasanomaly_200811.nc" + +# ---- + HadCRUT4: + src: "obs/ukmo/hadcrut_v4.6/" + monthly_mean: {"tasanomaly":""} + daily_mean: + reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" + +# ---- + GPCC: + src: "obs/noaa/gpcc-v2018/" + monthly_mean: {"prlr":""} + daily_mean: + reference_grid: "/esarchive/obs/noaa/gpcc-v2018/monthly_mean/prlr_200811.nc" + +# ---- + HadSLP2: + src: "obs/ukmo/hadslp_v2/" + monthly_mean: {"psl":""} + daily_mean: + reference_grid: "/esarchive/obs/ukmo/hadslp_v2/monthly_mean/psl_200811.nc" + diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 1f4e62b3..37ec54e9 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -32,11 +32,11 @@ source("modules/data_load/regrid.R") # will change to CSOperational #------------------------- # Read from archive: #------------------------- - table <- archive$archive$Decadal[[model]][[freq]]$table[[var]] #'Amon' - grid <- archive$archive$Decadal[[model]][[freq]]$grid[[var]] - version <- archive$archive$Decadal[[model]][[freq]]$version[[var]] + table <- archive$archive$System[[model]][[freq]]$table[[var]] #'Amon' + grid <- archive$archive$System[[model]][[freq]]$grid[[var]] + version <- archive$archive$System[[model]][[freq]]$version[[var]] if (identical(member, 'all')) { - member <- strsplit(archive$archive$Decadal[[model]]$member, ',')[[1]] + member <- strsplit(archive$archive$System[[model]]$member, ',')[[1]] } #------------------------- @@ -48,7 +48,7 @@ source("modules/data_load/regrid.R") # will change to CSOperational circularsort <- CircularSort(-180, 180) } # generate transform params for system and ref - regrid_params <- get_regrid_params(recipe, archive$archive$Decadal) + regrid_params <- get_regrid_params(recipe, archive$archive) #time for daily data. If recipe asks for time step, we don't need to calculate here # fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') @@ -58,34 +58,35 @@ source("modules/data_load/regrid.R") # will change to CSOperational # Step 1: Load the exp #------------------------------------------- #monthly and daily - path <- file.path(archive$archive$src, archive$archive$Decadal[[model]]$src, + path <- file.path(archive$archive$src, archive$archive$System[[model]]$src, '$ensemble$', table, '$var$', grid, version) files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') #monthly - exp <- Start(dat = file.path(path, files), - var = var, - ensemble = member, - syear = paste0(sdates), - chunk = 'all', - chunk_depends = 'syear', - time = indices(time), - time_across = 'chunk', - merge_across_dims = TRUE, - latitude = values(list(lat_min, lat_max)), - latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lon_min, lon_max)), - longitude_reorder = circularsort, - transform = regrid_params$fcst.transform, - transform_extra_cells = 2, - transform_params = list(grid = regrid_params$fcst.gridtype, #nc file - method = regrid_params$fcst.gridmethod), - transform_vars = c('latitude', 'longitude'), - synonims = list(longitude = c('lon', 'longitude'), - latitude = c('lat', 'latitude')), - return_vars = list(latitude = NULL, longitude = NULL, - time = c('syear', 'chunk')), - retrieve = T) + hcst <- Start(dat = file.path(path, files), + var = var, + ensemble = member, + syear = paste0(sdates), + chunk = 'all', + chunk_depends = 'syear', + time = indices(time), + time_across = 'chunk', + merge_across_dims = TRUE, + largest_dims_length = TRUE, + latitude = values(list(lat_min, lat_max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lon_min, lon_max)), + longitude_reorder = circularsort, + transform = regrid_params$fcst.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$fcst.gridtype, #nc file + method = regrid_params$fcst.gridmethod), + transform_vars = c('latitude', 'longitude'), + synonims = list(longitude = c('lon', 'longitude'), + latitude = c('lat', 'latitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = c('syear', 'chunk')), + retrieve = T) #daily (not done yet) @@ -117,7 +118,7 @@ source("modules/data_load/regrid.R") # will change to CSOperational files.obs <- paste0('$var$_$file_date$.nc') - dates <- attr(exp, 'Variables')$common$time + dates <- attr(hcst, 'Variables')$common$time obs <- Start(dat = file.path(path.obs, files.obs), var = var, @@ -143,28 +144,28 @@ source("modules/data_load/regrid.R") # will change to CSOperational #------------------------------------------- -# Step 3. Verify the consistance btwn exp and obs +# Step 3. Verify the consistance btwn hcst and obs #------------------------------------------- # dimension - if (any(!names(dim(obs)) %in% names(dim(exp)))) { - stop("exp and obs don't share the same dimension names.") + if (any(!names(dim(obs)) %in% names(dim(hcst)))) { + stop("hcst and obs don't share the same dimension names.") } else { - match_ind <- match(names(dim(obs)), names(dim(exp))) - if (!all(dim(exp)[match_ind] == dim(obs))) stop("exp and obs don't share the same dimension length.") + match_ind <- match(names(dim(obs)), names(dim(hcst))) + if (!all(dim(hcst)[match_ind] == dim(obs))) stop("hcst and obs don't share the same dimension length.") } # time attribute - if (!identical(format(attr(exp, 'Variables')$common$time, '%Y%m'), + if (!identical(format(attr(hcst, 'Variables')$common$time, '%Y%m'), format(attr(obs, 'Variables')$common$time, '%Y%m'))) - stop("exp and obs don't share the same time.") + stop("hcst and obs don't share the same time.") # lat and lon attributes - if (!identical(as.vector(attr(exp, 'Variables')$common$latitude), + if (!identical(as.vector(attr(hcst, 'Variables')$common$latitude), as.vector(attr(obs, 'Variables')$common$latitude))) - stop("exp and obs don't share the same latitude.") - if (!identical(as.vector(attr(exp, 'Variables')$common$longitude), + stop("hcst and obs don't share the same latitude.") + if (!identical(as.vector(attr(hcst, 'Variables')$common$longitude), as.vector(attr(obs, 'Variables')$common$longitude))) - stop("exp and obs don't share the same longitude.") + stop("hcst and obs don't share the same longitude.") #------------------------------------------- diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index ab93c7f1..56945915 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -8,11 +8,11 @@ Analysis: freq: monthly_mean #daily_mean Datasets: System: - name: EC-Earth3-i2 #HadGEM3-GC31-MM #EC-Earth3-i1 #EC-Earth3-i2 - member: 'all' #r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2 + name: CanESM5 + member: r1i1p2f1,r2i1p2f1,r3i1p2f1 #'all' Multimodel: no Reference: - name: ERA5 + name: JRA-55 #ERA5 Time: sdate: start: 1960 @@ -24,13 +24,13 @@ Analysis: leadtimemin: 1 leadtimemax: 14 Region: - latmin: -90 - latmax: 90 + latmin: 10 #-90 + latmax: 20 #90 lonmin: 0 - lonmax: 359.9 + lonmax: 15 #359.9 Regrid: method: bilinear - type: to_system #to_reference + type: to_reference Workflow: Calibration: method: SBC -- GitLab From c1130349e5c09608da1527bca75973374cfaac56 Mon Sep 17 00:00:00 2001 From: aho Date: Thu, 17 Mar 2022 10:27:28 +0100 Subject: [PATCH 09/21] Test --- modules/data_load/recipe_decadal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index 56945915..c3c80204 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -30,7 +30,7 @@ Analysis: lonmax: 15 #359.9 Regrid: method: bilinear - type: to_reference + type: to_system #to_reference Workflow: Calibration: method: SBC -- GitLab From d42d4523d692367e0e5809a3c138d3ea44705c6a Mon Sep 17 00:00:00 2001 From: aho Date: Tue, 22 Mar 2022 16:32:13 +0100 Subject: [PATCH 10/21] Some correction --- conf/archive_decadal.yml | 32 +++++++++++++--------------- modules/data_load/recipe_decadal.yml | 3 ++- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index a43bb792..584cea5a 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -18,7 +18,8 @@ archive: reference_grid: "/esarchive/exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20190713/tas_Amon_EC-Earth3_dcppA-hindcast_s1960-r1i1p1f1_gr_196011-196110.nc" #'r512x256' # ---- -#NOTE: EC-Earth3-i2 the first file of each sdate has 2 time step only (Nov-Dec). The rest files are Jan to Dec. + #NOTE: EC-Earth3-i2 the first file of each sdate has 2 time step only (Nov-Dec). + # The rest files are Jan to Dec. EC-Earth3-i2: src: "exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" monthly_mean: @@ -66,7 +67,7 @@ archive: # ---- HadGEM3-GC31-MM: - src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" # dcppB-forecast also? (depend on sdate) + src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon", "tos":"Omon"} grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr", "tos":"gr"} @@ -86,7 +87,7 @@ archive: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} -#QUESTION: Cual??? Diff members have diff versions + #NOTE: It should be the lastest version with all the files inside version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} daily_mean: grid: @@ -114,35 +115,33 @@ archive: # ---- CESM1-1-CAM5-CMIP5: -#QUESTION: ????? - src: "/esarchive/exp/ncar/cesm-dple/" + src: "exp/ncar/cesm-dple-dcppA-hindcast/cmorfiles/DCPP/NCAR/CESM1-1-CAM5-CMIP5/dcppA-hindcast" monthly_mean: - table: - grid: - version: + table: {"tas":"Amon", "pr":"Amon"} + grid: {"tas":"gn", "pr":"gn"} + version: {"tas":"v20200101", "pr":"v20200101"} daily_mean: grid: version: calendar: - member: + member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1, r9i1p1f1, r10i1p1f1, r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1,r17i1p1f1,r18i1p1f1, r19i1p1f1, r20i1p1f1,r21i1p1f1,r22i1p1f1,r23i1p1f1,r24i1p1f1,r25i1p1f1,r26i1p1f1,r27i1p1f1,r28i1p1f1, r29i1p1f1, r30i1p1f1, r31i1p1f1,r32i1p1f1,r33i1p1f1,r34i1p1f1,r35i1p1f1,r36i1p1f1,r37i1p1f1,r38i1p1f1, r39i1p1f1, r40i1p1f1 initial_month: 11 - reference_grid: + reference_grid: "/esarchive/exp/ncar/cesm-dple-dcppA-hindcast/cmorfiles/DCPP/NCAR/CESM1-1-CAM5-CMIP5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_CESM1-1-CAM5-CMIP5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- -#QUESTION: WRONG? CMCC-CM2-SR5: - src: "exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/" + src: "exp/CMIP6/dcppA-hindcast/cmcc-cm2-sr5/cmip6-dcppA-hindcast_i1p1/DCPP/CMCC-CM2-SR5/dcppA-hindcast/" monthly_mean: - table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} - grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} - version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320", "tasmin":"v20200320", "tasmax":"v20200320"} + table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "prc":"Amon", "ts":"Amon"} + grid: {"tas":"gn", "pr":"gn", "psl":"gn", "prc":"gn", "ts":"gn"} + version: {"tas":"v20210312", "pr":"v20210312", "psl":"v20210312", "prc":"v20200101", "ts":"v20200101"} daily_mean: grid: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/cmcc-cm2-sr5/cmip6-dcppA-hindcast_i1p1/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20210312/tas_Amon_CMCC-CM2-SR5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- #QUESTION: missing in spreadsheet @@ -278,7 +277,6 @@ archive: # ---- ERA5: src: "recon/ecmwf/era5/" -#QUESTION: tas_f1h or tas_f1h-r1440x721cds? monthly_mean: {"tas":"_f1h-r1440x721cds", "prlr":"_f1h-r1440x721cds", "psl":"_f1h-r1440x721cds"} daily_mean: calendar: "360-day" diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index c3c80204..cb0664d7 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -20,7 +20,8 @@ Analysis: # fyear: # start: 1 # end: 1 - season: 'Annual' #only for daily data + season: 'Annual' +# leadtime: leadtimemin: 1 leadtimemax: 14 Region: -- GitLab From 337c2342c38a2d9adb4a160334189ed4ae3ef10f Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 28 Mar 2022 15:01:25 +0200 Subject: [PATCH 11/21] Some minor modification --- conf/archive_decadal.yml | 7 +++-- modules/data_load/load_decadal.R | 46 ++++++++++++++++++++++++++-- modules/data_load/recipe_decadal.yml | 4 +-- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 584cea5a..f2cc6405 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -88,13 +88,14 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} #NOTE: It should be the lastest version with all the files inside - version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} + version: {"tas":"v20200504"} +# version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} daily_mean: grid: version: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 - initial_month: + initial_month: 1 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/bcc-csm2-mr/cmip6-dcppA-hindcast_i1p1/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r8i1p1f1/Amon/tas/gn/v20200101/tas_Amon_BCC-CSM2-MR_dcppA-hindcast_s2008-r8i1p1f1_gn_200801-201712.nc" # ---- @@ -110,7 +111,7 @@ archive: version: calendar: member: r1i1p2f1,r2i1p2f1,r3i1p2f1,r4i1p2f1,r5i1p2f1,r6i1p2f1,r7i1p2f1,r8i1p2f1, r9i1p2f1, r10i1p2f1, r11i1p2f1,r12i1p2f1,r13i1p2f1,r14i1p2f1,r15i1p2f1,r16i1p2f1,r17i1p2f1,r18i1p2f1, r19i1p2f1, r20i1p2f1,r21i1p2f1,r22i1p2f1,r23i1p2f1,r24i1p2f1,r25i1p2f1,r26i1p2f1,r27i1p2f1,r28i1p2f1, r29i1p2f1, r30i1p2f1, r31i1p2f1,r32i1p2f1,r33i1p2f1,r34i1p2f1,r35i1p2f1,r36i1p2f1,r37i1p2f1,r38i1p2f1, r39i1p2f1, r40i1p2f1 - initial_month: 1 + initial_month: 13 #next year Jan reference_grid: "/esarchive/exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/r1i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_dcppA-hindcast_s2008-r1i1p2f1_gn_200901-201812.nc" # ---- diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 37ec54e9..400e5a31 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -5,7 +5,9 @@ setwd('/esarchive/scratch/aho/git/auto-s2s/') library(yaml) library(startR) -source("modules/data_load/regrid.R") # will change to CSOperational +#Does not work now. Why?? +#source("https://earth.bsc.es/gitlab/es/csoperational/-/raw/master/R/get_regrid_params.R") +source('/esarchive/scratch/aho/git/csoperational/R/get_regrid_params.R') archive <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/conf/archive_decadal.yml") recipe <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/modules/data_load/recipe_decadal.yml") @@ -50,6 +52,9 @@ source("modules/data_load/regrid.R") # will change to CSOperational # generate transform params for system and ref regrid_params <- get_regrid_params(recipe, archive$archive) + # Only if the time length in each chunk may differ that we need largest_dims_length to be TRUE. Otherwise, set FALSE to increase efficiency. + need_largest_dims_length <- ifelse(model == 'EC-Earth3-i2', TRUE, FALSE) + #time for daily data. If recipe asks for time step, we don't need to calculate here # fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') @@ -72,7 +77,7 @@ source("modules/data_load/regrid.R") # will change to CSOperational time = indices(time), time_across = 'chunk', merge_across_dims = TRUE, - largest_dims_length = TRUE, + largest_dims_length = need_largest_dims_length, latitude = values(list(lat_min, lat_max)), latitude_reorder = Sort(decreasing = TRUE), longitude = values(list(lon_min, lon_max)), @@ -120,6 +125,9 @@ source("modules/data_load/regrid.R") # will change to CSOperational dates <- attr(hcst, 'Variables')$common$time +#//////////////// +# Method 1: use hcst time attr as obs time selector +#//////////////// obs <- Start(dat = file.path(path.obs, files.obs), var = var, file_date = unique(format(dates, '%Y%m')), @@ -142,6 +150,40 @@ source("modules/data_load/regrid.R") # will change to CSOperational time = 'file_date'), retrieve = TRUE) +#//////////////// +# Method 2: reshape hcst time attr's date into an array with time dim then as obs date selector +#//////////////// +dim(dates) +#syear time +# 2 14 +dates_arr <- array(format(dates, '%Y%m'), dim = dim(dates)) + + obs <- Start(dat = file.path(path.obs, files.obs), + var = var, + file_date = dates_arr, # [syear, time] +# time = dates, +# time_across = 'file_date', +# merge_across_dims = TRUE, + split_multiselected_dims = TRUE, + latitude = values(list(lat_min, lat_max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lon_min, lon_max)), + longitude_reorder = circularsort, + transform = regrid_params$obs.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$obs.gridtype, #nc file + method = regrid_params$obs.gridmethod), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat','latitude'), + longitude = c('lon','longitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = 'file_date'), + retrieve = TRUE) +#PROBLEMMMMMMMMMMMM +dim(attr(obs, 'Variables')$common$time) +#syear time time time time +# 2 14 1 14 1 + #------------------------------------------- # Step 3. Verify the consistance btwn hcst and obs diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/recipe_decadal.yml index cb0664d7..fbe56079 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/recipe_decadal.yml @@ -8,8 +8,8 @@ Analysis: freq: monthly_mean #daily_mean Datasets: System: - name: CanESM5 - member: r1i1p2f1,r2i1p2f1,r3i1p2f1 #'all' + name: BCC-CSM2-MR #CanESM5 + member: r1i1p1f1,r2i1p1f1,r3i1p1f1 #'all' Multimodel: no Reference: name: JRA-55 #ERA5 -- GitLab From 0da16b1ecb90bf1b1a59df460a507966d5afa1b2 Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 11 Apr 2022 19:23:17 +0200 Subject: [PATCH 12/21] Modify loading module to be similar with seasonal. Test one daily data case. --- conf/archive_decadal.yml | 7 +- modules/data_load/load_decadal.R | 261 +++++++++++++----- .../{ => testing_recipes}/recipe_decadal.yml | 2 +- .../testing_recipes/recipe_decadal_daily.yml | 48 ++++ 4 files changed, 240 insertions(+), 78 deletions(-) rename modules/data_load/{ => testing_recipes}/recipe_decadal.yml (97%) create mode 100644 modules/data_load/testing_recipes/recipe_decadal_daily.yml diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index f2cc6405..584785d1 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -88,7 +88,7 @@ archive: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} #NOTE: It should be the lastest version with all the files inside - version: {"tas":"v20200504"} + version: {"tas":"v20200101"} # version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} daily_mean: grid: @@ -279,8 +279,9 @@ archive: ERA5: src: "recon/ecmwf/era5/" monthly_mean: {"tas":"_f1h-r1440x721cds", "prlr":"_f1h-r1440x721cds", "psl":"_f1h-r1440x721cds"} - daily_mean: - calendar: "360-day" + daily_mean: {"tas":"_f1h-r1440x721cds/", "rsds":"_f1h-r1440x721cds/", + "prlr":"_f1h-r1440x721cds/", "sfcWind":"_f1h-r1440x721cds/"} + calendar: "360-day" reference_grid: "/esarchive/recon/ecmwf/era5/monthly_mean/tas_f1h-r1440x721cds/tas_201805.nc" # ---- diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 400e5a31..b5d0417b 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -3,42 +3,122 @@ # 2. recipe.yml # 3. Load_decadal.R (V) setwd('/esarchive/scratch/aho/git/auto-s2s/') -library(yaml) -library(startR) -#Does not work now. Why?? -#source("https://earth.bsc.es/gitlab/es/csoperational/-/raw/master/R/get_regrid_params.R") -source('/esarchive/scratch/aho/git/csoperational/R/get_regrid_params.R') - - archive <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/conf/archive_decadal.yml") - recipe <- read_yaml("/esarchive/scratch/aho/git/auto-s2s/modules/data_load/recipe_decadal.yml") + +## TODO: remove paths to personal scratchs +source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") +source("/esarchive/scratch/vagudets/repos/cstools/R/s2dv_cube.R") +# Load required libraries/funs +source("modules/data_load/dates2load.R") +source("tools/libs.R") + +## TODO: Move to CSOperational +# Conversion from startR_array to s2dv_array +#------------------------------------------------------------------- +attr2array <- function(attr){ + return(array(as.vector(attr), dim(attr))) +} + +startR_to_s2dv <- function(startR_array){ + + dates_dims <- dim(Subset(startR_array, + along=c('dat','var', + 'latitude', 'longitude', 'ensemble'), + list(1,1,1,1,1), drop="selected")) + + #sdates_dims <- dim(Subset(startR_array, + # along=c('dat','var','time','sweek','sday', + # 'latitude', 'longitude', 'ensemble'), + # list(1,1,1,1,1,1,1,1), drop="selected")) + + dates_start <- attr(startR_array, 'Variables')$common$time + dates_end <- attr(startR_array, 'Variables')$common$time + #sdates <- unlist(attr(startR_array, 'FileSelectors')$dat1$file_date) + + dim(dates_start) <- dates_dims + dim(dates_end) <- dates_dims + #dim(sdates) <- sdates_dims + + ## TODO: change this line when time attributes work correctly? + time_dims <- c("time", "sday", "sweek", "syear", "fcst_syear") + + s2dv_object <- s2dv_cube(data = attr2array(startR_array), + lon = attr2array(attr(startR_array, + 'Variables')$common$longitude), + lat = attr2array(attr(startR_array, + 'Variables')$common$latitude), + Variable = list(varName = names(attr(startR_array, 'Variables')$common)[4], + level = NULL), + Dates = list(start = dates_start, + end = dates_end), + #sdate = sdates), + time_dims = time_dims, + when = Sys.time(), + source_files = attr2array(attr(startR_array, "Files")) + #Datasets=list(exp1=list(InitializationsDates=list(Member_1="01011990", + # Members="Member_1"))) + ) + return(s2dv_object) +} + +#==================================================================== +#==================================================================== + +# recipe_file <- "modules/data_load/testing_recipes/recipe_decadal.yml" +# recipe_file <- "modules/data_load/testing_recipes/recipe_decadal_daily.yml" + +load_datasets <- function(recipe_file) { + + recipe <- read_yaml(recipe_file) + recipe$filename <- recipe_file + archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive + + #------------------------- # Read from recipe: #------------------------- - model <- recipe$Analysis$Datasets$System$name #'HadGEM3' - reference <- recipe$Analysis$Datasets$Reference$name #'era5' + exp.name <- recipe$Analysis$Datasets$System$name #'HadGEM3' + ref.name <- recipe$Analysis$Datasets$Reference$name #'era5' member <- strsplit(recipe$Analysis$Datasets$System$member, ',')[[1]] #c("r1i1p1f2", "r2i1p1f2") - var <- recipe$Analysis$Variables$name #'tas' - freq <- recipe$Analysis$Variables$freq #monthly_mean - lat_min <- as.numeric(recipe$Analysis$Region$latmin) #0 - lat_max <- as.numeric(recipe$Analysis$Region$latmax) #10 - lon_min <- as.numeric(recipe$Analysis$Region$lonmin) #0 - lon_max <- as.numeric(recipe$Analysis$Region$lonmax) #10 + variable <- recipe$Analysis$Variables$name #'tas' + store.freq <- recipe$Analysis$Variables$freq #monthly_mean + lats.min <- as.numeric(recipe$Analysis$Region$latmin) #0 + lats.max <- as.numeric(recipe$Analysis$Region$latmax) #10 + lons.min <- as.numeric(recipe$Analysis$Region$lonmin) #0 + lons.max <- as.numeric(recipe$Analysis$Region$lonmax) #10 + + # change to: sdates <- dates2load(recipe, logger) sdates <- as.numeric(recipe$Analysis$Time$sdate$start):as.numeric(recipe$Analysis$Time$sdate$end) #1960:2015 # fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 + + # change to: +# idxs <- NULL +# idxs$hcst <- get_timeidx(sdates$hcst, +# recipe$Analysis$Time$leadtimemin, +# recipe$Analysis$Time$leadtimemax, +# time_freq=store.freq) time <- as.numeric(recipe$Analysis$Time$leadtimemin):as.numeric(recipe$Analysis$Time$leadtimemax) - regrid <- recipe$Analysis$Regrid$type # to_system + #TODO: daily data + # If daily data & time is "month", need a function to calculate the indices. + # E.g., time = 1:4 means Dec to March, then indices will be c(31:(31+30+31+28+31)) for non-leap yr and c(31:(31+30+31+28), (31+30+31+28+2):(31+30+31+28+2+30)) for leap yr. + if (store.freq == "daily_mean") { + # for 1993-1994, Dec to March + time <- c(31:(31+31+31+28+30)) + } - # only used for daily data season <- recipe$Analysis$Time$season #------------------------- # Read from archive: #------------------------- - table <- archive$archive$System[[model]][[freq]]$table[[var]] #'Amon' - grid <- archive$archive$System[[model]][[freq]]$grid[[var]] - version <- archive$archive$System[[model]][[freq]]$version[[var]] + if (store.freq == "monthly_mean") { + table <- archive$System[[exp.name]][[store.freq]]$table[[variable]] #'Amon' + } else { + table <- 'day' + } + grid <- archive$System[[exp.name]][[store.freq]]$grid[[variable]] + version <- archive$System[[exp.name]][[store.freq]]$version[[variable]] if (identical(member, 'all')) { - member <- strsplit(archive$archive$System[[model]]$member, ',')[[1]] + member <- strsplit(archive$System[[exp.name]]$member, ',')[[1]] } #------------------------- @@ -50,10 +130,10 @@ source('/esarchive/scratch/aho/git/csoperational/R/get_regrid_params.R') circularsort <- CircularSort(-180, 180) } # generate transform params for system and ref - regrid_params <- get_regrid_params(recipe, archive$archive) + regrid_params <- get_regrid_params(recipe, archive) # Only if the time length in each chunk may differ that we need largest_dims_length to be TRUE. Otherwise, set FALSE to increase efficiency. - need_largest_dims_length <- ifelse(model == 'EC-Earth3-i2', TRUE, FALSE) + need_largest_dims_length <- ifelse(exp.name == 'EC-Earth3-i2', TRUE, FALSE) #time for daily data. If recipe asks for time step, we don't need to calculate here # fdays <- get_days_dcpp(fyears = fyears, season = season, init = 'Nov', calendar = '360-days') @@ -63,24 +143,26 @@ source('/esarchive/scratch/aho/git/csoperational/R/get_regrid_params.R') # Step 1: Load the exp #------------------------------------------- #monthly and daily - path <- file.path(archive$archive$src, archive$archive$System[[model]]$src, - '$ensemble$', table, '$var$', grid, version) - files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') + hcst.path <- file.path(archive$src, archive$System[[exp.name]]$src, + '$ensemble$', table, '$var$', grid, version) + hcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') -#monthly - hcst <- Start(dat = file.path(path, files), - var = var, + if (store.freq == "monthly_mean") { + #monthly + hcst <- Start(dat = file.path(hcst.path, hcst.files), + var = variable, ensemble = member, syear = paste0(sdates), +#split_multiselected_dims = T, chunk = 'all', chunk_depends = 'syear', time = indices(time), time_across = 'chunk', merge_across_dims = TRUE, largest_dims_length = need_largest_dims_length, - latitude = values(list(lat_min, lat_max)), + latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lon_min, lon_max)), + longitude = values(list(lons.min, lons.max)), longitude_reorder = circularsort, transform = regrid_params$fcst.transform, transform_extra_cells = 2, @@ -93,16 +175,25 @@ source('/esarchive/scratch/aho/git/csoperational/R/get_regrid_params.R') time = c('syear', 'chunk')), retrieve = T) + # change syear to c(sday, sweek, syear) + dim(hcst) <- c(dim(hcst)[1:3], sday = 1, sweek = 1, dim(hcst)[4:7]) + tmp_time_attr <- attr(hcst, 'Variables')$common$time + dim(attr(hcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(hcst)[6:7]) + + # Change class from startR_array to s2dv_cube + hcst <- startR_to_s2dv(hcst) + } else if (store.freq == "daily_mean") { + # Should be ok to use the call above!!!!!!!!!! #daily (not done yet) - data <- Start(dat = path, + data <- Start(dat = file.path(hcst.path, hcst.files), var = variable, syear = paste0(sdates), aux = 'all', aux_depends = 'syear', - latitude = values(list(lat_min, lat_max)), + latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lon_min, lon_max)), + longitude = values(list(lons.min, lons.max)), longitude_reorder = CircularSort(0, 360), fday = indices(fdays), ensemble = members, @@ -113,31 +204,44 @@ source('/esarchive/scratch/aho/git/csoperational/R/get_regrid_params.R') fday = c('syear', 'aux')), retrieve = F) + } #------------------------------------------- # Step 2. Load the reference #------------------------------------------- - path.obs <- file.path(archive$archive$src, archive$archive$Reference[[reference]]$src, freq, - paste0(var, archive$archive$Reference[[reference]][[freq]][[var]])) - files.obs <- paste0('$var$_$file_date$.nc') - + obs.path <- file.path(archive$src, archive$Reference[[ref.name]]$src, store.freq, + paste0(variable, archive$Reference[[ref.name]][[store.freq]][[variable]])) + obs.files <- paste0('$var$_$file_date$.nc') - dates <- attr(hcst, 'Variables')$common$time + # Get from startR_cube +# dates <- attr(hcst, 'Variables')$common$time + # Get from s2dv_cube + dates <- hcst$Dates$start + dates_file <- sapply(dates, format, '%Y%m') + dim(dates_file) <- dim(dates) + if (store.freq == "daily_mean") { #//////////////// # Method 1: use hcst time attr as obs time selector #//////////////// - obs <- Start(dat = file.path(path.obs, files.obs), - var = var, + + # Set hour to 12:00 to ensure correct date retrieval for daily data + lubridate::hour(dates) <- 12 + lubridate::minute(dates) <- 00 + # Restore correct dimensions + dim(dates) <- dim(dates_file) + + obs <- Start(dat = file.path(obs.path, obs.files), + var = variable, file_date = unique(format(dates, '%Y%m')), - time = dates, # [syear, time] + time = dates, # [sday, sweek, syear, time] time_across = 'file_date', merge_across_dims = TRUE, split_multiselected_dims = TRUE, - latitude = values(list(lat_min, lat_max)), + latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lon_min, lon_max)), + longitude = values(list(lons.min, lons.max)), longitude_reorder = circularsort, transform = regrid_params$obs.transform, transform_extra_cells = 2, @@ -150,24 +254,18 @@ source('/esarchive/scratch/aho/git/csoperational/R/get_regrid_params.R') time = 'file_date'), retrieve = TRUE) + } else if (store.freq == "monthly_mean") { #//////////////// # Method 2: reshape hcst time attr's date into an array with time dim then as obs date selector #//////////////// -dim(dates) -#syear time -# 2 14 -dates_arr <- array(format(dates, '%Y%m'), dim = dim(dates)) - - obs <- Start(dat = file.path(path.obs, files.obs), - var = var, - file_date = dates_arr, # [syear, time] -# time = dates, -# time_across = 'file_date', -# merge_across_dims = TRUE, + + obs <- Start(dat = file.path(obs.path, obs.files), + var = variable, + file_date = dates_file, #dates_arr, # [sday, sweek, syear, time] split_multiselected_dims = TRUE, - latitude = values(list(lat_min, lat_max)), + latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lon_min, lon_max)), + longitude = values(list(lons.min, lons.max)), longitude_reorder = circularsort, transform = regrid_params$obs.transform, transform_extra_cells = 2, @@ -179,39 +277,54 @@ dates_arr <- array(format(dates, '%Y%m'), dim = dim(dates)) return_vars = list(latitude = NULL, longitude = NULL, time = 'file_date'), retrieve = TRUE) -#PROBLEMMMMMMMMMMMM + } + + dim(attr(obs, 'Variables')$common$time) -#syear time time time time -# 2 14 1 14 1 +# sday sweek syear time +# 1 1 2 14 + + + # TODO: Reorder obs dims to match hcst dims? + # Adds ensemble dim to obs (for consistency with hcst/fcst) + default_dims <- c(dat = 1, var = 1, sweek = 1, + sday = 1, syear = 1, time = 1, + latitude = 1, longitude = 1, ensemble = 1) + default_dims[names(dim(obs))] <- dim(obs) + dim(obs) <- default_dims + + + # Change class from startR_array to s2dv_cube + obs <- startR_to_s2dv(obs) + #------------------------------------------- # Step 3. Verify the consistance btwn hcst and obs #------------------------------------------- # dimension - if (any(!names(dim(obs)) %in% names(dim(hcst)))) { + if (any(!names(dim(obs$data)) %in% names(dim(hcst$data)))) { stop("hcst and obs don't share the same dimension names.") } else { - match_ind <- match(names(dim(obs)), names(dim(hcst))) - if (!all(dim(hcst)[match_ind] == dim(obs))) stop("hcst and obs don't share the same dimension length.") + ens_ind <- which(names(dim(obs$data)) == 'ensemble') + match_ind <- match(names(dim(obs$data))[-ens_ind], names(dim(hcst$data))) + if (!all(dim(hcst$data)[match_ind] == dim(obs$data)[-ens_ind])) stop("hcst and obs don't share the same dimension length.") } # time attribute - if (!identical(format(attr(hcst, 'Variables')$common$time, '%Y%m'), - format(attr(obs, 'Variables')$common$time, '%Y%m'))) + if (!identical(format(hcst$Dates$start, '%Y%m'), + format(obs$Dates$start, '%Y%m'))) stop("hcst and obs don't share the same time.") # lat and lon attributes - if (!identical(as.vector(attr(hcst, 'Variables')$common$latitude), - as.vector(attr(obs, 'Variables')$common$latitude))) + if (!identical(as.vector(hcst$lat), + as.vector(obs$lat))) stop("hcst and obs don't share the same latitude.") - if (!identical(as.vector(attr(hcst, 'Variables')$common$longitude), - as.vector(attr(obs, 'Variables')$common$longitude))) + if (!identical(as.vector(hcst$lon), + as.vector(obs$lon))) stop("hcst and obs don't share the same longitude.") -#------------------------------------------- -# Step 4. Convert from startR_array to s2dv_array -#------------------------------------------- - + return(list(hcst = hcst, fcst = fcst, obs = obs)) +} diff --git a/modules/data_load/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml similarity index 97% rename from modules/data_load/recipe_decadal.yml rename to modules/data_load/testing_recipes/recipe_decadal.yml index fbe56079..5306cefd 100644 --- a/modules/data_load/recipe_decadal.yml +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -38,7 +38,7 @@ Analysis: Skill: metric: RPSS Indicators: - index: no + index: FALSE Output_format: S2S4E Run: Loglevel: INFO diff --git a/modules/data_load/testing_recipes/recipe_decadal_daily.yml b/modules/data_load/testing_recipes/recipe_decadal_daily.yml new file mode 100644 index 00000000..b3e6ade9 --- /dev/null +++ b/modules/data_load/testing_recipes/recipe_decadal_daily.yml @@ -0,0 +1,48 @@ +Description: + Author: An-Chi Ho + '': split version +Analysis: + Horizon: Decadal + Variables: + name: tas + freq: daily_mean + Datasets: + System: + name: EC-Earth3-i4 #BCC-CSM2-MR #CanESM5 + member: r1i4p1f1,r2i4p1f1,r3i4p1f1 #'all' + Multimodel: no + Reference: + name: ERA5 + Time: + sdate: + start: 1993 + end: 1994 #2018 +# fyear: +# start: 1 +# end: 1 + season: 'Annual' +# leadtime: + leadtimemin: 1 + leadtimemax: 4 + Region: + latmin: 10 #-90 + latmax: 20 #90 + lonmin: 0 + lonmax: 15 #359.9 + Regrid: + method: bilinear + type: to_system #to_reference + Workflow: + Calibration: + method: SBC + Skill: + metric: RPSS + Indicators: + index: FALSE + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/aho/git/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/aho/git/auto-s2s/ + -- GitLab From 5e04fa3465ddc1e029811882b038d461545dfe80 Mon Sep 17 00:00:00 2001 From: aho Date: Wed, 13 Apr 2022 13:18:01 +0200 Subject: [PATCH 13/21] Format modification --- modules/data_load/load_decadal.R | 38 +++++-------------- .../testing_recipes/recipe_decadal.yml | 6 +-- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index b5d0417b..883b1642 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -90,7 +90,8 @@ load_datasets <- function(recipe_file) { sdates <- as.numeric(recipe$Analysis$Time$sdate$start):as.numeric(recipe$Analysis$Time$sdate$end) #1960:2015 # fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 - # change to: + #TODO: change to: + #NOTE: Maybe not needed. # idxs <- NULL # idxs$hcst <- get_timeidx(sdates$hcst, # recipe$Analysis$Time$leadtimemin, @@ -124,7 +125,7 @@ load_datasets <- function(recipe_file) { #------------------------- # derived from above: #------------------------- - if (lon_min >= 0) { + if (lons.min >= 0) { circularsort <- CircularSort(0, 360) } else { circularsort <- CircularSort(-180, 180) @@ -147,8 +148,7 @@ load_datasets <- function(recipe_file) { '$ensemble$', table, '$var$', grid, version) hcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') - if (store.freq == "monthly_mean") { - #monthly + # monthly & daily hcst <- Start(dat = file.path(hcst.path, hcst.files), var = variable, ensemble = member, @@ -181,30 +181,9 @@ load_datasets <- function(recipe_file) { dim(attr(hcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(hcst)[6:7]) # Change class from startR_array to s2dv_cube + suppressWarnings( hcst <- startR_to_s2dv(hcst) - - } else if (store.freq == "daily_mean") { - # Should be ok to use the call above!!!!!!!!!! -#daily (not done yet) - data <- Start(dat = file.path(hcst.path, hcst.files), - var = variable, - syear = paste0(sdates), - aux = 'all', - aux_depends = 'syear', - latitude = values(list(lats.min, lats.max)), - latitude_reorder = Sort(decreasing = TRUE), - longitude = values(list(lons.min, lons.max)), - longitude_reorder = CircularSort(0, 360), - fday = indices(fdays), - ensemble = members, - synonims = list(fday = c('fday', 'time'), - longitude = c('lon', 'longitude'), - latitude = c('lat', 'latitude')), - return_vars = list(latitude = NULL, longitude = NULL, - fday = c('syear', 'aux')), - retrieve = F) - - } + ) #------------------------------------------- @@ -280,7 +259,7 @@ load_datasets <- function(recipe_file) { } -dim(attr(obs, 'Variables')$common$time) +#dim(attr(obs, 'Variables')$common$time) # sday sweek syear time # 1 1 2 14 @@ -295,8 +274,9 @@ dim(attr(obs, 'Variables')$common$time) # Change class from startR_array to s2dv_cube + suppressWarnings( obs <- startR_to_s2dv(obs) - + ) #------------------------------------------- diff --git a/modules/data_load/testing_recipes/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml index 5306cefd..501f699e 100644 --- a/modules/data_load/testing_recipes/recipe_decadal.yml +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -12,11 +12,11 @@ Analysis: member: r1i1p1f1,r2i1p1f1,r3i1p1f1 #'all' Multimodel: no Reference: - name: JRA-55 #ERA5 + name: ERA5 #JRA-55 Time: sdate: - start: 1960 - end: 1961 #2018 + start: 1961 + end: 1971 #2018 # fyear: # start: 1 # end: 1 -- GitLab From 2c39071b14585556104200607040983584a247ff Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 30 May 2022 09:12:57 +0200 Subject: [PATCH 14/21] add EC-Earth3-i4 forecast --- conf/archive_decadal.yml | 7 +- modules/data_load/load_decadal.R | 93 ++++++++++++++++--- .../testing_recipes/recipe_decadal.yml | 15 +-- 3 files changed, 91 insertions(+), 24 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index 584785d1..d724504a 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -3,7 +3,7 @@ archive: System: # ---- EC-Earth3-i1: - src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? + src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" monthly_mean: #NOTE: tos is under both Amon and Omon --> wait to be changed table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} @@ -37,7 +37,10 @@ archive: # ---- EC-Earth3-i4: - src: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? +# src: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? + src: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/" +# src: {"1960:2020": "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/", +# "2021:2021": "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppB-forecast/"} monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "clt":"Amon", "hfls":"Amon", "hurs":"Amon", "huss":"Amon", "rsds":"Amon", "rsut":"Amon", "ta":"Amon", diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 883b1642..098b7d1b 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -87,9 +87,9 @@ load_datasets <- function(recipe_file) { lons.max <- as.numeric(recipe$Analysis$Region$lonmax) #10 # change to: sdates <- dates2load(recipe, logger) - sdates <- as.numeric(recipe$Analysis$Time$sdate$start):as.numeric(recipe$Analysis$Time$sdate$end) #1960:2015 -# fyear <- as.numeric(recipe$Analysis$Time$fyear$start):as.numeric(recipe$Analysis$Time$fyear$end) #1:1 - + sdates_hcst <- as.numeric(recipe$Analysis$Time$hcst_start):as.numeric(recipe$Analysis$Time$hcst_end) #1960:2015 + sdates_fcst <- as.numeric(recipe$Analysis$Time$fcst) + #TODO: change to: #NOTE: Maybe not needed. # idxs <- NULL @@ -141,10 +141,10 @@ load_datasets <- function(recipe_file) { #------------------------------------------- -# Step 1: Load the exp +# Step 1: Load the hcst #------------------------------------------- #monthly and daily - hcst.path <- file.path(archive$src, archive$System[[exp.name]]$src, + hcst.path <- file.path(archive$src, archive$System[[exp.name]]$src, 'dcppA-hindcast', '$ensemble$', table, '$var$', grid, version) hcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') @@ -152,7 +152,7 @@ load_datasets <- function(recipe_file) { hcst <- Start(dat = file.path(hcst.path, hcst.files), var = variable, ensemble = member, - syear = paste0(sdates), + syear = paste0(sdates_hcst), #split_multiselected_dims = T, chunk = 'all', chunk_depends = 'syear', @@ -178,6 +178,9 @@ load_datasets <- function(recipe_file) { # change syear to c(sday, sweek, syear) dim(hcst) <- c(dim(hcst)[1:3], sday = 1, sweek = 1, dim(hcst)[4:7]) tmp_time_attr <- attr(hcst, 'Variables')$common$time + if (!identical(dim(tmp_time_attr), dim(hcst)[6:7])) { + stop("hcst has problem in matching data and time attr dimension.") + } dim(attr(hcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(hcst)[6:7]) # Change class from startR_array to s2dv_cube @@ -185,9 +188,64 @@ load_datasets <- function(recipe_file) { hcst <- startR_to_s2dv(hcst) ) +#------------------------------------------- +# Step 2: Load the fcst +#------------------------------------------- + if (!is.null(recipe$Analysis$Time$fcst)) { + + #monthly and daily + fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src, 'dcppB-forecast', + '$ensemble$', table, '$var$', grid, version) + fcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') + + # monthly & daily + fcst <- Start(dat = file.path(fcst.path, fcst.files), + var = variable, + ensemble = member, + syear = paste0(sdates_fcst), +#split_multiselected_dims = T, + chunk = 'all', + chunk_depends = 'syear', + time = indices(time), + time_across = 'chunk', + merge_across_dims = TRUE, + largest_dims_length = need_largest_dims_length, + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = TRUE), + longitude = values(list(lons.min, lons.max)), + longitude_reorder = circularsort, + transform = regrid_params$fcst.transform, + transform_extra_cells = 2, + transform_params = list(grid = regrid_params$fcst.gridtype, #nc file + method = regrid_params$fcst.gridmethod), + transform_vars = c('latitude', 'longitude'), + synonims = list(longitude = c('lon', 'longitude'), + latitude = c('lat', 'latitude')), + return_vars = list(latitude = NULL, longitude = NULL, + time = c('syear', 'chunk')), + retrieve = T) + + # change syear to c(sday, sweek, syear) + dim(fcst) <- c(dim(fcst)[1:3], sday = 1, sweek = 1, dim(fcst)[4:7]) + tmp_time_attr <- attr(fcst, 'Variables')$common$time + if (!identical(dim(tmp_time_attr), dim(fcst)[6:7])) { + stop("fcst has problem in matching data and time attr dimension.") + } + dim(attr(fcst, 'Variables')$common$time) <- c(sday = 1, sweek = 1, dim(fcst)[6:7]) + + # Change class from startR_array to s2dv_cube + suppressWarnings( + fcst <- startR_to_s2dv(fcst) + ) + + if (!identical(dim(hcst$data)[-6], dim(fcst$data)[-6])) { + stop("hcst and fcst do not share the same dimension structure.") + } + + } #------------------------------------------- -# Step 2. Load the reference +# Step 3. Load the reference #------------------------------------------- obs.path <- file.path(archive$src, archive$Reference[[ref.name]]$src, store.freq, paste0(variable, archive$Reference[[ref.name]][[store.freq]][[variable]])) @@ -264,14 +322,19 @@ load_datasets <- function(recipe_file) { # 1 1 2 14 - # TODO: Reorder obs dims to match hcst dims? - # Adds ensemble dim to obs (for consistency with hcst/fcst) - default_dims <- c(dat = 1, var = 1, sweek = 1, - sday = 1, syear = 1, time = 1, - latitude = 1, longitude = 1, ensemble = 1) - default_dims[names(dim(obs))] <- dim(obs) - dim(obs) <- default_dims +# # TODO: Reorder obs dims to match hcst dims? +# # Adds ensemble dim to obs (for consistency with hcst/fcst) +# default_dims <- c(dat = 1, var = 1, sweek = 1, +# sday = 1, syear = 1, time = 1, +# latitude = 1, longitude = 1, ensemble = 1) +# default_dims[names(dim(obs))] <- dim(obs) +# dim(obs) <- default_dims + if (!identical(dim(obs), dim(hcst$data)[-3])) { + stop("obs and hcst dimensions do not match.") + } + # Add ensemble dim to obs + dim(obs) <- c(dim(obs)[1:2], ensemble = 1, dim(obs)[3:8]) # Change class from startR_array to s2dv_cube suppressWarnings( @@ -280,7 +343,7 @@ load_datasets <- function(recipe_file) { #------------------------------------------- -# Step 3. Verify the consistance btwn hcst and obs +# Step 4. Verify the consistance btwn hcst and obs #------------------------------------------- # dimension if (any(!names(dim(obs$data)) %in% names(dim(hcst$data)))) { diff --git a/modules/data_load/testing_recipes/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml index 501f699e..4ad4ad60 100644 --- a/modules/data_load/testing_recipes/recipe_decadal.yml +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -8,18 +8,19 @@ Analysis: freq: monthly_mean #daily_mean Datasets: System: - name: BCC-CSM2-MR #CanESM5 - member: r1i1p1f1,r2i1p1f1,r3i1p1f1 #'all' + name: EC-Earth3-i4 #BCC-CSM2-MR #CanESM5 + member: 'all' #r1i1p1f1,r2i1p1f1,r3i1p1f1 Multimodel: no Reference: name: ERA5 #JRA-55 Time: sdate: - start: 1961 - end: 1971 #2018 -# fyear: -# start: 1 -# end: 1 + fcst: 2021 +# fcst_sday: '1101' + hcst_start: 2010 #'1993' + hcst_end: 2020 #'2016' +# start: 1961 +# end: 1971 #2018 season: 'Annual' # leadtime: leadtimemin: 1 -- GitLab From 7aceeaf0e99bf15355be776d99437677d71c4c7e Mon Sep 17 00:00:00 2001 From: aho Date: Wed, 8 Jun 2022 09:53:27 +0200 Subject: [PATCH 15/21] Add all the forecasts --- conf/archive_decadal.yml | 80 +++++++++++++------ modules/data_load/load_decadal.R | 60 ++------------ .../testing_recipes/recipe_decadal.yml | 8 +- 3 files changed, 63 insertions(+), 85 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index d724504a..ba62dc74 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -3,7 +3,9 @@ archive: System: # ---- EC-Earth3-i1: - src: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + src: + hcst: "exp/ecearth/a1ua/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + fcst: monthly_mean: #NOTE: tos is under both Amon and Omon --> wait to be changed table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tos":["Amon", "Omon"]} @@ -21,7 +23,9 @@ archive: #NOTE: EC-Earth3-i2 the first file of each sdate has 2 time step only (Nov-Dec). # The rest files are Jan to Dec. EC-Earth3-i2: - src: "exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/ec-earth3/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon"} grid: {"tas":"gr"} @@ -37,8 +41,9 @@ archive: # ---- EC-Earth3-i4: -# src: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" #also dcppB-forecast? - src: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/" + src: + hcst: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/" + fcst: "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppB-forecast/" # src: {"1960:2020": "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppA-hindcast/", # "2021:2021": "exp/ecearth/a3w5/original_files/cmorfiles/DCPP/EC-Earth-Consortium/EC-Earth3/dcppB-forecast/"} monthly_mean: @@ -70,7 +75,9 @@ archive: # ---- HadGEM3-GC31-MM: - src: "exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/" + fcst: "exp/CMIP6/dcppB-forecast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppB-forecast/" monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon", "tos":"Omon"} grid: {"tas":"gr", "psl":"gr", "pr":"gr", "ts":"gr", "tos":"gr"} @@ -82,11 +89,13 @@ archive: calendar: "360-day" member: r1i1p1f2,r2i1p1f2,r3i1p1f2,r4i1p1f2,r5i1p1f2,r6i1p1f2,r7i1p1f2,r8i1p1f2,r9i1p1f2,r10i1p1f2 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Amon/tas/gr/v20200316/tas_Amon_HadGEM3_dcppA-hindcast_s2018-r1i1p1f2_gr_201811-202903.nc" #'r432x324' + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Amon/tas/gr/v20200316/tas_Amon_HadGEM3_dcppA-hindcast_s2018-r1i1p1f2_gr_201811-202903.nc" #'r432x324' # ---- BCC-CSM2-MR: - src: "exp/CMIP6/dcppA-hindcast/bcc-csm2-mr/cmip6-dcppA-hindcast_i1p1/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/BCC-CSM2-MR/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} @@ -99,11 +108,13 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 initial_month: 1 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/bcc-csm2-mr/cmip6-dcppA-hindcast_i1p1/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r8i1p1f1/Amon/tas/gn/v20200101/tas_Amon_BCC-CSM2-MR_dcppA-hindcast_s2008-r8i1p1f1_gn_200801-201712.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/BCC-CSM2-MR/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r8i1p1f1/Amon/tas/gn/v20200101/tas_Amon_BCC-CSM2-MR_dcppA-hindcast_s2008-r8i1p1f1_gn_200801-201712.nc" # ---- CanESM5: - src: "exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/" + src: + hcst: "exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/" + fcst: "exp/canesm5/cmip6-dcppB-forecast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppB-forecast/" monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} @@ -119,7 +130,9 @@ archive: # ---- CESM1-1-CAM5-CMIP5: - src: "exp/ncar/cesm-dple-dcppA-hindcast/cmorfiles/DCPP/NCAR/CESM1-1-CAM5-CMIP5/dcppA-hindcast" + src: + hcst: "exp/ncar/cesm-dple-dcppA-hindcast/cmorfiles/DCPP/NCAR/CESM1-1-CAM5-CMIP5/dcppA-hindcast" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon"} grid: {"tas":"gn", "pr":"gn"} @@ -134,7 +147,9 @@ archive: # ---- CMCC-CM2-SR5: - src: "exp/CMIP6/dcppA-hindcast/cmcc-cm2-sr5/cmip6-dcppA-hindcast_i1p1/DCPP/CMCC-CM2-SR5/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/" + fcst: "exp/CMIP6/dcppB-forecast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppB-forecast/" monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "prc":"Amon", "ts":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "prc":"gn", "ts":"gn"} @@ -145,7 +160,7 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/cmcc-cm2-sr5/cmip6-dcppA-hindcast_i1p1/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20210312/tas_Amon_CMCC-CM2-SR5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20210312/tas_Amon_CMCC-CM2-SR5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- #QUESTION: missing in spreadsheet @@ -165,7 +180,9 @@ archive: # ---- IPSL-CM6A-LR: - src: "exp/CMIP6/dcppA-hindcast/ipsl-cm6a-lr/cmip6-dcppA-hindcast_i1p1/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/IPSL-CM6A-LR/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "sfcWind":"Amon"} grid: {"tas":"gr", "pr":"gr", "psl":"gr", "sfcWind":"gr"} @@ -176,11 +193,13 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 1 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/ipsl-cm6a-lr/cmip6-dcppA-hindcast_i1p1/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20200504/tas_Amon_IPSL-CM6A-LR_dcppA-hindcast_s2008-r1i1p1f1_gr_200901-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/IPSL-CM6A-LR/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20200504/tas_Amon_IPSL-CM6A-LR_dcppA-hindcast_s2008-r1i1p1f1_gr_200901-201812.nc" # ---- MIROC6: - src: "exp/CMIP6/dcppA-hindcast/miroc6/cmip6-dcppA-hindcast_i1p1/DCPP/MIROC/MIROC6/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/MIROC6/DCPP/MIROC/MIROC6/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} @@ -191,11 +210,13 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/miroc6/cmip6-dcppA-hindcast_i1p1/DCPP/MIROC/MIROC6/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200417/tas_Amon_MIROC6_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MIROC6/DCPP/MIROC/MIROC6/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200417/tas_Amon_MIROC6_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- MPI-ESM1.2-HR: - src: "exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "tasmin":"Amon", "tasmax":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} @@ -206,11 +227,13 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- MPI-ESM1.2-LR: - src: "exp/CMIP6/dcppA-hindcast/mpi-esm1-2-lr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-LR/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon", "ts":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn", "ts":"gn"} @@ -221,11 +244,13 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1,r11i1p1f1,r12i1p1f1,r13i1p1f1,r14i1p1f1,r15i1p1f1,r16i1p1f1 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-lr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MPI-ESM1-2-LR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-LR/DCPP/MPI-M/MPI-ESM1-2-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MPI-ESM1-2-LR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" # ---- MRI-ESM2-0: - src: "exp/CMIP6/dcppA-hindcast/mri-esm2-0/cmip6-dcppA-hindcast_i1p1/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/MRI-ESM2-0/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} @@ -236,11 +261,14 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/mri-esm2-0/cmip6-dcppA-hindcast_i1p1/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MRI-ESM2-0_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201312.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MRI-ESM2-0/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MRI-ESM2-0_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201312.nc" # ---- + #NOTE: NorCPM1-i1 and i2 are under the same directory NorCPM1-i1: - src: "exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i1p1/DCPP/NCC/NorCPM1/dcppA-hindcast/" + src: + hcst: "exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/" + fcst: monthly_mean: table: {"tas":"Amon", "pr":"Amon", "psl":"Amon"} grid: {"tas":"gn", "pr":"gn", "psl":"gn"} @@ -251,11 +279,11 @@ archive: calendar: member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 10 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i1p1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_NorCPM1_dcppA-hindcast_s2008-r1i1p1f1_gn_200810-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_NorCPM1_dcppA-hindcast_s2008-r1i1p1f1_gn_200810-201812.nc" # ---- NorCPM1-i2: - src: "exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i2p1/DCPP/NCC/NorCPM1/dcppA-hindcast/" + src: "exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/" monthly_mean: table: {"pr":"Amon", "psl":"Amon"} grid: {"pr":"gn", "psl":"gn"} @@ -266,7 +294,7 @@ archive: calendar: member: r1i2p1f1,r2i2p1f1,r3i2p1f1,r4i2p1f1,r5i2p1f1,r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 initial_month: 10 - reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/norcpm1/cmip6-dcppA-hindcast_i2p1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" + reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" # ===================================== diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 098b7d1b..e58176e9 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -11,56 +11,6 @@ source("/esarchive/scratch/vagudets/repos/cstools/R/s2dv_cube.R") source("modules/data_load/dates2load.R") source("tools/libs.R") -## TODO: Move to CSOperational -# Conversion from startR_array to s2dv_array -#------------------------------------------------------------------- -attr2array <- function(attr){ - return(array(as.vector(attr), dim(attr))) -} - -startR_to_s2dv <- function(startR_array){ - - dates_dims <- dim(Subset(startR_array, - along=c('dat','var', - 'latitude', 'longitude', 'ensemble'), - list(1,1,1,1,1), drop="selected")) - - #sdates_dims <- dim(Subset(startR_array, - # along=c('dat','var','time','sweek','sday', - # 'latitude', 'longitude', 'ensemble'), - # list(1,1,1,1,1,1,1,1), drop="selected")) - - dates_start <- attr(startR_array, 'Variables')$common$time - dates_end <- attr(startR_array, 'Variables')$common$time - #sdates <- unlist(attr(startR_array, 'FileSelectors')$dat1$file_date) - - dim(dates_start) <- dates_dims - dim(dates_end) <- dates_dims - #dim(sdates) <- sdates_dims - - ## TODO: change this line when time attributes work correctly? - time_dims <- c("time", "sday", "sweek", "syear", "fcst_syear") - - s2dv_object <- s2dv_cube(data = attr2array(startR_array), - lon = attr2array(attr(startR_array, - 'Variables')$common$longitude), - lat = attr2array(attr(startR_array, - 'Variables')$common$latitude), - Variable = list(varName = names(attr(startR_array, 'Variables')$common)[4], - level = NULL), - Dates = list(start = dates_start, - end = dates_end), - #sdate = sdates), - time_dims = time_dims, - when = Sys.time(), - source_files = attr2array(attr(startR_array, "Files")) - #Datasets=list(exp1=list(InitializationsDates=list(Member_1="01011990", - # Members="Member_1"))) - ) - return(s2dv_object) -} - -#==================================================================== #==================================================================== # recipe_file <- "modules/data_load/testing_recipes/recipe_decadal.yml" @@ -144,7 +94,7 @@ load_datasets <- function(recipe_file) { # Step 1: Load the hcst #------------------------------------------- #monthly and daily - hcst.path <- file.path(archive$src, archive$System[[exp.name]]$src, 'dcppA-hindcast', + hcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$hcst, '$ensemble$', table, '$var$', grid, version) hcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') @@ -185,7 +135,7 @@ load_datasets <- function(recipe_file) { # Change class from startR_array to s2dv_cube suppressWarnings( - hcst <- startR_to_s2dv(hcst) + hcst <- as.s2dv_cube(hcst) ) #------------------------------------------- @@ -194,7 +144,7 @@ load_datasets <- function(recipe_file) { if (!is.null(recipe$Analysis$Time$fcst)) { #monthly and daily - fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src, 'dcppB-forecast', + fcst.path <- file.path(archive$src, archive$System[[exp.name]]$src$fcst, '$ensemble$', table, '$var$', grid, version) fcst.files <- paste0('$var$_', table, '_*_*_s$syear$-$ensemble$_', grid, '_$chunk$.nc') @@ -235,7 +185,7 @@ load_datasets <- function(recipe_file) { # Change class from startR_array to s2dv_cube suppressWarnings( - fcst <- startR_to_s2dv(fcst) + fcst <- as.s2dv_cube(fcst) ) if (!identical(dim(hcst$data)[-6], dim(fcst$data)[-6])) { @@ -338,7 +288,7 @@ load_datasets <- function(recipe_file) { # Change class from startR_array to s2dv_cube suppressWarnings( - obs <- startR_to_s2dv(obs) + obs <- as.s2dv_cube(obs) ) diff --git a/modules/data_load/testing_recipes/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml index 4ad4ad60..fa3c58e0 100644 --- a/modules/data_load/testing_recipes/recipe_decadal.yml +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -8,17 +8,17 @@ Analysis: freq: monthly_mean #daily_mean Datasets: System: - name: EC-Earth3-i4 #BCC-CSM2-MR #CanESM5 + name: BCC-CSM2-MR #EC-Earth3-i4 #CanESM5 member: 'all' #r1i1p1f1,r2i1p1f1,r3i1p1f1 Multimodel: no Reference: name: ERA5 #JRA-55 Time: sdate: - fcst: 2021 + fcst: #2021 # fcst_sday: '1101' - hcst_start: 2010 #'1993' - hcst_end: 2020 #'2016' + hcst_start: 1990 #'1993' + hcst_end: 1991 #'2016' # start: 1961 # end: 1971 #2018 season: 'Annual' -- GitLab From 1871b0062b45a10cb07c1a43e3d88f051ab89e1d Mon Sep 17 00:00:00 2001 From: aho Date: Wed, 8 Jun 2022 12:22:03 +0200 Subject: [PATCH 16/21] Add loading summary --- modules/data_load/load_decadal.R | 35 +++++++++++++++++-- .../testing_recipes/recipe_decadal.yml | 6 ++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index e58176e9..6b753f5e 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -22,7 +22,6 @@ load_datasets <- function(recipe_file) { recipe$filename <- recipe_file archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive_decadal.yml"))$archive - #------------------------- # Read from recipe: #------------------------- @@ -293,7 +292,39 @@ load_datasets <- function(recipe_file) { #------------------------------------------- -# Step 4. Verify the consistance btwn hcst and obs +# Step 4. Print the summary of data +#------------------------------------------- + + # Print a summary of the loaded data for the user, for each object + # (hcst, obs and fcst). + ## TODO: Incorporate into logger + ## TODO: Adapt to daily/subseasonal cases + ## TODO: Add check for missing files/NAs + data_summary <- function(object) { + # Get name and start dates + object_name <- deparse(substitute(object)) + sdate_min <- format(min(as.Date(object$Dates[[1]])), format = '%b %Y') + sdate_max <- format(max(as.Date(object$Dates[[1]])), format = '%b %Y') + + print("DATA SUMMARY:") + print(paste0(object_name, " range: ", sdate_min, " to ", sdate_max)) + print(paste0(object_name, " dimensions: ")) + print(dim(object$data)) + print(paste0("Statistical summary of the data in ", object_name, ":")) + print(summary(object$data)) + print("---------------------------------------------") + + } + + data_summary(hcst) + data_summary(obs) + if (!is.null(fcst)) { + data_summary(fcst) + } + + +#------------------------------------------- +# Step 5. Verify the consistance btwn hcst and obs #------------------------------------------- # dimension if (any(!names(dim(obs$data)) %in% names(dim(hcst$data)))) { diff --git a/modules/data_load/testing_recipes/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml index fa3c58e0..f69edc02 100644 --- a/modules/data_load/testing_recipes/recipe_decadal.yml +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -8,17 +8,17 @@ Analysis: freq: monthly_mean #daily_mean Datasets: System: - name: BCC-CSM2-MR #EC-Earth3-i4 #CanESM5 + name: EC-Earth3-i4 #CanESM5 member: 'all' #r1i1p1f1,r2i1p1f1,r3i1p1f1 Multimodel: no Reference: name: ERA5 #JRA-55 Time: sdate: - fcst: #2021 + fcst: 2021 # fcst_sday: '1101' hcst_start: 1990 #'1993' - hcst_end: 1991 #'2016' + hcst_end: 1992 #'2016' # start: 1961 # end: 1971 #2018 season: 'Annual' -- GitLab From 16ffd4e70da9e8a858fd97fdcfdc2b694649bd78 Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 20 Jun 2022 10:56:22 +0200 Subject: [PATCH 17/21] Revise recipes --- modules/data_load/load_decadal.R | 26 +++---------------- .../testing_recipes/recipe_decadal.yml | 4 +-- .../testing_recipes/recipe_decadal_daily.yml | 12 ++++----- 3 files changed, 10 insertions(+), 32 deletions(-) diff --git a/modules/data_load/load_decadal.R b/modules/data_load/load_decadal.R index 6b753f5e..c9da2da2 100644 --- a/modules/data_load/load_decadal.R +++ b/modules/data_load/load_decadal.R @@ -296,30 +296,10 @@ load_datasets <- function(recipe_file) { #------------------------------------------- # Print a summary of the loaded data for the user, for each object - # (hcst, obs and fcst). - ## TODO: Incorporate into logger - ## TODO: Adapt to daily/subseasonal cases - ## TODO: Add check for missing files/NAs - data_summary <- function(object) { - # Get name and start dates - object_name <- deparse(substitute(object)) - sdate_min <- format(min(as.Date(object$Dates[[1]])), format = '%b %Y') - sdate_max <- format(max(as.Date(object$Dates[[1]])), format = '%b %Y') - - print("DATA SUMMARY:") - print(paste0(object_name, " range: ", sdate_min, " to ", sdate_max)) - print(paste0(object_name, " dimensions: ")) - print(dim(object$data)) - print(paste0("Statistical summary of the data in ", object_name, ":")) - print(summary(object$data)) - print("---------------------------------------------") - - } - - data_summary(hcst) - data_summary(obs) + data_summary(hcst, store.freq) + data_summary(obs, store.freq) if (!is.null(fcst)) { - data_summary(fcst) + data_summary(fcst, store.freq) } diff --git a/modules/data_load/testing_recipes/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml index f69edc02..73bde46f 100644 --- a/modules/data_load/testing_recipes/recipe_decadal.yml +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -5,7 +5,7 @@ Analysis: Horizon: Decadal Variables: name: tas - freq: monthly_mean #daily_mean + freq: monthly_mean Datasets: System: name: EC-Earth3-i4 #CanESM5 @@ -35,7 +35,7 @@ Analysis: type: to_system #to_reference Workflow: Calibration: - method: SBC + method: bias Skill: metric: RPSS Indicators: diff --git a/modules/data_load/testing_recipes/recipe_decadal_daily.yml b/modules/data_load/testing_recipes/recipe_decadal_daily.yml index b3e6ade9..b3143035 100644 --- a/modules/data_load/testing_recipes/recipe_decadal_daily.yml +++ b/modules/data_load/testing_recipes/recipe_decadal_daily.yml @@ -15,13 +15,11 @@ Analysis: name: ERA5 Time: sdate: - start: 1993 - end: 1994 #2018 -# fyear: -# start: 1 -# end: 1 + fcst: 2021 +# fcst_sday: '1101' + hcst_start: 1993 + hcst_end: 1994 season: 'Annual' -# leadtime: leadtimemin: 1 leadtimemax: 4 Region: @@ -34,7 +32,7 @@ Analysis: type: to_system #to_reference Workflow: Calibration: - method: SBC + method: qmap Skill: metric: RPSS Indicators: -- GitLab From 9134fb07f7045b25de49b34e3a6aaa2fa3044cac Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 20 Jun 2022 14:34:03 +0200 Subject: [PATCH 18/21] Add daily_mean data --- conf/archive_decadal.yml | 73 ++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/conf/archive_decadal.yml b/conf/archive_decadal.yml index ba62dc74..e41d46a0 100644 --- a/conf/archive_decadal.yml +++ b/conf/archive_decadal.yml @@ -12,8 +12,8 @@ archive: grid: {"tas":"gr"} version: {"tas":"v20190713"} daily_mean: - grid: {"tas":"gr"} - version: {"tas":"v20190713"} + grid: {"clt":"gr", "hurs":"gr", "hursmin":"gr", "pr":"gr", "psl":"gr", "rsds":"gr", "sfcWind":"gr", "sfcWindmax":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} + version: {"clt":"v20190713", "hurs":"v20190713", "hursmin":"v20190713", "pr":"v20190713", "psl":"v20190713", "rsds":"v20190713", "sfcWind":"v20190713", "sfcWindmax":"v20190713", "tas":"v20190713", "tasmax":"v20190713", "tasmin":"v20190713"} calendar: "proleptic_gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 @@ -31,8 +31,8 @@ archive: grid: {"tas":"gr"} version: {"tas":"v20200730"} daily_mean: - grid: {"tas":"gr"} - version: {"tas":"v20200731"} + grid: {"pr":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} + version: {"pr":"v20200508", "tas":"v20200731", "tasmax":"v20200730", "tasmin":"v20200730"} calendar: "proleptic_gregorian" #NOTE:There are many members but not all of them are available on ESGF (only r6-10 available). Then, we might have some variables for the rest of the members (r1-5 and r11-15), but not for all the variables. That's why i'm only using r6-10 member: r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 @@ -103,9 +103,9 @@ archive: version: {"tas":"v20200101"} # version: {"tas":[v20191126, v20191213, v20191219, v20200110, v20200114, v20200504, v20191209, v20191218, v20200101, v20200113, v20200401]} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "sfcWind":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn", "uas":"gn", "vas":"gn"} + version: {"pr":"v20200114", "sfcWind":"v20200101", "tas":"v20200408", "tasmax":"v20200114", "tasmin":"v20200114", "uas":"v20200114", "vas":"v20200114"} + calendar: "365_day" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1 initial_month: 1 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/BCC-CSM2-MR/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r8i1p1f1/Amon/tas/gn/v20200101/tas_Amon_BCC-CSM2-MR_dcppA-hindcast_s2008-r8i1p1f1_gn_200801-201712.nc" @@ -121,9 +121,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} version: {"tas":"v20190429", "pr":"v20190429", "psl":"v20190429", "tasmin":"v20190429", "tasmax":"v20190429"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20190429", "tas":"v20190429", "tasmax":"v20190429", "tasmin":"v20190429"} + calendar: "365_day" member: r1i1p2f1,r2i1p2f1,r3i1p2f1,r4i1p2f1,r5i1p2f1,r6i1p2f1,r7i1p2f1,r8i1p2f1, r9i1p2f1, r10i1p2f1, r11i1p2f1,r12i1p2f1,r13i1p2f1,r14i1p2f1,r15i1p2f1,r16i1p2f1,r17i1p2f1,r18i1p2f1, r19i1p2f1, r20i1p2f1,r21i1p2f1,r22i1p2f1,r23i1p2f1,r24i1p2f1,r25i1p2f1,r26i1p2f1,r27i1p2f1,r28i1p2f1, r29i1p2f1, r30i1p2f1, r31i1p2f1,r32i1p2f1,r33i1p2f1,r34i1p2f1,r35i1p2f1,r36i1p2f1,r37i1p2f1,r38i1p2f1, r39i1p2f1, r40i1p2f1 initial_month: 13 #next year Jan reference_grid: "/esarchive/exp/canesm5/cmip6-dcppA-hindcast_i1p2/original_files/cmorfiles/DCPP/CCCma/CanESM5/dcppA-hindcast/r1i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_dcppA-hindcast_s2008-r1i1p2f1_gn_200901-201812.nc" @@ -155,9 +155,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn", "prc":"gn", "ts":"gn"} version: {"tas":"v20210312", "pr":"v20210312", "psl":"v20210312", "prc":"v20200101", "ts":"v20200101"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20210909", "tasmax":"v20210909", "tasmin":"v20210909"} + calendar: "365_day" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20210312/tas_Amon_CMCC-CM2-SR5_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -188,9 +188,9 @@ archive: grid: {"tas":"gr", "pr":"gr", "psl":"gr", "sfcWind":"gr"} version: {"tas":"v20200504", "pr":"v20200504", "psl":"v20200504", "sfcWind":"v20200504"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gr", "tas":"gr", "tasmax":"gr", "tasmin":"gr"} + version: {"pr":"v20200108", "tas":"v20200504", "tasmax":"v20200504", "tasmin":"v20200504"} + calendar: "gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 1 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/IPSL-CM6A-LR/DCPP/IPSL/IPSL-CM6A-LR/dcppA-hindcast/r1i1p1f1/Amon/tas/gr/v20200504/tas_Amon_IPSL-CM6A-LR_dcppA-hindcast_s2008-r1i1p1f1_gr_200901-201812.nc" @@ -205,9 +205,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} version: {"tas":"v20200417", "pr":["v20200416","v20200504"], "psl":"v20200504", "tasmin":"v20200417", "tasmax":"v20200504"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20191217", "tas":"v20200416", "tasmax":"v20200416", "tasmin":"v20200416"} + calendar: "standard" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MIROC6/DCPP/MIROC/MIROC6/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200417/tas_Amon_MIROC6_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -222,9 +222,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn", "tasmin":"gn", "tasmax":"gn"} version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320", "sfcWind":"v20200320"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20210128", "tas":"v20200320", "tasmax":"v20200101", "tasmin":"v20200101"} + calendar: "standard" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201812.nc" @@ -256,9 +256,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn"} version: {"tas":"v20200101", "pr":"v20200101", "psl":"v20200101"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20210122", "tasmax":"v20210122", "tasmin":"v20210122"} + calendar: "proleptic_gregorian" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 11 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/MRI-ESM2-0/DCPP/MRI/MRI-ESM2-0/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200101/tas_Amon_MRI-ESM2-0_dcppA-hindcast_s2008-r1i1p1f1_gn_200811-201312.nc" @@ -274,9 +274,9 @@ archive: grid: {"tas":"gn", "pr":"gn", "psl":"gn"} version: {"tas":"v20200320", "pr":"v20200320", "psl":"v20200320"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20191005", "tas":"v20200320", "tasmax":"v20191005", "tasmin":"v20191005"} + calendar: "noleap" member: r1i1p1f1,r2i1p1f1,r3i1p1f1,r4i1p1f1,r5i1p1f1,r6i1p1f1,r7i1p1f1,r8i1p1f1,r9i1p1f1,r10i1p1f1 initial_month: 10 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i1p1f1/Amon/tas/gn/v20200320/tas_Amon_NorCPM1_dcppA-hindcast_s2008-r1i1p1f1_gn_200810-201812.nc" @@ -289,9 +289,9 @@ archive: grid: {"pr":"gn", "psl":"gn"} version: {"pr":"v20200101", "psl":"v20200101"} daily_mean: - grid: - version: - calendar: + grid: {"pr":"gn", "tas":"gn", "tasmax":"gn", "tasmin":"gn"} + version: {"pr":"v20191005", "tas":"v20200101", "tasmax":"v20191005", "tasmin":"v20191005"} + calendar: "noleap" member: r1i2p1f1,r2i2p1f1,r3i2p1f1,r4i2p1f1,r5i2p1f1,r6i2p1f1,r7i2p1f1,r8i2p1f1,r9i2p1f1,r10i2p1f1 initial_month: 10 reference_grid: "/esarchive/exp/CMIP6/dcppA-hindcast/NorCPM1/DCPP/NCC/NorCPM1/dcppA-hindcast/r1i2p1f1/Amon/pr/gn/v20200101/pr_Amon_NorCPM1_dcppA-hindcast_s2008-r1i2p1f1_gn_200810-201812.nc" @@ -302,7 +302,7 @@ archive: Reference: GHCNv4: src: "obs/noaa/ghcn_v4/" - monthly_mean: {"tasanomaly":""} + monthly_mean: {"tas":"", "tasanomaly":""} daily_mean: reference_grid: "/esarchive/obs/noaa/ghcn_v4/monthly_mean/tasanomaly/tasanomaly_201811.nc" @@ -323,7 +323,7 @@ archive: JRA-55: src: "recon/jma/jra55/" monthly_mean: {"tas":"_f6h", "psl":"_f6h", "tos":"", "pr":"_s0-3h", "prlr":"_s0-3h"} - daily_mean: + daily_mean: {"tas":"_f6h", "psl":"_f6h", "prlr":"_s0-3h", "sfcWind":"_f6h"} reference_grid: "/esarchive/recon/jma/jra55/monthly_mean/tas_f6h/tas_200811.nc" # ---- @@ -340,13 +340,6 @@ archive: daily_mean: reference_grid: "/esarchive/obs/ukmo/hadcrut_v4.6/monthly_mean/tasanomaly/tasanomaly_200811.nc" -# ---- - GPCC: - src: "obs/noaa/gpcc-v2018/" - monthly_mean: {"prlr":""} - daily_mean: - reference_grid: "/esarchive/obs/noaa/gpcc-v2018/monthly_mean/prlr_200811.nc" - # ---- HadSLP2: src: "obs/ukmo/hadslp_v2/" -- GitLab From e5d89d0d21561a4e6dd32e49f49e53e5d9cd9532 Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 20 Jun 2022 14:46:50 +0200 Subject: [PATCH 19/21] Change file name --- modules/data_load/{load_decadal.R => Loading_decadal.R} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename modules/data_load/{load_decadal.R => Loading_decadal.R} (100%) diff --git a/modules/data_load/load_decadal.R b/modules/data_load/Loading_decadal.R similarity index 100% rename from modules/data_load/load_decadal.R rename to modules/data_load/Loading_decadal.R -- GitLab From e709ebf6fc98e8293b79d3053f52451b31e68d86 Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 20 Jun 2022 17:33:29 +0200 Subject: [PATCH 20/21] Creat check_latlon.R and refine recipe. --- tools/check_latlon.R | 90 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tools/check_latlon.R diff --git a/tools/check_latlon.R b/tools/check_latlon.R new file mode 100644 index 00000000..c317acd2 --- /dev/null +++ b/tools/check_latlon.R @@ -0,0 +1,90 @@ +# Check if the lat and lon inputs abide the regulation. +# +# (x) ←|-------------|-------------|-------------|-------------|-------------| →(x) +# -360 -180 0 180 360 540 +# |___________________________|___________________________| +# [0, 360] [0, 360] +# +# |___________________________|___________________________| +# [-180, 180] [-180, 180] +# Premise: lonmin < lonmax +# [Case 1: lonmin >= 0] +# 1-0: lonmax <360 →[0, 360] +# 1-1:lonmin >= 180 & lonmax > 360 →[-180, 180] +# 1-2:lonmin < 180 & lonmax > 360 →[0, 360] + warning* +# +# [Case 2: lonmax < 0] +# 2-0: lonmin >= -180 & lonmax < 180 →[-180, 180] +# 2-1: lonmin < -180 & lonmax < 0 →[0, 360] +# 2-2: lonmin < -180 & lonmax > 0 →[-180, 180] + warning* +# 2-3 lonmax > 180 →[-180, 180] + warning* +# +# *warning: The region is not one integrated box. + +check_latlon <- function(latmin, latmax, lonmin, lonmax) { + + if (any(!sapply(c(latmin, latmax, lonmin, lonmax), is.numeric)) | + any(sapply(c(latmin, latmax, lonmin, lonmax), length) != 1)) + stop("latmin, latmax, lonmin, and lonmax must be a number.") + # lat + if (any(c(latmin, latmax) < -90) | any(c(latmin, latmax) > 90)) + stop("latmin must be within [-90, 90].") + # lon + if (lonmin > lonmax) + stop("lonmax must not be smaller than lonmin.") + # Adjust lon if it is way out of scope + while (lonmin > 360 & lonmax > 360) { + lonmin <- lonmin - 360 + lonmax <- lonmax - 360 + } + while (lonmin < -180 & lonmax < -180) { + lonmin <- lonmin + 360 + lonmax <- lonmax + 360 + } + + warning_discrete_lon <- FALSE + warning_unconsidered_case <- FALSE + ## case 1 + if (lonmin >= 0) { + if (lonmax < 360) { ## 1-0 + circularsort <- CircularSort(0, 360) + } else if (lonmin >= 180 & lonmax > 360) { ## 1-1 + circularsort <- CircularSort(-180, 180) + if (lonmax >= 540) + warning_discrete_lon <- TRUE + } else if (lonmin < 180 & lonmax > 360) { ## 1-2 + circularsort <- CircularSort(0, 360) + warning_discrete_lon <- TRUE + } else { + circularsort <- CircularSort(0, 360) + warning_unconsidered_case <- TRUE + } + + } + ## case 2 + else { + if (lonmin >= -180 & lonmax < 180) { ## 2-0 + circularsort <- CircularSort(-180, 180) + } else if (lonmin < -180 & lonmax < 0) { ## 2-1 + circularsort <- CircularSort(0, 360) + if (lonmin < -360) + warning_discrete_lon <- TRUE + } else if (lonmin < -180 & lonmax > 0) { ## 2-2 + circularsort <- CircularSort(-180, 180) + warning_discrete_lon <- TRUE + } else if (lonmax > 180) { ## 2-3 + circularsort <- CircularSort(-180, 180) + warning_discrete_lon <- TRUE + } else { + circularsort <- CircularSort(-180, 180) + warning_unconsidered_case <- TRUE + } + } + + if (warning_discrete_lon) + warning("The longitude of returned data won't be continuous. Check the metadata to see the longitude range.") + if (warning_unconsidered_case) + warning("The longitude case is not being considered yet. Please report this case to maintainers and check the longitude of returned data.") + + return(circularsort) +} -- GitLab From 302a995ee43834cd7c3ff31cd8d9d7bb02b86a4a Mon Sep 17 00:00:00 2001 From: aho Date: Mon, 20 Jun 2022 17:34:25 +0200 Subject: [PATCH 21/21] Use check_latlon.R and refine recipe. --- modules/data_load/Loading_decadal.R | 18 +++++------------- .../testing_recipes/recipe_decadal.yml | 5 +---- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/modules/data_load/Loading_decadal.R b/modules/data_load/Loading_decadal.R index c9da2da2..97b18c01 100644 --- a/modules/data_load/Loading_decadal.R +++ b/modules/data_load/Loading_decadal.R @@ -39,13 +39,6 @@ load_datasets <- function(recipe_file) { sdates_hcst <- as.numeric(recipe$Analysis$Time$hcst_start):as.numeric(recipe$Analysis$Time$hcst_end) #1960:2015 sdates_fcst <- as.numeric(recipe$Analysis$Time$fcst) - #TODO: change to: - #NOTE: Maybe not needed. -# idxs <- NULL -# idxs$hcst <- get_timeidx(sdates$hcst, -# recipe$Analysis$Time$leadtimemin, -# recipe$Analysis$Time$leadtimemax, -# time_freq=store.freq) time <- as.numeric(recipe$Analysis$Time$leadtimemin):as.numeric(recipe$Analysis$Time$leadtimemax) #TODO: daily data # If daily data & time is "month", need a function to calculate the indices. @@ -55,7 +48,8 @@ load_datasets <- function(recipe_file) { time <- c(31:(31+31+31+28+30)) } - season <- recipe$Analysis$Time$season +#NOTE: May be used in the future +# season <- recipe$Analysis$Time$season #------------------------- # Read from archive: @@ -74,11 +68,9 @@ load_datasets <- function(recipe_file) { #------------------------- # derived from above: #------------------------- - if (lons.min >= 0) { - circularsort <- CircularSort(0, 360) - } else { - circularsort <- CircularSort(-180, 180) - } + # Check lat and lon and decide CircularSort + circularsort <- check_latlon(latmin = lats.min, latmax = lats.max, lonmin = lons.min, lonmax = lons.max) + # generate transform params for system and ref regrid_params <- get_regrid_params(recipe, archive) diff --git a/modules/data_load/testing_recipes/recipe_decadal.yml b/modules/data_load/testing_recipes/recipe_decadal.yml index 73bde46f..b7b6a109 100644 --- a/modules/data_load/testing_recipes/recipe_decadal.yml +++ b/modules/data_load/testing_recipes/recipe_decadal.yml @@ -19,10 +19,7 @@ Analysis: # fcst_sday: '1101' hcst_start: 1990 #'1993' hcst_end: 1992 #'2016' -# start: 1961 -# end: 1971 #2018 - season: 'Annual' -# leadtime: +# season: 'Annual' leadtimemin: 1 leadtimemax: 14 Region: -- GitLab