From 5887bf1a4cfca3b6a8cd721e68c1544d06cb71f6 Mon Sep 17 00:00:00 2001 From: lpalma Date: Mon, 4 Apr 2022 11:55:39 +0200 Subject: [PATCH 1/4] wrapped up load fun + testing --- modules/data_load/dates2load.R | 35 ++++++- modules/data_load/load.R | 94 +++++++++++-------- .../data_load/testing_recipes/recipe_1.yml | 12 +-- .../data_load/testing_recipes/recipe_2.yml | 2 +- modules/test.R | 22 ++++- out-logs/ini | 0 tools/prepare_outputs.R | 2 + 7 files changed, 114 insertions(+), 53 deletions(-) delete mode 100755 out-logs/ini diff --git a/modules/data_load/dates2load.R b/modules/data_load/dates2load.R index eda5f239..27be7221 100644 --- a/modules/data_load/dates2load.R +++ b/modules/data_load/dates2load.R @@ -12,6 +12,9 @@ #'@return a list of two arrays containing file dates for hcst and fcst #' #'@export + +library(lubridate) + dates2load <- function(recipe, logger){ recipe <- recipe$Analysis$Time @@ -51,4 +54,34 @@ dates2load <- function(recipe, logger){ dim(data) <- default_dims return(data) - } +} + +get_timeidx <- function(sdates, ltmin, ltmax, + time_freq="monthly_mean"){ + + if (time_freq == "daily_mean"){ + + sdates <- ymd(sdates) + vdates <- rep(sdates,each=length(ltmin:ltmax)) + months(ltmin:ltmax) + + months <- month(vdates) + months <- months[!is.na(months)] + + ndays <- sapply(sdates,FUN=function(x) sum(days_in_month(x+months(ltmin:ltmax-1)))) + + idx_min <- (sdates + months(ltmin-1) - sdates) + 1 + idx_max <- idx_min + ndays + + } else if (time_freq == "monthly_mean") { + + idx_min <- ltmin + idx_max <- ltmax + + } + + # TODO: 6 hourly case + #idx1 <- (sdates + months(ltmin-1) - sdates)*4 + #idx2 <- idx1 + ndays*4 - 1 + + return(indices(idx_min:idx_max)) +} diff --git a/modules/data_load/load.R b/modules/data_load/load.R index a6b2a5f0..047eafab 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -1,5 +1,3 @@ - - ## TODO: remove paths to personal scratchs source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") source("/esarchive/scratch/vagudets/repos/cstools/R/s2dv_cube.R") @@ -30,6 +28,8 @@ startR_to_s2dv <- function(startR_array){ dates_end <- attr(startR_array, 'Variables')$common$time #sdates <- unlist(attr(startR_array, 'FileSelectors')$dat1$file_date) + print(dim(dates_start)) + print(dates_dims) dim(dates_start) <- dates_dims dim(dates_end) <- dates_dims #dim(sdates) <- sdates_dims @@ -38,11 +38,11 @@ startR_to_s2dv <- function(startR_array){ s2dv_object <- s2dv_cube(data = attr2array(startR_array), lon = attr2array(attr(startR_array, - 'Variables')$dat1$longitude), + 'Variables')$dat1$longitude), lat = attr2array(attr(startR_array, - 'Variables')$dat1$latitude), - Variable = list(varName = names(attr(startR_array, - 'Variables')$common)[2], + 'Variables')$dat1$latitude), + Variable = list(varName=names(attr(startR_array, + 'Variables')$common)[2], level = NULL), Dates = list(start = dates_start, end = dates_end), @@ -50,17 +50,16 @@ startR_to_s2dv <- function(startR_array){ time_dims = time_dims, when = Sys.time(), source_files = attr2array(attr(startR_array, "Files")) - #Datasets = list(exp1 = list(InitializationsDates = list(Member_1 = "01011990", - # Members = "Member_1"))) + #Datasets=list(exp1=list(InitializationsDates=list(Member_1="01011990", + # Members="Member_1"))) ) - return(s2dv_object) - } # RECIPE FOR TESTING # -------------------------------------------------------------------------------- # recipe_file <- "modules/data_load/testing_recipes/recipe_2.yml" +# recipe_file <- "modules/data_load/testing_recipes/recipe_1.yml" load_datasets <- function(recipe_file){ @@ -74,13 +73,12 @@ load_datasets <- function(recipe_file){ folder <- logger$foldername log_file <- logger$logname logger <- logger$logger + # ------------------------------------------- - # Set params ----------------------------------------- + hcst.inityear <- recipe$Analysis$Time$hcst_start hcst.endyear <- recipe$Analysis$Time$hcst_end - ltmin <- recipe$Analysis$Time$leadtimemin - ltmax <- recipe$Analysis$Time$leadtimemax lats.min <- recipe$Analysis$Region$latmin lats.max <- recipe$Analysis$Region$latmax lons.min <- recipe$Analysis$Region$lonmin @@ -90,7 +88,21 @@ load_datasets <- function(recipe_file){ variable <- recipe$Analysis$Variables$name store.freq <- recipe$Analysis$Variables$freq - + + # get sdates array + sdates <- dates2load(recipe, logger) + + idxs <- NULL + idxs$hcst <- get_timeidx(sdates$hcst, + recipe$Analysis$Time$leadtimemin, + recipe$Analysis$Time$leadtimemax, + time_freq=store.freq) + + idxs$fcst <- get_timeidx(sdates$fcst, + recipe$Analysis$Time$leadtimemin, + recipe$Analysis$Time$leadtimemax, + time_freq=store.freq) + ## TODO: Examine this verifications part, verify if it's necessary # stream <- verifications$stream # sdates <- verifications$fcst.sdate @@ -98,9 +110,6 @@ load_datasets <- function(recipe_file){ ## TODO: define fcst.name ##fcst.name <- recipe$Analysis$Datasets$System[[sys]]$name - # get sdates array - sdates <- dates2load(recipe, logger) - # get esarchive datasets dict: archive <- read_yaml(paste0(recipe$Run$code_dir, "conf/archive.yml"))$archive exp_descrip <- archive$System[[exp.name]] @@ -124,15 +133,15 @@ load_datasets <- function(recipe_file){ # ----------- obs.path <- paste0(archive$src, obs.dir, store.freq, "/$var$", - reference_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") + reference_descrip[[store.freq]][[variable]],"$var$_$file_date$.nc") hcst.path <- paste0(archive$src, hcst.dir, store.freq, "/$var$", - exp_descrip[[store.freq]][[variable]], "$var$_$file_date$.nc") + exp_descrip[[store.freq]][[variable]],"$var$_$file_date$.nc") fcst.path <- paste0(archive$src, hcst.dir, store.freq, "/$var$", - exp_descrip[[store.freq]][[variable]], "$var$_$fcst_syear$.nc") + exp_descrip[[store.freq]][[variable]],"$var$_$fcst_syear$.nc") # Define regrid parameters: #------------------------------------------------------------------- @@ -151,7 +160,7 @@ load_datasets <- function(recipe_file){ hcst <- Start(dat = hcst.path, var = variable, file_date = sdates$hcst, - time = indices(ltmin:ltmax), + time = idxs$hcst, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), @@ -162,7 +171,7 @@ load_datasets <- function(recipe_file){ transform_vars = c('latitude', 'longitude'), synonims = list(latitude = c('lat', 'latitude'), longitude = c('lon', 'longitude'), - ensemble = c('member', 'ensemble')), + ensemble = c('member', 'ensemble')), ensemble = indices(1:hcst.nmember), return_vars = list(latitude = 'dat', longitude = 'dat', @@ -179,7 +188,7 @@ load_datasets <- function(recipe_file){ fcst <- Start(dat = fcst.path, var = variable, fcst_syear = sdates$fcst, - time = indices(ltmin:ltmax), + time = idxs$fcst, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), longitude = values(list(lons.min, lons.max)), @@ -206,27 +215,19 @@ load_datasets <- function(recipe_file){ # Load reference #------------------------------------------------------------------- - - # Get reference dates from hcst Start call - ## TODO: Replace with new code once An-Chi's fix is released in StartR - #dates <- attr(hcst, 'Variables')$common$time dates <- hcst$Dates$start - # Get year and month for file_date - dates_file <- sapply(dates, format, '%Y%m') - dates.dims <- c(sday = 1, sweek = 1, time = 1) - dates.dims[names(dim(dates))] <- dim(dates) - # Set hour to 12:00 to ensure correct date retrieval for daily data - lubridate::hour(dates) <- 12 - lubridate::minute(dates) <- 00 - # Restore correct dimensions - dim(dates) <- dates.dims - names(dim(dates))[4] <- "syear" - dim(dates_file) <- dim(dates) - + # Separate Start() call for monthly vs daily data if (store.freq == "monthly_mean") { + dates_file <- sapply(dates, format, '%Y%m%d') + dates_file <- format(as.Date(dates_file, '%Y%m%d'), "%Y%m") + dim(dates_file) <- dim(Subset(hcst$data, + along=c('dat','var', + 'latitude', 'longitude', 'ensemble'), + list(1,1,1,1,1), drop="selected")) + obs <- Start(dat = obs.path, var = variable, file_date = dates_file, @@ -249,7 +250,20 @@ load_datasets <- function(recipe_file){ retrieve = TRUE) } else if (store.freq == "daily_mean") { - + + # Get year and month for file_date + dates_file <- sapply(dates, format, '%Y%m') + dim(dates_file) <- dim(dates) + #dates.dims <- c(sday = 1, sweek = 1, time = 1) + #dates.dims <- c(sday = 1, sweek = 1) + #dates.dims[names(dim(dates))] <- dim(dates) + # Set hour to 12:00 to ensure correct date retrieval for daily data + lubridate::hour(dates) <- 12 + lubridate::minute(dates) <- 00 + # Restore correct dimensions + dim(dates) <- dim(dates_file) + #names(dim(dates))[4] <- "syear" + obs <- Start(dat = obs.path, var = variable, file_date = sort(unique(dates_file)), diff --git a/modules/data_load/testing_recipes/recipe_1.yml b/modules/data_load/testing_recipes/recipe_1.yml index c7cdfca1..b6f53843 100644 --- a/modules/data_load/testing_recipes/recipe_1.yml +++ b/modules/data_load/testing_recipes/recipe_1.yml @@ -1,6 +1,6 @@ Description: - Author: N.Pérez-Zanón - '': split version + Author: V. Agudetse + Analysis: Horizon: Seasonal Variables: @@ -14,8 +14,8 @@ Analysis: name: era5 Time: sdate: - fcst_syear: '2017' - fcst_sday: '0101' + fcst_syear: '2020' + fcst_sday: '1101' hcst_start: '1993' hcst_end: '2016' leadtimemin: 2 @@ -39,5 +39,5 @@ Analysis: Run: Loglevel: INFO Terminal: yes - output_dir: /esarchive/scratch/vagudets/repos/auto-s2s/out-logs/ - code_dir: /esarchive/scratch/vagudets/repos/auto-s2s/ + output_dir: /esarchive/scratch/lpalma/git/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/lpalma/git/auto-s2s/ diff --git a/modules/data_load/testing_recipes/recipe_2.yml b/modules/data_load/testing_recipes/recipe_2.yml index b8d81ad7..b8f2d13a 100644 --- a/modules/data_load/testing_recipes/recipe_2.yml +++ b/modules/data_load/testing_recipes/recipe_2.yml @@ -1,6 +1,6 @@ Description: Author: V. Agudetse - '': split version + Analysis: Horizon: Seasonal Variables: diff --git a/modules/test.R b/modules/test.R index 9bd30de4..f88c476f 100644 --- a/modules/test.R +++ b/modules/test.R @@ -1,22 +1,26 @@ +dates_dims <- dim(Subset(hcst, + along=c('dat','var', + 'latitude', 'longitude', 'ensemble'), + list(1,1,1,1,1), drop="selected")) + +dim(attr(hcst, 'Variables')$common$time) recipe_file <- "modules/data_load/testing_recipes/recipe_2.yml" +recipe_file <- "modules/data_load/testing_recipes/recipe_1.yml" source("modules/data_load/load.R") data <- load_datasets(recipe_file) -hcst <- data$hcst -obs <- data$obs - method <- "bias" mm=F ncores=4 na.rm=T # Hcst calibration -hcst <- CSTools::CST_Calibration(hcst,obs, +hcst <- CSTools::CST_Calibration(data$hcst, data$obs, cal.method = method, eval.method = "leave-one-out", multi.model = mm, @@ -28,4 +32,12 @@ hcst <- CSTools::CST_Calibration(hcst,obs, sdate_dim = "syear", ncores = ncores) - +# For daily +hcst <- CSTools::CST_QuantileMapping(data$hcst, + data$obs, + exp_cor = NULL, + sample_dims = c("syear", "time", "ensemble"), + sample_length = NULL, + method = "QUANT", + ncores = ncores, + n.rm = na.rm) diff --git a/out-logs/ini b/out-logs/ini deleted file mode 100755 index e69de29b..00000000 diff --git a/tools/prepare_outputs.R b/tools/prepare_outputs.R index 06f8270f..5efa87d4 100644 --- a/tools/prepare_outputs.R +++ b/tools/prepare_outputs.R @@ -31,6 +31,8 @@ prepare_outputs <- function(recipe) { # Create output folders: folder_name <- paste0(gsub(".yml", "", gsub("/", "_", recipe$filename)), "_", gsub(" ", "", gsub(":", "", gsub("-", "", Sys.time())))) + + print(output_dir) print(folder_name) dir.create(file.path(output_dir, folder_name, 'plots'), recursive = TRUE) -- GitLab From 1f8e3b78f51df15672db9c0cddaefcfd5c93ff30 Mon Sep 17 00:00:00 2001 From: lpalma Date: Tue, 5 Apr 2022 18:16:12 +0200 Subject: [PATCH 2/4] daily loading working passing dates to time arg in startR call --- modules/data_load/dates2load.R | 36 +++++++++----- modules/data_load/load.R | 47 +++++++++++++------ .../data_load/testing_recipes/recipe_1.yml | 2 +- .../data_load/testing_recipes/recipe_3.yml | 43 +++++++++++++++++ 4 files changed, 101 insertions(+), 27 deletions(-) create mode 100644 modules/data_load/testing_recipes/recipe_3.yml diff --git a/modules/data_load/dates2load.R b/modules/data_load/dates2load.R index 27be7221..107ce776 100644 --- a/modules/data_load/dates2load.R +++ b/modules/data_load/dates2load.R @@ -17,18 +17,24 @@ library(lubridate) dates2load <- function(recipe, logger){ + temp_freq <- recipe$Analysis$Variables$freq recipe <- recipe$Analysis$Time # hcst dates file_dates <- paste0(strtoi(recipe$hcst_start):strtoi(recipe$hcst_end), recipe$sdate$fcst_sday) - file_dates <- .add_dims(file_dates, "hcst") + + if (temp_freq == "monthly_mean"){ + file_dates <- .add_dims(file_dates, "hcst") + } # fcst dates (if fcst_year empty it creates an empty object) if (! is.null(recipe$sdate$fcst_syear)){ file_dates.fcst <- paste0(recipe$sdate$fcst_syear, recipe$sdate$fcst_sday) - file_dates.fcst <- .add_dims(file_dates.fcst, "fcst") + if (temp_freq == "monthly_mean"){ + file_dates.fcst <- .add_dims(file_dates.fcst, "fcst") + } } else { file_dates.fcst <- NULL info(logger, @@ -61,21 +67,29 @@ get_timeidx <- function(sdates, ltmin, ltmax, if (time_freq == "daily_mean"){ - sdates <- ymd(sdates) - vdates <- rep(sdates,each=length(ltmin:ltmax)) + months(ltmin:ltmax) + sdates <- ymd(sdates) + idx_min <- sdates + months(ltmin) + idx_max <- sdates + months(ltmax+1) - days(1) - months <- month(vdates) - months <- months[!is.na(months)] + indxs <- array(numeric(), c( + file_date=length(sdates), + time=(as.integer(idx_max[1]-idx_min[1])+1) + #syear=length(sdates) + #sday=1,sweek=1)) + )) - ndays <- sapply(sdates,FUN=function(x) sum(days_in_month(x+months(ltmin:ltmax-1)))) - - idx_min <- (sdates + months(ltmin-1) - sdates) + 1 - idx_max <- idx_min + ndays + for (sdate in 1:length(sdates)){ + days <- seq(idx_min[sdate],idx_max[sdate], by='days') + indxs[sdate,] <- days[!(format(days, "%m%d") == "0229")] + } + indxs <- as.POSIXct(indxs*86400, + tz = 'UTC', origin = '1970-01-01') } else if (time_freq == "monthly_mean") { idx_min <- ltmin idx_max <- ltmax + indxs <- indices(idx_min:idx_max) } @@ -83,5 +97,5 @@ get_timeidx <- function(sdates, ltmin, ltmax, #idx1 <- (sdates + months(ltmin-1) - sdates)*4 #idx2 <- idx1 + ndays*4 - 1 - return(indices(idx_min:idx_max)) + return(indxs) } diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 047eafab..58edee49 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -1,10 +1,14 @@ -## TODO: remove paths to personal scratchs +## TODO: remove paths to personal scratchs source("/esarchive/scratch/vagudets/repos/csoperational/R/get_regrid_params.R") source("/esarchive/scratch/vagudets/repos/cstools/R/s2dv_cube.R") # Load required libraries/funs source("modules/data_load/dates2load.R") source("tools/libs.R") +path <- "/esarchive/scratch/lpalma/git/startR-test-attrDims/R/" +ff <- lapply(list.files(path), function(x) paste0(path, x)) +invisible(lapply(ff, source)) + ## TODO: Move to CSOperational # Conversion from startR_array to s2dv_array #------------------------------------------------------------------- @@ -28,11 +32,10 @@ startR_to_s2dv <- function(startR_array){ dates_end <- attr(startR_array, 'Variables')$common$time #sdates <- unlist(attr(startR_array, 'FileSelectors')$dat1$file_date) - print(dim(dates_start)) - print(dates_dims) dim(dates_start) <- dates_dims dim(dates_end) <- dates_dims #dim(sdates) <- sdates_dims + ## TODO: change this line when time attributes work correctly? time_dims <- c("time", "sday", "sweek", "syear", "fcst_syear") @@ -58,6 +61,7 @@ startR_to_s2dv <- function(startR_array){ # RECIPE FOR TESTING # -------------------------------------------------------------------------------- +# recipe_file <- "modules/data_load/testing_recipes/recipe_3.yml" # recipe_file <- "modules/data_load/testing_recipes/recipe_2.yml" # recipe_file <- "modules/data_load/testing_recipes/recipe_1.yml" @@ -141,7 +145,7 @@ load_datasets <- function(recipe_file){ fcst.path <- paste0(archive$src, hcst.dir, store.freq, "/$var$", - exp_descrip[[store.freq]][[variable]],"$var$_$fcst_syear$.nc") + exp_descrip[[store.freq]][[variable]],"$var$_$file_date$.nc") # Define regrid parameters: #------------------------------------------------------------------- @@ -154,6 +158,12 @@ load_datasets <- function(recipe_file){ } else { circularsort <- CircularSort(-180, 180) } + + if (recipe$Analysis$Variables$freq == "monthly_mean"){ + split_multiselected_dims = TRUE + } else { + split_multiselected_dims = FALSE + } # Load hindcast #------------------------------------------------------------------- @@ -171,23 +181,33 @@ load_datasets <- function(recipe_file){ transform_vars = c('latitude', 'longitude'), synonims = list(latitude = c('lat', 'latitude'), longitude = c('lon', 'longitude'), - ensemble = c('member', 'ensemble')), + ensemble = c('member', 'ensemble')), ensemble = indices(1:hcst.nmember), return_vars = list(latitude = 'dat', longitude = 'dat', time = 'file_date'), - split_multiselected_dims = TRUE, + split_multiselected_dims = split_multiselected_dims, retrieve = TRUE) - + + if (recipe$Analysis$Variables$freq == "daily_mean"){ + # Adjusts dims for daily case, could be removed if startR allows + # multidim split + names(dim(hcst))[which(names(dim(hcst)) == 'file_date')] <- "syear" + default_dims <- c(dat = 1, var = 1, sweek = 1, + sday = 1, syear = 1, time = 1, + latitude = 1, longitude = 1, ensemble = 1) + default_dims[names(dim(hcst))] <- dim(hcst) + dim(hcst) <- default_dims + } hcst <- startR_to_s2dv(hcst) # Load forecast #------------------------------------------------------------------- if (!is.null(recipe$Analysis$Time$sdate$fcst_syear)){ - + fcst <- Start(dat = fcst.path, var = variable, - fcst_syear = sdates$fcst, + file_date = sdates$fcst, time = idxs$fcst, latitude = values(list(lats.min, lats.max)), latitude_reorder = Sort(), @@ -203,10 +223,11 @@ load_datasets <- function(recipe_file){ ensemble = indices(1:fcst.nmember), return_vars = list(latitude = 'dat', longitude = 'dat', - time = 'fcst_syear'), - split_multiselected_dims = TRUE, + time = 'file_date'), + split_multiselected_dims = split_multiselected_dims, retrieve = TRUE) + names(dim(fcst))[which(names(dim(fcst)) == 'file_date')] <- "fcst_year" fcst <- startR_to_s2dv(fcst) } else { @@ -254,15 +275,11 @@ load_datasets <- function(recipe_file){ # Get year and month for file_date dates_file <- sapply(dates, format, '%Y%m') dim(dates_file) <- dim(dates) - #dates.dims <- c(sday = 1, sweek = 1, time = 1) - #dates.dims <- c(sday = 1, sweek = 1) - #dates.dims[names(dim(dates))] <- dim(dates) # Set hour to 12:00 to ensure correct date retrieval for daily data lubridate::hour(dates) <- 12 lubridate::minute(dates) <- 00 # Restore correct dimensions dim(dates) <- dim(dates_file) - #names(dim(dates))[4] <- "syear" obs <- Start(dat = obs.path, var = variable, diff --git a/modules/data_load/testing_recipes/recipe_1.yml b/modules/data_load/testing_recipes/recipe_1.yml index b6f53843..68715462 100644 --- a/modules/data_load/testing_recipes/recipe_1.yml +++ b/modules/data_load/testing_recipes/recipe_1.yml @@ -18,7 +18,7 @@ Analysis: fcst_sday: '1101' hcst_start: '1993' hcst_end: '2016' - leadtimemin: 2 + leadtimemin: 1 leadtimemax: 4 Region: latmin: -10 diff --git a/modules/data_load/testing_recipes/recipe_3.yml b/modules/data_load/testing_recipes/recipe_3.yml new file mode 100644 index 00000000..ed82f43c --- /dev/null +++ b/modules/data_load/testing_recipes/recipe_3.yml @@ -0,0 +1,43 @@ +Description: + Author: V. Agudetse + +Analysis: + Horizon: Seasonal + Variables: + name: tas + freq: daily_mean + Datasets: + System: + name: system5c3s + Multimodel: no + Reference: + name: era5 + Time: + sdate: + fcst_syear: '2020' + fcst_sday: '1101' + hcst_start: '1993' + hcst_end: '1995' + leadtimemin: 0 + leadtimemax: 0 + Region: + latmin: -10 + latmax: 10 + lonmin: 0 + lonmax: 20 + Regrid: + method: bilinear + type: to_system + Workflow: + Calibration: + method: SBC + Skill: + metric: RPSS + Indicators: + index: no + Output_format: S2S4E +Run: + Loglevel: INFO + Terminal: yes + output_dir: /esarchive/scratch/lpalma/git/auto-s2s/out-logs/ + code_dir: /esarchive/scratch/lpalma/git/auto-s2s/ -- GitLab From f65aed67ab992fb2f3444ff05ca14585d1189437 Mon Sep 17 00:00:00 2001 From: lpalma Date: Wed, 6 Apr 2022 11:01:27 +0200 Subject: [PATCH 3/4] fixed timestamp selection by adding 12h UTC to every date --- modules/data_load/dates2load.R | 6 ++++++ modules/test.R | 30 +++++++++++------------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/modules/data_load/dates2load.R b/modules/data_load/dates2load.R index 107ce776..cff4a00f 100644 --- a/modules/data_load/dates2load.R +++ b/modules/data_load/dates2load.R @@ -84,6 +84,12 @@ get_timeidx <- function(sdates, ltmin, ltmax, } indxs <- as.POSIXct(indxs*86400, tz = 'UTC', origin = '1970-01-01') + lubridate::hour(indxs) <- 12 + lubridate::minute(indxs) <- 00 + dim(indxs) <- list(file_date=length(sdates), + time=(as.integer(idx_max[1]-idx_min[1])+1)) + + } else if (time_freq == "monthly_mean") { diff --git a/modules/test.R b/modules/test.R index f88c476f..a9ae12af 100644 --- a/modules/test.R +++ b/modules/test.R @@ -1,13 +1,5 @@ -dates_dims <- dim(Subset(hcst, - along=c('dat','var', - 'latitude', 'longitude', 'ensemble'), - list(1,1,1,1,1), drop="selected")) - -dim(attr(hcst, 'Variables')$common$time) - - recipe_file <- "modules/data_load/testing_recipes/recipe_2.yml" recipe_file <- "modules/data_load/testing_recipes/recipe_1.yml" source("modules/data_load/load.R") @@ -20,17 +12,17 @@ ncores=4 na.rm=T # Hcst calibration -hcst <- CSTools::CST_Calibration(data$hcst, data$obs, - cal.method = method, - eval.method = "leave-one-out", - multi.model = mm, - na.fill = TRUE, - na.rm = na.rm, - apply_to = NULL, - alpha = NULL, - memb_dim = "ensemble", - sdate_dim = "syear", - ncores = ncores) +#hcst <- CSTools::CST_Calibration(data$hcst, data$obs, +# cal.method = method, +# eval.method = "leave-one-out", +# multi.model = mm, +# na.fill = TRUE, +# na.rm = na.rm, +# apply_to = NULL, +# alpha = NULL, +# memb_dim = "ensemble", +# sdate_dim = "syear", +# ncores = ncores) # For daily hcst <- CSTools::CST_QuantileMapping(data$hcst, -- GitLab From 3c2b03a3a102b55d72323e07603863e545631f60 Mon Sep 17 00:00:00 2001 From: lpalma Date: Wed, 6 Apr 2022 16:52:41 +0200 Subject: [PATCH 4/4] Added some comments to the code --- modules/data_load/dates2load.R | 8 ++++++-- modules/data_load/load.R | 6 ++++-- modules/test.R | 2 ++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/modules/data_load/dates2load.R b/modules/data_load/dates2load.R index cff4a00f..28d7905e 100644 --- a/modules/data_load/dates2load.R +++ b/modules/data_load/dates2load.R @@ -62,6 +62,12 @@ dates2load <- function(recipe, logger){ } +# Gets the corresponding dates or indices according +# to the sdate/leadtimes requested in the recipe +# +# The leadtimes are defined by months +# Ex. 20201101 with leadtimes 1-4 corresponds to +# the forecasting times covering December to March get_timeidx <- function(sdates, ltmin, ltmax, time_freq="monthly_mean"){ @@ -89,8 +95,6 @@ get_timeidx <- function(sdates, ltmin, ltmax, dim(indxs) <- list(file_date=length(sdates), time=(as.integer(idx_max[1]-idx_min[1])+1)) - - } else if (time_freq == "monthly_mean") { idx_min <- ltmin diff --git a/modules/data_load/load.R b/modules/data_load/load.R index 58edee49..0083e9a9 100755 --- a/modules/data_load/load.R +++ b/modules/data_load/load.R @@ -204,7 +204,10 @@ load_datasets <- function(recipe_file){ # Load forecast #------------------------------------------------------------------- if (!is.null(recipe$Analysis$Time$sdate$fcst_syear)){ - + # the call uses file_date instead of fcst_syear so that it can work + # with the daily case and the current version of startR not allowing + # multiple dims split + fcst <- Start(dat = fcst.path, var = variable, file_date = sdates$fcst, @@ -236,7 +239,6 @@ load_datasets <- function(recipe_file){ # Load reference #------------------------------------------------------------------- - ## TODO: Replace with new code once An-Chi's fix is released in StartR dates <- hcst$Dates$start # Separate Start() call for monthly vs daily data diff --git a/modules/test.R b/modules/test.R index a9ae12af..744f6ecd 100644 --- a/modules/test.R +++ b/modules/test.R @@ -1,7 +1,9 @@ +recipe_file <- "modules/data_load/testing_recipes/recipe_3.yml" recipe_file <- "modules/data_load/testing_recipes/recipe_2.yml" recipe_file <- "modules/data_load/testing_recipes/recipe_1.yml" + source("modules/data_load/load.R") data <- load_datasets(recipe_file) -- GitLab