From 5b976763de837308608428a221379183b122487c Mon Sep 17 00:00:00 2001 From: aho Date: Fri, 9 Oct 2020 16:43:53 +0200 Subject: [PATCH 1/3] Add one 'if' condition to ensure the first file of each dataset exists. If not, metadata is assigned to NULL. --- R/Start.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/Start.R b/R/Start.R index 718ae42..6222749 100644 --- a/R/Start.R +++ b/R/Start.R @@ -4141,7 +4141,7 @@ Start <- function(..., # dim = indices/selectors, loaded_metadata_count <- 1 for (kk in 1:length(return_metadata)) { for (jj in 1:length(return_metadata[[kk]])) { - if (dataset_has_files[kk]) { + if (dataset_has_files[kk] & kk %in% loaded_metadata_files) { return_metadata[[kk]][jj] <- loaded_metadata[[loaded_metadata_count]] names(return_metadata[[kk]])[jj] <- names(loaded_metadata[[loaded_metadata_count]]) loaded_metadata_count <- loaded_metadata_count + 1 -- GitLab From ef4291fb581d0d8c4d6b0c210f43c41072b14757 Mon Sep 17 00:00:00 2001 From: aho Date: Fri, 9 Oct 2020 18:23:03 +0200 Subject: [PATCH 2/3] Revise the 'if' condition --- R/Start.R | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/R/Start.R b/R/Start.R index 6222749..32c61ae 100644 --- a/R/Start.R +++ b/R/Start.R @@ -4141,9 +4141,14 @@ Start <- function(..., # dim = indices/selectors, loaded_metadata_count <- 1 for (kk in 1:length(return_metadata)) { for (jj in 1:length(return_metadata[[kk]])) { - if (dataset_has_files[kk] & kk %in% loaded_metadata_files) { - return_metadata[[kk]][jj] <- loaded_metadata[[loaded_metadata_count]] - names(return_metadata[[kk]])[jj] <- names(loaded_metadata[[loaded_metadata_count]]) + + if (dataset_has_files[kk]) { + if (loaded_metadata_count %in% loaded_metadata_files) { + return_metadata[[kk]][jj] <- loaded_metadata[[loaded_metadata_count]] + names(return_metadata[[kk]])[jj] <- names(loaded_metadata[[loaded_metadata_count]]) + } else { + return_metadata[[kk]][jj] <- NULL + } loaded_metadata_count <- loaded_metadata_count + 1 } else { return_metadata[[kk]][jj] <- NULL -- GitLab From df61813afb2108f4f440c878527fc1eeee4330f2 Mon Sep 17 00:00:00 2001 From: aho Date: Fri, 9 Oct 2020 18:35:26 +0200 Subject: [PATCH 3/3] Add new unit test for two dats but first file is missing --- tests/testthat/test-Start-metadata_dims.R | 68 +++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/testthat/test-Start-metadata_dims.R b/tests/testthat/test-Start-metadata_dims.R index 0e53f07..1c0bc91 100644 --- a/tests/testthat/test-Start-metadata_dims.R +++ b/tests/testthat/test-Start-metadata_dims.R @@ -374,3 +374,71 @@ test_that("7. Two data sets, while one is missing", { dim = c(dat = 2, var = 1, sdate = 1)) ) }) + +test_that("8. Two data sets, both have files but the first file is missing", { +path_list <- list( + MPI = list(name = 'MPI_ESM', + path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/', + 'cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/', + 'dcppA-hindcast/$member$/day/$var$/gn/v20200101/', + '$var$_day_MPI-ESM1-2-HR_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc')), + Had = list(name = 'HadGEM3', + path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/', + 'cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/', + 'dcppA-hindcast/$member$/day/$var$/gn/v20200101/', + '$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc'))) + +data <- Start(dataset = path_list, + var = 'tasmin', + member = list(c('r1i1p1f1', 'r2i1p1f2')), + sdate = paste0(2018), + chunk = list(c('20181101-20281231', '20181101-20281230')), + time = indices(1), #'all', + lat = values(list(0, 14)), + lon = values(list(0, 28)), + synonims = list(time = c('fmonth','time'), + lon = c('lon', 'longitude'), + lat = c('lat', 'latitude')), + return_vars = list(lat = 'dataset', lon = 'dataset'), + lat_reorder = Sort(), + num_procs = 1, + retrieve = T) + + expect_equal( + length(data[is.na(data)]), + 5500 + ) + expect_equal( + length(attr(data, "Variables")$MPI_ESM), + 3 + ) + expect_equal( + length(attr(data, "Variables")$MPI_ESM$lon), + 30 + ) + expect_equal( + names(attr(data, "Variables")$MPI_ESM), + c('lat', 'lon', 'tasmin') + ) + expect_equal( + length(attr(data, "Variables")$HadGEM3), + 2 + ) + expect_equal( + length(attr(data, "Variables")$HadGEM3$lon), + 34 + ) + expect_equal( + names(attr(data, "Variables")$HadGEM3), + c('lat', 'lon') + ) + expect_equal( + attr(data, 'Files'), + array(c("/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/day/tasmin/gn/v20200101/tasmin_day_MPI-ESM1-2-HR_dcppA-hindcast_s2018-r1i1p1f1_gn_20181101-20281231.nc", + NA, NA, NA, NA, NA, NA, + "/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r2i1p1f2/day/tasmin/gn/v20200101/tasmin_day_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r2i1p1f2_gn_20181101-20281230.nc"), + dim = c(dataset = 2, var = 1, member = 2, sdate = 1, chunk = 2)) + ) + + +}) -- GitLab