From 5ea9d45126d610f6b8565b170221dbcfbfbb9607 Mon Sep 17 00:00:00 2001 From: aho Date: Tue, 11 Aug 2020 21:31:24 +0200 Subject: [PATCH 1/4] A typo leads to wrong reordered metadata when return_vars is NULL. --- R/Start.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/Start.R b/R/Start.R index 62d7e8c..6bf0b72 100644 --- a/R/Start.R +++ b/R/Start.R @@ -1978,7 +1978,7 @@ Start <- function(..., # dim = indices/selectors, picked_common_vars[[var_to_read]] <- new_array pick_ordered <- FALSE if (var_to_read %in% unlist(var_params)) { - if (associated_dim_name %in% names(dim_reorder_param) && !aiat) { + if (associated_dim_name %in% names(dim_reorder_params) && !aiat) { picked_common_vars_ordered[[var_to_read]] <- new_array pick_ordered <- TRUE } -- GitLab From 9eb02f4c0ee9fef8960f371f98cfe9c1cf2e09de Mon Sep 17 00:00:00 2001 From: aho Date: Wed, 12 Aug 2020 16:31:45 +0200 Subject: [PATCH 2/4] Bugfix for transformation lon/lat metadatawhen return_vars is NULL --- R/Start.R | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/R/Start.R b/R/Start.R index 6bf0b72..70b5320 100644 --- a/R/Start.R +++ b/R/Start.R @@ -3416,7 +3416,13 @@ Start <- function(..., # dim = indices/selectors, if (i == length(dat)) { for (common_var_to_crop in names(common_vars_to_crop)) { if (inner_dim %in% names(dim(common_vars_to_crop[[common_var_to_crop]]))) { + + if (type_of_var_to_crop == 'transformed' & !aiat) { + common_vars_to_crop[[common_var_to_crop]] <- Subset(transformed_subset_var, inner_dim, crop_indices) + } else { #old code common_vars_to_crop[[common_var_to_crop]] <- Subset(common_vars_to_crop[[common_var_to_crop]], inner_dim, crop_indices) + } + } } } -- GitLab From 3c7e816d80657fee54008dbbf7991d41edac7734 Mon Sep 17 00:00:00 2001 From: aho Date: Thu, 13 Aug 2020 03:22:54 +0200 Subject: [PATCH 3/4] Add unit tests for checking metadata when using reorder or transform, under the condition that returns_var = NULL/'dat'. --- tests/testthat/test-Start-reorder-metadata.R | 276 ++++++++++++++++++ .../testthat/test-Start-transform-metadata.R | 272 +++++++++++++++++ 2 files changed, 548 insertions(+) create mode 100644 tests/testthat/test-Start-reorder-metadata.R create mode 100644 tests/testthat/test-Start-transform-metadata.R diff --git a/tests/testthat/test-Start-reorder-metadata.R b/tests/testthat/test-Start-reorder-metadata.R new file mode 100644 index 0000000..b522a36 --- /dev/null +++ b/tests/testthat/test-Start-reorder-metadata.R @@ -0,0 +1,276 @@ +context("Start() reorder metadata check") +# Ensure returns_vars = NULL or 'dat' have the same metadata + +test_that("1. Sort() and CircularSort(0, 360)", { + +# Original lon is [-180, 180] +path_exp <- '/esarchive/recon/ecmwf/era5/original_files/reorder/daily_mean/$var$/$var$_$sdate$.nc' +lons.min <- -10 +lons.max <- 10 +lats.min <- 10 +lats.max <- 20 + +# return_vars is NULL +res_null <- Start(dat = list(list(path = path_exp)), + var = 'tas', + sdate = '199212', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(), + longitude_reorder = CircularSort(0, 360), + longitude = values(list(lons.min, lons.max)), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude')), + return_vars = list(latitude = NULL, + longitude = NULL, + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_null, 'Variables')$common$longitude)), + 2 + ) + expect_equal( + is.list(attr(attr(res_null, 'Variables')$common$latitude, 'variables')), + TRUE + ) + expect_equal( + length(attr(attr(res_null, 'Variables')$common$latitude, 'variables')$latitude), + 7 + ) + expect_equal( + length(attr(attr(res_null, 'Variables')$common$longitude, 'variables')$longitude), + 7 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[1], + 10.25761, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[35], + 19.81264, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$latitude)), + 35 + ) + + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[1], + 0.00000, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[71], + 359.71875, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$longitude)), + 71 + ) + +# return_vars is dat +res_dat <- Start(dat = list(list(path = path_exp)), + var = 'tas', + sdate = '199212', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(), + longitude_reorder = CircularSort(0, 360), + longitude = values(list(lons.min, lons.max)), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude')), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_dat, 'Variables')$dat1$longitude)), + 2 + ) + expect_equal( + is.list(attr(attr(res_dat, 'Variables')$dat1$latitude, 'variables')), + TRUE + ) + expect_equal( + length(attr(attr(res_dat, 'Variables')$dat1$latitude, 'variables')$latitude), + 7 + ) + expect_equal( + length(attr(attr(res_dat, 'Variables')$dat1$longitude, 'variables')$longitude), + 7 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[1], + 10.25761, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[35], + 19.81264, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$latitude)), + 35 + ) + + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[1], + 0.00000, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[71], + 359.71875, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$longitude)), + 71 + ) + +}) + +test_that("2. Sort(decreasing = TRUE) and CircularSort(-180, 180)", { + +# Original lon is [0, 360] +path_exp <- '/esarchive/exp/ecmwf/system5_m1/daily_mean/$var$_f6h/$var$_$sdate$.nc' +lons.min <- 190 +lons.max <- 200 +lats.min <- 10 +lats.max <- 20 + +# return_vars is NULL +res_null <- Start(dat = list(list(path = path_exp)), + var = 'psl', + member = 'all', + sdate = '19821201', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = T), + longitude_reorder = CircularSort(-180, 180), + longitude = values(list(lons.min, lons.max)), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude'), + member = c('ensemble', 'realization')), + return_vars = list(latitude = NULL, + longitude = NULL, + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_null, 'Variables')$common$longitude)), + 2 + ) + expect_equal( + is.list(attr(attr(res_null, 'Variables')$common$latitude, 'variables')), + TRUE + ) + expect_equal( + length(attr(attr(res_null, 'Variables')$common$latitude, 'variables')$latitude), + 7 + ) + expect_equal( + length(attr(attr(res_null, 'Variables')$common$longitude, 'variables')$longitude), + 7 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[1], + 19.81264, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[35], + 10.25761, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$latitude)), + 35 + ) + + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[1], + -170.0000, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[37], + -160.0000, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$longitude)), + 37 + ) + +# return_vars is 'dat' +res_dat <- Start(dat = list(list(path = path_exp)), + var = 'psl', + member = 'all', + sdate = '19821201', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = T), + longitude_reorder = CircularSort(-180, 180), + longitude = values(list(lons.min, lons.max)), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude'), + member = c('ensemble', 'realization')), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_dat, 'Variables')$dat1$longitude)), + 2 + ) + expect_equal( + is.list(attr(attr(res_dat, 'Variables')$dat1$latitude, 'variables')), + TRUE + ) + expect_equal( + length(attr(attr(res_dat, 'Variables')$dat1$latitude, 'variables')$latitude), + 7 + ) + expect_equal( + length(attr(attr(res_dat, 'Variables')$dat1$longitude, 'variables')$longitude), + 7 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[1], + 19.81264, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[35], + 10.25761, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$latitude)), + 35 + ) + + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[1], + -170.0000, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[37], + -160.0000, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$longitude)), + 37 + ) + +}) diff --git a/tests/testthat/test-Start-transform-metadata.R b/tests/testthat/test-Start-transform-metadata.R new file mode 100644 index 0000000..f19a02a --- /dev/null +++ b/tests/testthat/test-Start-transform-metadata.R @@ -0,0 +1,272 @@ +context("Start() transform metadata check") +# Ensure returns_vars = NULL or 'dat' have the same metadata + +test_that("1. Sort() and CircularSort(0, 360)", { + +# Original lon is [-180, 180] +path_exp <- '/esarchive/recon/ecmwf/era5/original_files/reorder/daily_mean/$var$/$var$_$sdate$.nc' +lons.min <- -10 +lons.max <- 10 +lats.min <- 10 +lats.max <- 20 + +# return_vars is NULL +res_null <- Start(dat = list(list(path = path_exp)), + var = 'tas', + sdate = '199212', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(), + longitude_reorder = CircularSort(0, 360), + longitude = values(list(lons.min, lons.max)), + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', + method = 'conservative', + crop = c(lons.min, lons.max, + lats.min, lats.max)), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude')), + return_vars = list(latitude = NULL, + longitude = NULL, + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_null, 'Variables')$common$latitude)), + 1 + ) + expect_equal( + length(attributes(attr(res_null, 'Variables')$common$longitude)), + 1 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[1], + 10, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[11], + 20, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$latitude)), + 11 + ) + + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[1], + 0, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[21], + 359, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$longitude)), + 21 + ) + +# return_vars is dat +res_dat <- Start(dat = list(list(path = path_exp)), + var = 'tas', + sdate = '199212', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(), + longitude_reorder = CircularSort(0, 360), + longitude = values(list(lons.min, lons.max)), + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', + method = 'conservative', + crop = c(lons.min, lons.max, + lats.min, lats.max)), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude')), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_dat, 'Variables')$dat1$latitude)), + 1 + ) + expect_equal( + length(attributes(attr(res_dat, 'Variables')$dat1$longitude)), + 1 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[1], + 10, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[11], + 20, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$latitude)), + 11 + ) + + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[1], + 0, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[21], + 359, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$longitude)), + 21 + ) + +}) + +test_that("2. Sort(decreasing = TRUE) and CircularSort(-180, 180)", { + +# Original lon is [0, 360] +path_exp <- '/esarchive/exp/ecmwf/system5_m1/daily_mean/$var$_f6h/$var$_$sdate$.nc' +lons.min <- 190 +lons.max <- 200 +lats.min <- 10 +lats.max <- 20 + +# return_vars is NULL +res_null <- Start(dat = list(list(path = path_exp)), + var = 'psl', + member = 'all', + sdate = '19821201', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = T), + longitude_reorder = CircularSort(-180, 180), + longitude = values(list(lons.min, lons.max)), + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', + method = 'conservative', + crop = c(lons.min, lons.max, + lats.min, lats.max)), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude'), + member = c('ensemble', 'realization')), + return_vars = list(latitude = NULL, + longitude = NULL, + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_null, 'Variables')$common$latitude)), + 1 + ) + expect_equal( + length(attributes(attr(res_null, 'Variables')$common$longitude)), + 1 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[1], + 20, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$latitude)[11], + 10, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$latitude)), + 11 + ) + + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[1], + -170, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_null, 'Variables')$common$longitude)[11], + -160, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_null, 'Variables')$common$longitude)), + 11 + ) + +# return_vars is 'dat' +res_dat <- Start(dat = list(list(path = path_exp)), + var = 'psl', + member = 'all', + sdate = '19821201', + time = indices(4), + latitude = values(list(lats.min, lats.max)), + latitude_reorder = Sort(decreasing = T), + longitude_reorder = CircularSort(-180, 180), + longitude = values(list(lons.min, lons.max)), + transform = CDORemapper, + transform_extra_cells = 2, + transform_params = list(grid = 'r360x181', + method = 'conservative', + crop = c(lons.min, lons.max, + lats.min, lats.max)), + transform_vars = c('latitude', 'longitude'), + synonims = list(latitude = c('lat', 'latitude'), + longitude = c('lon', 'longitude'), + member = c('ensemble', 'realization')), + return_vars = list(latitude = 'dat', + longitude = 'dat', + time = 'sdate'), + retrieve = F) + + expect_equal( + length(attributes(attr(res_dat, 'Variables')$dat1$latitude)), + 1 + ) + expect_equal( + length(attributes(attr(res_dat, 'Variables')$dat1$longitude)), + 1 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[1], + 20, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$latitude)[11], + 10, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$latitude)), + 11 + ) + + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[1], + -170, + tolerance = 0.0001 + ) + expect_equal( + as.vector(attr(res_dat, 'Variables')$dat1$longitude)[11], + -160, + tolerance = 0.0001 + ) + expect_equal( + length(as.vector(attr(res_dat, 'Variables')$dat1$longitude)), + 11 + ) + +}) -- GitLab From 2bc4a848b6b0219e008020d5b2ebbfb18f1520cf Mon Sep 17 00:00:00 2001 From: aho Date: Thu, 13 Aug 2020 03:31:21 +0200 Subject: [PATCH 4/4] Update NEWS.md for bugfix of metadata --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index e38cd18..4ebe2d6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# startR v2.0.1 (Release date: 2020-08-) +- Bugfix for metadata in the condition that reorder or transform is applied and 'return_vars' is NULL. +- Bugfix for the missing first file case. It showed an error before when the first file is not found but now it works. + # startR v2.0.0 (Release date: 2020-08-06) - Adopt Roxygen2 documentation format - Remove Subset() to avoid duplicated function. Use ClimProjDiags::Subset instead. -- GitLab