Commit a02a66c7 authored by aho's avatar aho
Browse files

Merge branch 'develop-return_vars_null_reshape' into 'master'

Develop return vars null reshape

See merge request !187
parents 42ca5c19 30cc26c4
Pipeline #7177 passed with stage
in 60 minutes and 56 seconds
......@@ -3333,7 +3333,6 @@ Start <- function(..., # dim = indices/selectors,
selector_store_position <- chunk
}
sub_array_of_indices <- transformed_indices[which(indices_chunk == chunk)]
sub_array_of_indices <- transformed_indices[which(indices_chunk == chunk)]
if (with_transform) {
# If the provided selectors are expressed in the world
# before transformation
......@@ -3716,11 +3715,13 @@ Start <- function(..., # dim = indices/selectors,
tmp_fun <- function (x, y) {
any(names(dim(x)) %in% y)
}
inner_dim_has_split_dim <- names(which(unlist(lapply(
picked_common_vars, tmp_fun, names(all_split_dims)))))
if (!identical(inner_dim_has_split_dim, character(0))) {
# If merge_across_dims also, it will be replaced later
saved_reshaped_attr <- attr(picked_common_vars[[inner_dim_has_split_dim]], 'variables')
if (!is.null(picked_common_vars)) {
inner_dim_has_split_dim <- names(which(unlist(lapply(
picked_common_vars, tmp_fun, names(all_split_dims)))))
if (!identical(inner_dim_has_split_dim, character(0))) {
# If merge_across_dims also, it will be replaced later
saved_reshaped_attr <- attr(picked_common_vars[[inner_dim_has_split_dim]], 'variables')
}
}
}
}
......@@ -3785,7 +3786,7 @@ Start <- function(..., # dim = indices/selectors,
if (!merge_across_dims & split_multiselected_dims & identical(inner_dim_has_split_dim, character(0))) {
final_dims_fake_metadata <- NULL
} else {
if (!merge_across_dims & split_multiselected_dims) {
if (!merge_across_dims & split_multiselected_dims & !is.null(picked_common_vars)) {
if (any(names(all_split_dims[[1]]) %in% names(dim(picked_common_vars[[inner_dim_has_split_dim]]))) &
names(all_split_dims)[1] != inner_dim_has_split_dim) {
if (inner_dim_has_split_dim %in% names(final_dims)) {
......@@ -4000,7 +4001,7 @@ Start <- function(..., # dim = indices/selectors,
picked_common_vars[[across_inner_dim]] <- metadata_tmp
attr(picked_common_vars[[across_inner_dim]], 'variables') <- saved_reshaped_attr
}
if (split_multiselected_dims) {
if (split_multiselected_dims & !is.null(picked_common_vars)) {
if (!identical(inner_dim_has_split_dim, character(0))) {
metadata_tmp <- array(picked_common_vars[[inner_dim_has_split_dim]], dim = final_dims_fake_metadata)
# Convert numeric back to dates
......@@ -4129,7 +4130,7 @@ Start <- function(..., # dim = indices/selectors,
picked_common_vars[[across_inner_dim]] <- metadata_tmp
attr(picked_common_vars[[across_inner_dim]], 'variables') <- saved_reshaped_attr
}
if (split_multiselected_dims) {
if (split_multiselected_dims & !is.null(picked_common_vars)) {
if (!identical(inner_dim_has_split_dim, character(0))) {
metadata_tmp <- array(picked_common_vars[[inner_dim_has_split_dim]], dim = final_dims_fake_metadata)
# Convert numeric back to dates
......
......@@ -8,8 +8,8 @@
context("Chunk over dimensions that have dependency relationship")
path <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/',
'cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/',
path <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/',
'HadGEM3-GC31-MM/dcppA-hindcast/',
'r1i1p1f2/Omon/tos/gn/v20200417/',
'$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-r1i1p1f2_gn_$chunk$.nc')
sdates <- c('2016', '2017', '2018')
......
......@@ -4,7 +4,7 @@ context("Irregular regriding in the workflow")
test_that("1. ex2_13", {
path <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/cmcc-cm2-sr5/cmip6-dcppA-hindcast_i1p1/',
path <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/CMCC-CM2-SR5/',
'DCPP/CMCC/CMCC-CM2-SR5/dcppA-hindcast/$member$/Omon/$var$/gn/v20210312/',
'$var$_*_s$sdate$-$member$_gn_$aux$.nc')
suppressWarnings(
......
context("DCPP successfull retrieved for depends and across parameters.")
test_that("Chunks of DCPP files- Local execution", {
path <- '/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-r1i1p1f2_gn_$chunk$.nc'
path <- '/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-r1i1p1f2_gn_$chunk$.nc'
sdates <- c('2017', '2018')
suppressWarnings(
......@@ -21,7 +20,7 @@ suppressWarnings(
# [sdate = 2, chunk = 3]
suppressWarnings(
dat_2018_chunk3 <- Start(dat = '/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r1i1p1f2_gn_202201-202212.nc',
dat_2018_chunk3 <- Start(dat = '/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r1i1p1f2_gn_202201-202212.nc',
var = 'tos', time = 'all', i = indices(1:10), j = indices(1:10),
retrieve = TRUE)
)
......@@ -29,7 +28,7 @@ expect_equal(dat[1,1,2,25:36,,], dat_2018_chunk3[1,1,,,])
# [sdate = 1, chunk = 2]
suppressWarnings(
dat_2017_chunk2 <- Start(dat = '/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s2017-r1i1p1f2_gn_202001-202012.nc',
dat_2017_chunk2 <- Start(dat = '/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s2017-r1i1p1f2_gn_202001-202012.nc',
var = 'tos', time = 'all', i = indices(1:10), j = indices(1:10),
retrieve = TRUE)
)
......@@ -37,7 +36,7 @@ expect_equal(dat[1,1,1,13:24,,], dat_2017_chunk2[1,1,,,])
# [sdate = 2, chunk = 1]
suppressWarnings(
dat_2018_chunk1 <- Start(dat = '/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r1i1p1f2_gn_202001-202012.nc',
dat_2018_chunk1 <- Start(dat = '/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r1i1p1f2_gn_202001-202012.nc',
var = 'tos', time = 'all', i = indices(1:10), j = indices(1:10),
retrieve = TRUE)
)
......
context("Start() different calendar")
test_that("1. 360_day, daily, unit = 'days since 1850-01-01'", {
path_hadgem3 <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/',
'cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/',
path_hadgem3 <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast//HadGEM3-GC31-MM/',
'DCPP/MOHC/HadGEM3-GC31-MM/',
'dcppA-hindcast/r1i1p1f2/day/$var$/gn/v20200101/',
'$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-r1i1p1f2_gn_$fyear$.nc')
......@@ -48,7 +47,7 @@ expect_equal(
})
test_that("2. 365_day, daily, unit = 'days since 1984-01-01'", {
path_bcc_csm2 <- '/esarchive/exp/CMIP6/dcppA-hindcast/bcc-csm2-mr/cmip6-dcppA-hindcast_i1p1/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r1i1p1f1/day/$var$/gn/v20200114/$var$_day_BCC-CSM2-MR_dcppA-hindcast_s$sdate$-r1i1p1f1_gn_19800101-19891231.nc'
path_bcc_csm2 <- '/esarchive/exp/CMIP6/dcppA-hindcast/BCC-CSM2-MR/DCPP/BCC/BCC-CSM2-MR/dcppA-hindcast/r1i1p1f1/day/$var$/gn/v20200114/$var$_day_BCC-CSM2-MR_dcppA-hindcast_s$sdate$-r1i1p1f1_gn_19800101-19891231.nc'
suppressWarnings(
data <- Start(dat = path_bcc_csm2,
......@@ -79,9 +78,8 @@ expect_equal(
test_that("3. standard, daily, unit = 'days since 1850-1-1 00:00:00'", {
path_mpi_esm <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/',
'cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/',
path_mpi_esm <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/',
'DCPP/MPI-M/MPI-ESM1-2-HR/',
'dcppA-hindcast/r1i1p1f1/day/$var$/gn/v20200101/',
'$var$_day_MPI-ESM1-2-HR_dcppA-hindcast_s$sdate$-r1i1p1f1_gn_$fyear$.nc')
......@@ -120,8 +118,8 @@ expect_equal(
test_that("4. standard, monthly, unit = 'days since 1850-1-1 00:00:00'", {
path_mpi_esm <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/',
'cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/',
path_mpi_esm <- paste0('/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/',
'DCPP/MPI-M/MPI-ESM1-2-HR/',
'dcppA-hindcast/r1i1p1f1/Amon/$var$/gn/v20200320/',
'$var$_Amon_MPI-ESM1-2-HR_dcppA-hindcast_s$sdate$-r1i1p1f1_gn_200011-201012.nc')
sdate <- '2000'
......
......@@ -5,7 +5,7 @@
context("Start() using values() to define dependency relations")
path <- '/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-r1i1p1f2_gn_$chunk$.nc'
path <- '/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r1i1p1f2/Omon/tos/gn/v20200417/$var$_Omon_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-r1i1p1f2_gn_$chunk$.nc'
sdates <- c('2016', '2017', '2018')
chunks <- array(dim = c(chunk = 3, sdate = 3))
......
......@@ -426,13 +426,13 @@ suppressWarnings(
test_that("8. Two data sets, both have files but the first file is missing", {
path_list <- list(
MPI = list(name = 'MPI_ESM',
path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/',
'cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/',
path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/',
'DCPP/MPI-M/MPI-ESM1-2-HR/',
'dcppA-hindcast/$member$/day/$var$/gn/v20200101/',
'$var$_day_MPI-ESM1-2-HR_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc')),
Had = list(name = 'HadGEM3',
path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/',
'cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/',
path = paste0('/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/',
'DCPP/MOHC/HadGEM3-GC31-MM/',
'dcppA-hindcast/$member$/day/$var$/gn/v20200101/',
'$var$_day_HadGEM3-GC31-MM_dcppA-hindcast_s$sdate$-$member$_gn_$chunk$.nc')))
suppressWarnings(
......@@ -482,9 +482,9 @@ data <- Start(dataset = path_list,
)
expect_equal(
attr(data, 'Files'),
array(c("/esarchive/exp/CMIP6/dcppA-hindcast/mpi-esm1-2-hr/cmip6-dcppA-hindcast_i1p1/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/day/tasmin/gn/v20200101/tasmin_day_MPI-ESM1-2-HR_dcppA-hindcast_s2018-r1i1p1f1_gn_20181101-20281231.nc",
array(c("/esarchive/exp/CMIP6/dcppA-hindcast/MPI-ESM1-2-HR/DCPP/MPI-M/MPI-ESM1-2-HR/dcppA-hindcast/r1i1p1f1/day/tasmin/gn/v20200101/tasmin_day_MPI-ESM1-2-HR_dcppA-hindcast_s2018-r1i1p1f1_gn_20181101-20281231.nc",
NA, NA, NA, NA, NA, NA,
"/esarchive/exp/CMIP6/dcppA-hindcast/hadgem3-gc31-mm/cmip6-dcppA-hindcast_i1p1/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r2i1p1f2/day/tasmin/gn/v20200101/tasmin_day_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r2i1p1f2_gn_20181101-20281230.nc"),
"/esarchive/exp/CMIP6/dcppA-hindcast/HadGEM3-GC31-MM/DCPP/MOHC/HadGEM3-GC31-MM/dcppA-hindcast/r2i1p1f2/day/tasmin/gn/v20200101/tasmin_day_HadGEM3-GC31-MM_dcppA-hindcast_s2018-r2i1p1f2_gn_20181101-20281230.nc"),
dim = c(dataset = 2, var = 1, member = 2, sdate = 1, chunk = 2))
)
......
......@@ -615,6 +615,45 @@ datesF,
dates
)
# no return_vars
suppressWarnings(
data <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
'$var$/$var$_$file_date$.nc'),
var = 'tas',
file_date = file_date, #[syear = 3, smonth = 2]
time = indices(1:2),
latitude = indices(1),
longitude = indices(1),
split_multiselected_dims = TRUE,
# return_vars = list(latitude = NULL,
# longitude = NULL,
# time = 'file_date'),
retrieve = TRUE)
)
expect_equal(
names(attributes(data)$Variables$common),
'tas'
)
suppressWarnings(
data <- Start(dat = paste0('/esarchive/recon/ecmwf/erainterim/6hourly/',
'$var$/$var$_$file_date$.nc'),
var = 'tas',
file_date = file_date, #[syear = 3, smonth = 2]
time = indices(1:2),
latitude = indices(1),
longitude = indices(1),
split_multiselected_dims = TRUE,
# return_vars = list(latitude = NULL,
# longitude = NULL,
# time = 'file_date'),
retrieve = FALSE)
)
expect_equal(
names(attributes(data)$Variables$common),
NULL
)
})
test_that("9. split file dim that contains 'time', and 'time' inner dim is implicit", {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment