ByChunks(): fail chunking when using param 'split_multiselected_dims' in Start()
Thanks @cchou for reporting this issue. When the startR object is derived from Start() using parameter 'split_multiselected_dims', and the involved dimensions are not all used in operation (i.e., the 'target_dims' in Step()), the error shows:
Error in chunk(chunks_indices_split_dims, chunks_split_dims, eval(start_call[[names(split_dims)[k]]])) : Multidimensional chunking only available when selector values provided.
Before the bug is fixed, to avoid this error, try to use all the split dimensions (in the following example, 'time' and 'sdate') as 'target_dims'.
pname_exp <- "/esarchive/exp/ecmwf/system5c3s/daily_mean/$var$_f6h/$var$_$sdate$.nc"
# Define a region
lons.w <- 340
lons.e <- 359
lats.s <- 30
lats.n <- 45
yearini <- 1993
yearend <- 2019
exp <- Start(dat = pname_exp, var = "tas",
sdate = paste0(yearini:yearend, "0401"),
time = indices(1:(30 + 31)),
ensemble = "all",
longitude_reorder = CircularSort(0, 360),
longitude = values(list(lons.w, lons.e)),
latitude_reorder = Sort(),
latitude = values(list(lats.s, lats.n)),
return_vars = list(latitude = 'dat', #NULL,
longitude = 'dat',
time = c("sdate")),
retrieve = FALSE)
# Retrieve the attributes for obs
dates <- attr(exp, "Variables")$common$time
dates_file <- sort(unique(gsub('-', '', sapply(as.character(dates), substr, 1, 7))))
pname_obs <- "/esarchive/recon/ecmwf/era5/daily_mean/$var$_f1h/$var$_$file_date$.nc"
obs <- Start(dat = pname_obs,
var = "tas",
file_date = dates_file,
time = values(dates), # this dimension will be splitted into 'time' and 'sdate'
longitude_reorder = CircularSort(0, 360),
longitude = values(list(lons.w, lons.e)),
latitude_reorder = Sort(),
latitude = values(list(lats.s, lats.n)),
time_var = "time",
time_tolerance = as.difftime(1, units = "hours"),
time_across = "file_date",
merge_across_dims = TRUE,
split_multiselected_dims = TRUE,
transform = CDORemapper,
transform_extra_cells = 2,
transform_params = list(grid = "r360x181",
method = "conservative",
crop = c(lons.w, lons.e, lats.s, lats.n)),
transform_vars = c("latitude", "longitude"),
synonims = list(latitude = c('lat', 'latitude'),
longitude = c('lon', 'longitude')),
return_vars = list(latitude = 'dat',
longitude = 'dat',
time = c("file_date")),
retrieve = FALSE)
# The splitted dimensions are 'time' and 'sdate' in obs
# try a simple function
wrap_cal <- function(obs) {
return(obs)
}
step <- Step(wrap_cal,
target_dims = c('time','sdate'), # only works when the two dims are involved
output_dims = c('time','sdate')
)
wf <- AddStep(obs, step)
res <- Compute(wf,
chunks = list(latitude = 1, longitude = 1))