From 2a6d8bb729d3ba35223fc164b740b993f4e34097 Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Thu, 8 Oct 2020 17:09:15 +0200
Subject: [PATCH 1/3] Fix metadata reading bug when certain dataset is missing

---
 R/Start.R | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/R/Start.R b/R/Start.R
index 463d915..718ae42 100644
--- a/R/Start.R
+++ b/R/Start.R
@@ -4141,9 +4141,14 @@ Start <- function(..., # dim = indices/selectors,
         loaded_metadata_count <- 1
         for (kk in 1:length(return_metadata)) {
           for (jj in 1:length(return_metadata[[kk]])) {
-            return_metadata[[kk]][jj] <- loaded_metadata[[loaded_metadata_count]]
-            names(return_metadata[[kk]])[jj] <- names(loaded_metadata[[loaded_metadata_count]])
-            loaded_metadata_count <- loaded_metadata_count + 1
+            if (dataset_has_files[kk]) {
+              return_metadata[[kk]][jj] <- loaded_metadata[[loaded_metadata_count]]
+              names(return_metadata[[kk]])[jj] <- names(loaded_metadata[[loaded_metadata_count]])
+              loaded_metadata_count <- loaded_metadata_count + 1
+            } else {
+              return_metadata[[kk]][jj] <- NULL
+            }
+
           }
         }
       }
@@ -4230,8 +4235,10 @@ Start <- function(..., # dim = indices/selectors,
       var_backup <- attr(data_array, 'Variables')
       for (kk in 1:length(var_backup)) {
         sublist_names <- lapply(var_backup, names)[[kk]]
-        for (jj in 1:length(sublist_names)) {
-          picked_vars[[kk]][[sublist_names[jj]]] <- var_backup[[kk]][[jj]]
+        if (!is.null(sublist_names)) {
+          for (jj in 1:length(sublist_names)) {
+            picked_vars[[kk]][[sublist_names[jj]]] <- var_backup[[kk]][[jj]]
+          }
         }
       }
       var_backup <- NULL
-- 
GitLab


From fb1c3888db490395522f7c589c33f8207946178a Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Thu, 8 Oct 2020 18:17:07 +0200
Subject: [PATCH 2/3] Add new unit test for missing dataset test

---
 tests/testthat/test-Start-metadata_dims.R | 45 +++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/tests/testthat/test-Start-metadata_dims.R b/tests/testthat/test-Start-metadata_dims.R
index b4ef3cf..0e53f07 100644
--- a/tests/testthat/test-Start-metadata_dims.R
+++ b/tests/testthat/test-Start-metadata_dims.R
@@ -329,3 +329,48 @@ data <- Start(repos = mask_path,
   )
 
 })
+
+test_that("7. Two data sets, while one is missing", {
+ repos <- "/esarchive/exp/ecmwf/system5_m1/monthly_mean/$var$_f6h/$var$_$sdate$.nc"
+ # incorrect path. Therefore repos2 doesn't have any valid files 
+ repos2 <- "/esarchive/exp/ecmwf/system4_m1/monthly_mean/$var$_f2h/$var$_$sdate$.nc"  # correct one is _f6h
+  var <- 'tas'
+  data <- Start(dat = list(list(name = 'system4_m1', path = repos2),
+                           list(name = 'system5_m1', path = repos)),
+                var = var,
+                sdate = '20170101',
+                ensemble = indices(1),
+                time = indices(1),
+                lat = 'all',
+                lon = 'all',
+                synonims = list(lat = c('lat', 'latitude'),
+                                lon = c('lon', 'longitude')),
+                return_vars = list(time = 'sdate',
+                                   longitude = 'dat',
+                                   latitude = 'dat'),
+                metadata_dims = 'dat',  # it can be omitted since it is automatically specified as 'dat'
+                retrieve = T
+               )
+
+  expect_equal(
+  length(data[is.na(data)]),
+  829440
+  )
+  expect_equal(
+  attr(data, "Variables")$system4_m1,
+  NULL
+  )
+  expect_equal(
+  length(attr(data, "Variables")$system5_m1$longitude),
+  1296
+  )
+  expect_equal(
+  length(attr(data, "Variables")$system5_m1),
+  3
+  )
+  expect_equal(
+  attr(data, 'Files'),
+  array(c(NA, "/esarchive/exp/ecmwf/system5_m1/monthly_mean/tas_f6h/tas_20170101.nc"),
+        dim = c(dat = 2, var = 1, sdate = 1))
+  )
+})
-- 
GitLab


From a552968c85aefd184591e98de9d8d53a1344872c Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Thu, 8 Oct 2020 18:19:32 +0200
Subject: [PATCH 3/3] Add new fix in NEWS.md

---
 NEWS.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/NEWS.md b/NEWS.md
index 295a972..ba58630 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,6 @@
+# startR v2.0.4 (Release date: )
+- Bugfix for metadata retrieving when there are more than one dataset and one of them is missing.
+
 # startR v2.0.1 (Release date: 2020-09-10)
 - /dev/shm automatic cleaning on Compute()
 # startR v2.0.1 (Release date: 2020-08-25)
-- 
GitLab