From 00a0f58ea97515a4b2adf73a74cb778a5a029f5d Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Wed, 11 Oct 2023 16:21:09 +0200
Subject: [PATCH 01/12] first draft

---
 .../tutorial/PATC2023/griddes_system7c3s.txt  |  19 +++
 .../PATC2023/handson_1-data-loading.md        | 124 ++++++++++++++++++
 2 files changed, 143 insertions(+)
 create mode 100644 inst/doc/tutorial/PATC2023/griddes_system7c3s.txt
 create mode 100644 inst/doc/tutorial/PATC2023/handson_1-data-loading.md

diff --git a/inst/doc/tutorial/PATC2023/griddes_system7c3s.txt b/inst/doc/tutorial/PATC2023/griddes_system7c3s.txt
new file mode 100644
index 0000000..b6f1847
--- /dev/null
+++ b/inst/doc/tutorial/PATC2023/griddes_system7c3s.txt
@@ -0,0 +1,19 @@
+# Grid description file for Meteofrance System 7 (C3S)
+# Serves as reference_grid for archive.ym
+#
+# gridID 2
+#
+gridtype  = lonlat
+gridsize  = 64800
+xsize     = 360
+ysize     = 180
+xname     = longitude
+xlongname = "longitude"
+xunits    = "degrees_east"
+yname     = latitude
+ylongname = "latitude"
+yunits    = "degrees_north"
+xfirst    = 0.5
+xinc      = 1
+yfirst    = 89.5
+yinc      = -1
diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
new file mode 100644
index 0000000..0eb4b0c
--- /dev/null
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -0,0 +1,124 @@
+# Hands-on 1: Load data by startR
+
+## Goal
+Use startR to load the data and learn how to adjust data while loading data.
+
+## 0. Load required packages
+
+```r
+# Clean the session
+rm(list = ls())
+
+library(startR)
+```
+## 1. Load data from data repository (esarchive/)
+
+#TODO: update it
+**Data description**:  
+This sample data set contains a small cutout of gridded seasonal precipitation 
+forecast data from the Copernicus Climate Change ECMWF-System 5 forecast system.
+Specifically, for the 'prlr' (precipitation) variable, for the first 6 forecast 
+ensemble members, daily values, for all 31 days in March following the forecast 
+starting dates in November of years 2010 to 2012, for a small 4x4 pixel cutout in 
+a region in the North-Western Italian Alps (44N-47N, 6E-9E). The data resolution is 1 degree.
+
+Use the above information to define the variable, start dates, longitude and latitude.
+
+```r
+#TODO: update the path
+  # Use this one if on workstation or nord3 (have access to /esarchive)
+  path_exp <- "/esarchive/exp/meteofrance/system7c3s/monthly_mean/$var$_f6h/$var$_$syear$.nc"
+
+  # Use this one if on Marenostrum4 and log in with PATC2021 account
+  path_exp <- paste0('/gpfs/scratch/nct01/nct01127/d3_R_handson/esarchive/',
+                  'exp/ecmwf/system5c3s/daily_mean/',
+                  '$var$_s0-24h/$var$_$sdate$.nc')
+
+  var <- 'tas'
+  sdate_hsct <- paste0(1993:2016, '1101')
+  sdate_fcst <- '20201101'
+  lon.min <- -20
+  lon.max <- 40
+  lat.min <- 20
+  lat.max <- 80
+```
+
+Use Start() to load the data.
+
+```r
+  hcst <- Start(dat = path_exp,
+                var = var,
+                syear = sdate_fcst,
+                ensemble = 'all',
+                time = 1:2,
+                latitude = values(list(lat.min, lat.max)),
+                latitude_reorder = Sort(),
+                longitude = values(list(lon.min, lon.max)),
+                longitude_reorder = CircularSort(-180, 180),
+                synonims = list(syear = c('syear', 'sdate'),
+                                latitude = c('lat', 'latitude'),
+                                longitude = c('lon', 'longitude')),
+                return_vars = list(time = 'syear',
+                                   longitude = NULL, latitude = NULL),
+                retrieve = TRUE)
+```
+   
+```r
+  fcst <- Start(dat = path_exp,
+                var = var,
+                syear = sdate_fcst,
+                ensemble = 'all',
+                time = 1:2, 
+                latitude = values(list(lat.min, lat.max)),
+                latitude_reorder = Sort(),
+                longitude = values(list(lon.min, lon.max)),
+                longitude_reorder = CircularSort(-180, 180),
+                synonims = list(syear = c('syear', 'sdate'),
+                                latitude = c('lat', 'latitude'),
+                                longitude = c('lon', 'longitude')),
+                return_vars = list(time = 'syear',
+                                   longitude = NULL, latitude = NULL),
+                retrieve = TRUE)
+```
+
+
+path_obs <- '/esarchive/recon/ecmwf/era5/monthly_mean/$var$_f1h-r1440x721cds/$var$_$syear$.nc'
+
+# Create an date array from hcst dates
+Check the time attributes of hcst: Is it correct? 
+To load the corresponding obs data, we can use these time values as the selectors in obs Start() call.
+
+dim(attributes(hcst)$Variables$common$time)
+syear  time
+   24     2
+
+str(attributes(hcst)$Variables$common$time)
+ POSIXct[1:48], format: "1993-12-01" "1994-12-01" "1995-12-01" "1996-12-01" "1997-12-01" ...
+
+# Adjust the day to the correct month
+attributes(hcst)$Variables$common$time <- attributes(hcst)$Variables$common$time - lubridate::days(1)
+
+date_string <- format(attributes(hcst)$Variables$common$time, '%Y%m')
+sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
+
+```r
+  obs <- Start(dat = path_obs,
+               var = var,
+               syear = sdate_obs,
+               latitude = values(list(lat.min, lat.max)),
+               latitude_reorder = Sort(),
+               longitude = values(list(lon.min, lon.max)),
+               longitude_reorder = CircularSort(-180, 180),
+               transform = CDORemapper,
+               transform_params = list(grid = './griddes_system7c3s.txt',
+                                       method = 'bilinear'),
+               transform_vars = c('latitude', 'longitude'),
+               synonims = list(syear = c('syear', 'sdate'),
+                               latitude = c('lat', 'latitude'),
+                               longitude = c('lon', 'longitude')),
+               return_vars = list(time = 'syear',
+                                  longitude = NULL, latitude = NULL),
+               retrieve = TRUE)
+```
+
+
-- 
GitLab


From f80f6fc46ba65e86cb6d9b5539b1ac4892a6c917 Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Wed, 11 Oct 2023 16:31:18 +0200
Subject: [PATCH 02/12] several fixes

---
 .../tutorial/PATC2023/handson_1-data-loading.md   | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index 0eb4b0c..0daf8eb 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -35,7 +35,7 @@ Use the above information to define the variable, start dates, longitude and lat
                   '$var$_s0-24h/$var$_$sdate$.nc')
 
   var <- 'tas'
-  sdate_hsct <- paste0(1993:2016, '1101')
+  sdate_hcst <- paste0(1993:2016, '1101')
   sdate_fcst <- '20201101'
   lon.min <- -20
   lon.max <- 40
@@ -48,7 +48,7 @@ Use Start() to load the data.
 ```r
   hcst <- Start(dat = path_exp,
                 var = var,
-                syear = sdate_fcst,
+                syear = sdate_hcst,
                 ensemble = 'all',
                 time = 1:2,
                 latitude = values(list(lat.min, lat.max)),
@@ -82,12 +82,12 @@ Use Start() to load the data.
 ```
 
 
-path_obs <- '/esarchive/recon/ecmwf/era5/monthly_mean/$var$_f1h-r1440x721cds/$var$_$syear$.nc'
-
 # Create an date array from hcst dates
 Check the time attributes of hcst: Is it correct? 
+
 To load the corresponding obs data, we can use these time values as the selectors in obs Start() call.
 
+```r
 dim(attributes(hcst)$Variables$common$time)
 syear  time
    24     2
@@ -100,8 +100,11 @@ attributes(hcst)$Variables$common$time <- attributes(hcst)$Variables$common$time
 
 date_string <- format(attributes(hcst)$Variables$common$time, '%Y%m')
 sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
+```
 
 ```r
+  path_obs <- '/esarchive/recon/ecmwf/era5/monthly_mean/$var$_f1h-r1440x721cds/$var$_$syear$.nc'
+
   obs <- Start(dat = path_obs,
                var = var,
                syear = sdate_obs,
@@ -110,7 +113,9 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
                longitude = values(list(lon.min, lon.max)),
                longitude_reorder = CircularSort(-180, 180),
                transform = CDORemapper,
-               transform_params = list(grid = './griddes_system7c3s.txt',
+#TODO: Change to relative path
+#               transform_params = list(grid = './griddes_system7c3s.txt',
+               transform_params = list(grid = '/esarchive/scratch/aho/git/startR/inst/doc/tutorial/PATC2023/griddes_system7c3s.txt',
                                        method = 'bilinear'),
                transform_vars = c('latitude', 'longitude'),
                synonims = list(syear = c('syear', 'sdate'),
-- 
GitLab


From e4d467e0b2e21a990db624813b68590df425396e Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Wed, 11 Oct 2023 16:40:57 +0200
Subject: [PATCH 03/12] Turn data into s2dv_cube

---
 inst/doc/tutorial/PATC2023/handson_1-data-loading.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index 0daf8eb..8aeb5fb 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -126,4 +126,12 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
                retrieve = TRUE)
 ```
 
+# Turn the data into s2dv_cube
 
+```r
+library(CSTools)
+
+hcst <- as.s2dv_cube(hcst)
+fcst <- as.s2dv_cube(fcst)
+obs <- as.s2dv_cube(obs)
+```
-- 
GitLab


From d9dc000e4527dc0d9426c5a9957784bccb25882a Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Wed, 11 Oct 2023 16:58:34 +0200
Subject: [PATCH 04/12] Remove as.s2dv_cube

---
 inst/doc/tutorial/PATC2023/handson_1-data-loading.md | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index 8aeb5fb..6b280a4 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -125,13 +125,3 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
                                   longitude = NULL, latitude = NULL),
                retrieve = TRUE)
 ```
-
-# Turn the data into s2dv_cube
-
-```r
-library(CSTools)
-
-hcst <- as.s2dv_cube(hcst)
-fcst <- as.s2dv_cube(fcst)
-obs <- as.s2dv_cube(obs)
-```
-- 
GitLab


From d5c5edd156ae06c5aa415877c3cd123d9587298c Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Mon, 16 Oct 2023 09:47:15 +0200
Subject: [PATCH 05/12] Add split_multiselected_dims = T in obs call. Add
 checks for dataset consistency

---
 .../PATC2023/handson_1-data-loading.md        | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index 6b280a4..69da163 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -108,6 +108,7 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
   obs <- Start(dat = path_obs,
                var = var,
                syear = sdate_obs,
+               split_multiselected_dims = TRUE,
                latitude = values(list(lat.min, lat.max)),
                latitude_reorder = Sort(),
                longitude = values(list(lon.min, lon.max)),
@@ -125,3 +126,23 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
                                   longitude = NULL, latitude = NULL),
                retrieve = TRUE)
 ```
+
+## 2. Check if the datasets are consistent
+
+```r
+lat_hcst <- attributes(hcst)$Variables$common$latitude
+lon_hcst <- attributes(hcst)$Variables$common$longitude
+lat_obs <- attributes(obs)$Variables$common$latitude
+lon_obs <- attributes(obs)$Variables$common$longitude
+
+identical(c(lat_obs), c(lat_hcst))
+[1] TRUE
+identical(c(lon_obs), c(lon_hcst))
+[1] TRUE
+
+time_hcst <- attributes(hcst)$Variables$common$time
+time_obs <- attributes(obs)$Variables$common$time
+
+identical(format(time_hcst, '%Y%m'), format(time_obs, '%Y%m'))
+[1] TRUE
+```
-- 
GitLab


From 0d93bf916754769539f149d330c50b1ef3ecbc87 Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Wed, 18 Oct 2023 12:57:45 +0200
Subject: [PATCH 06/12] Don't create pipeline when it is draft

---
 .gitlab-ci.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c7deb1a..db7b631 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,5 +1,13 @@
 stages:
   - build
+
+workflow:
+  rules:
+    - if: $CI_COMMIT_TITLE =~ /-draft$/
+      when: never
+#    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+#    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
 build:
   stage: build
   script:
-- 
GitLab


From 64eed86f53d49b141a6e4894cb15323e9cfc1393 Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Thu, 19 Oct 2023 14:50:30 +0200
Subject: [PATCH 07/12] Update path, add explanation

---
 .../PATC2023/handson_1-data-loading.md        | 73 +++++++++++--------
 1 file changed, 44 insertions(+), 29 deletions(-)

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index 69da163..8a986f8 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -1,7 +1,7 @@
 # Hands-on 1: Load data by startR
 
 ## Goal
-Use startR to load the data and learn how to adjust data while loading data.
+Use startR to load the data and learn how to adjust data structure while loading data.
 
 ## 0. Load required packages
 
@@ -11,32 +11,28 @@ rm(list = ls())
 
 library(startR)
 ```
-## 1. Load data from data repository (esarchive/)
-
-#TODO: update it
 **Data description**:  
-This sample data set contains a small cutout of gridded seasonal precipitation 
-forecast data from the Copernicus Climate Change ECMWF-System 5 forecast system.
-Specifically, for the 'prlr' (precipitation) variable, for the first 6 forecast 
-ensemble members, daily values, for all 31 days in March following the forecast 
-starting dates in November of years 2010 to 2012, for a small 4x4 pixel cutout in 
-a region in the North-Western Italian Alps (44N-47N, 6E-9E). The data resolution is 1 degree.
+We will use two datasets in the hands-on. The experiment data are Meteo-France System 7 from ECMWF, and the observation ones are ERA5 from ECMWF. The data have been first processed into monthly mean data and stored in our data archive (esarchive).
+
+We're going to analyze the near-surface temperature (short name: tas) for seasonal forecast. We will focus on the Europe region (roughly 20W-40E, 20N-80N). The hindcast years are 1993 to 2016, and the forecast year is 2020. The initial month is November. To speed up the practice, we will only load the first two forecast time steps, but all the ensemble members are used to give a less biased result.
+
+## 1. Load experimental data from data repository
+
+### 1.1 Hindcast data
 
-Use the above information to define the variable, start dates, longitude and latitude.
+Understand the following script, run it, and check the result. 
 
 ```r
-#TODO: update the path
   # Use this one if on workstation or nord3 (have access to /esarchive)
   path_exp <- "/esarchive/exp/meteofrance/system7c3s/monthly_mean/$var$_f6h/$var$_$syear$.nc"
-
-  # Use this one if on Marenostrum4 and log in with PATC2021 account
-  path_exp <- paste0('/gpfs/scratch/nct01/nct01127/d3_R_handson/esarchive/',
-                  'exp/ecmwf/system5c3s/daily_mean/',
-                  '$var$_s0-24h/$var$_$sdate$.nc')
+  #----------------------------------------------------------------------
+  # Run these two lines if you're on Marenostrum4 and log in with training account
+  prefix <- '/gpfs/scratch/bsc32/bsc32734/bsc_training_2023/R_handson/'
+  path_exp <- paste0(prefix, path_exp)
+  #----------------------------------------------------------------------
 
   var <- 'tas'
   sdate_hcst <- paste0(1993:2016, '1101')
-  sdate_fcst <- '20201101'
   lon.min <- -20
   lon.max <- 40
   lat.min <- 20
@@ -63,7 +59,15 @@ Use Start() to load the data.
                 retrieve = TRUE)
 ```
    
+### 1.2 Forecast data
+
+The forecast data are from the same dataset as hindcast, but with different years. 
+Therefore, they share the same data path and strucutre. 
+Try to take the Start() call above and modify it to load the forecast data (hint: the start year is 2020.)
+
 ```r
+  sdate_fcst <- '20201101'
+
   fcst <- Start(dat = path_exp,
                 var = var,
                 syear = sdate_fcst,
@@ -81,29 +85,42 @@ Use Start() to load the data.
                 retrieve = TRUE)
 ```
 
+### 1.3 Observational data
 
-# Create an date array from hcst dates
-Check the time attributes of hcst: Is it correct? 
+We need the corresponding observational data to compare with the experimental data.
+So, the observational data should be loaded as the same dimensions as the experimental ones.
+To achieve this, we can use the metadata of the experimental data as the selectors for observational data. But be careful with the usage! We must verify the correctness and applicability first. 
 
-To load the corresponding obs data, we can use these time values as the selectors in obs Start() call.
+**Get the time values from hindcast data**
+Check the time attributes of `hcst`: Is it correct? 
 
 ```r
 dim(attributes(hcst)$Variables$common$time)
-syear  time
-   24     2
+#syear  time
+#   24     2
 
 str(attributes(hcst)$Variables$common$time)
- POSIXct[1:48], format: "1993-12-01" "1994-12-01" "1995-12-01" "1996-12-01" "1997-12-01" ...
+# POSIXct[1:48], format: "1993-12-01" "1994-12-01" "1995-12-01" "1996-12-01" "1997-12-01" ...
+```
 
-# Adjust the day to the correct month
-attributes(hcst)$Variables$common$time <- attributes(hcst)$Variables$common$time - lubridate::days(1)
+The values are not correct since they should start from November, not December. 
+But the array has the correct dimensions and we can take advantage of it. 
+What we're going to do here is to tune the values one month ahead so we can have the correct dates. 
+(ps., `lubridate` is a useful R package for time value manipulation!)
 
+```r
+attributes(hcst)$Variables$common$time <- attributes(hcst)$Variables$common$time - lubridate::days(1)
 date_string <- format(attributes(hcst)$Variables$common$time, '%Y%m')
 sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
 ```
 
 ```r
   path_obs <- '/esarchive/recon/ecmwf/era5/monthly_mean/$var$_f1h-r1440x721cds/$var$_$syear$.nc'
+  #----------------------------------------------------------------------
+  # Run these two lines if you're on Marenostrum4 and log in with training account
+  prefix <- '/gpfs/scratch/bsc32/bsc32734/bsc_training_2023/R_handson/'
+  path_obs <- paste0(prefix, path_obs)
+  #----------------------------------------------------------------------
 
   obs <- Start(dat = path_obs,
                var = var,
@@ -114,9 +131,7 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
                longitude = values(list(lon.min, lon.max)),
                longitude_reorder = CircularSort(-180, 180),
                transform = CDORemapper,
-#TODO: Change to relative path
-#               transform_params = list(grid = './griddes_system7c3s.txt',
-               transform_params = list(grid = '/esarchive/scratch/aho/git/startR/inst/doc/tutorial/PATC2023/griddes_system7c3s.txt',
+               transform_params = list(grid = '/gpfs/scratch/bsc32/bsc32734/bsc_training_2023/R_handson/griddes_system7c3s.txt',
                                        method = 'bilinear'),
                transform_vars = c('latitude', 'longitude'),
                synonims = list(syear = c('syear', 'sdate'),
-- 
GitLab


From 822f8ff38654bc2b06268f8581c3e8ae869d2001 Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Thu, 19 Oct 2023 16:31:30 +0200
Subject: [PATCH 08/12] Change grid to r360x181

---
 .../PATC2023/handson_1-data-loading.md        | 116 ++++++++++++++----
 1 file changed, 89 insertions(+), 27 deletions(-)

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index 8a986f8..eb9a310 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -8,7 +8,7 @@ Use startR to load the data and learn how to adjust data structure while loading
 ```r
 # Clean the session
 rm(list = ls())
-
+# Load package
 library(startR)
 ```
 **Data description**:  
@@ -18,7 +18,7 @@ We're going to analyze the near-surface temperature (short name: tas) for season
 
 ## 1. Load experimental data from data repository
 
-### 1.1 Hindcast data
+### 1.a Hindcast data
 
 Understand the following script, run it, and check the result. 
 
@@ -31,26 +31,20 @@ Understand the following script, run it, and check the result.
   path_exp <- paste0(prefix, path_exp)
   #----------------------------------------------------------------------
 
-  var <- 'tas'
   sdate_hcst <- paste0(1993:2016, '1101')
-  lon.min <- -20
-  lon.max <- 40
-  lat.min <- 20
-  lat.max <- 80
-```
-
-Use Start() to load the data.
 
-```r
   hcst <- Start(dat = path_exp,
-                var = var,
+                var = 'tas',
                 syear = sdate_hcst,
                 ensemble = 'all',
                 time = 1:2,
-                latitude = values(list(lat.min, lat.max)),
+                latitude = values(list(20, 80)),
                 latitude_reorder = Sort(),
-                longitude = values(list(lon.min, lon.max)),
+                longitude = values(list(-20, 40)),
                 longitude_reorder = CircularSort(-180, 180),
+                transform = CDORemapper,
+                transform_params = list(grid = 'r360x181', method = 'bilinear'),
+                transform_vars = c('latitude', 'longitude'),
                 synonims = list(syear = c('syear', 'sdate'),
                                 latitude = c('lat', 'latitude'),
                                 longitude = c('lon', 'longitude')),
@@ -58,8 +52,34 @@ Use Start() to load the data.
                                    longitude = NULL, latitude = NULL),
                 retrieve = TRUE)
 ```
+
+**Questions**
+
+(1) What are the dimensions of hcst? Use `dim()` to check.
+
+```r
+dim(hcst)
+#      dat       var     syear  ensemble      time  latitude longitude 
+#        1         1        24        25         2        61        61 
+```
    
-### 1.2 Forecast data
+(2) What is the structure of hcst? Use `str()` to check.
+```r
+str(hcst, max.level = 1)
+str(hcst, max.level = 2)
+str(hcst, max.level = 3)
+```
+
+(3) The metadata variables are stored in `attr(hcst, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
+```r
+metadata_attr <- attr(hcst, 'Variables')
+str(metadata_attr)
+metadata_attr$common$time
+metadata_attr$common$latitude
+metadata_attr$common$longitude
+```
+
+### 1.b Forecast data
 
 The forecast data are from the same dataset as hindcast, but with different years. 
 Therefore, they share the same data path and strucutre. 
@@ -69,14 +89,17 @@ Try to take the Start() call above and modify it to load the forecast data (hint
   sdate_fcst <- '20201101'
 
   fcst <- Start(dat = path_exp,
-                var = var,
+                var = 'tas',
                 syear = sdate_fcst,
                 ensemble = 'all',
                 time = 1:2, 
-                latitude = values(list(lat.min, lat.max)),
+                latitude = values(list(20, 80)),
                 latitude_reorder = Sort(),
-                longitude = values(list(lon.min, lon.max)),
+                longitude = values(list(-20, 40)),
                 longitude_reorder = CircularSort(-180, 180),
+                transform = CDORemapper,
+                transform_params = list(grid = 'r360x181', method = 'bilinear'),
+                transform_vars = c('latitude', 'longitude'),
                 synonims = list(syear = c('syear', 'sdate'),
                                 latitude = c('lat', 'latitude'),
                                 longitude = c('lon', 'longitude')),
@@ -85,13 +108,42 @@ Try to take the Start() call above and modify it to load the forecast data (hint
                 retrieve = TRUE)
 ```
 
-### 1.3 Observational data
+**Questions**
+
+Check the forecast data by the same methods for hindcast data.
+
+(1) What are the dimensions of fcst? Use `dim()` to check.
+
+```r
+dim(fcst)
+#      dat       var     syear  ensemble      time  latitude longitude 
+#        1         1         1        51         2        61        61 
+```
+   
+(2) What is the structure of hcst? Use `str()` to check.
+```r
+str(fcst, max.level = 1)
+str(fcst, max.level = 2)
+str(fcst, max.level = 3)
+```
+
+(3) The metadata variables are stored in `attr(fcst, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
+```r
+metadata_attr <- attr(fcst, 'Variables')
+str(metadata_attr)
+metadata_attr$common$time
+metadata_attr$common$latitude
+metadata_attr$common$longitude
+```
+
+### 1.c Observational data
 
 We need the corresponding observational data to compare with the experimental data.
 So, the observational data should be loaded as the same dimensions as the experimental ones.
 To achieve this, we can use the metadata of the experimental data as the selectors for observational data. But be careful with the usage! We must verify the correctness and applicability first. 
 
 **Get the time values from hindcast data**
+
 Check the time attributes of `hcst`: Is it correct? 
 
 ```r
@@ -106,14 +158,19 @@ str(attributes(hcst)$Variables$common$time)
 The values are not correct since they should start from November, not December. 
 But the array has the correct dimensions and we can take advantage of it. 
 What we're going to do here is to tune the values one month ahead so we can have the correct dates. 
-(ps., `lubridate` is a useful R package for time value manipulation!)
+(p.s. `lubridate` is a useful R package for time value manipulation!)
 
 ```r
 attributes(hcst)$Variables$common$time <- attributes(hcst)$Variables$common$time - lubridate::days(1)
 date_string <- format(attributes(hcst)$Variables$common$time, '%Y%m')
 sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
+print(sdate_obs)
 ```
 
+Now we have the correct date values, we can use them as the selectors of `syear` in the Start() call. In addition, we will use the reshaping feature in startR to get the desired dimensions.
+
+#TODO: Explain split_multiselected_dims
+
 ```r
   path_obs <- '/esarchive/recon/ecmwf/era5/monthly_mean/$var$_f1h-r1440x721cds/$var$_$syear$.nc'
   #----------------------------------------------------------------------
@@ -123,16 +180,15 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
   #----------------------------------------------------------------------
 
   obs <- Start(dat = path_obs,
-               var = var,
+               var = 'tas',
                syear = sdate_obs,
                split_multiselected_dims = TRUE,
-               latitude = values(list(lat.min, lat.max)),
+               latitude = values(list(20, 80)),
                latitude_reorder = Sort(),
-               longitude = values(list(lon.min, lon.max)),
+               longitude = values(list(-20, 40)),
                longitude_reorder = CircularSort(-180, 180),
                transform = CDORemapper,
-               transform_params = list(grid = '/gpfs/scratch/bsc32/bsc32734/bsc_training_2023/R_handson/griddes_system7c3s.txt',
-                                       method = 'bilinear'),
+               transform_params = list(grid = 'r360x181', method = 'bilinear'),
                transform_vars = c('latitude', 'longitude'),
                synonims = list(syear = c('syear', 'sdate'),
                                latitude = c('lat', 'latitude'),
@@ -144,15 +200,21 @@ sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
 
 ## 2. Check if the datasets are consistent
 
+```r
+dim(hcst)
+dim(fcst)
+dim(obs)
+```
+
 ```r
 lat_hcst <- attributes(hcst)$Variables$common$latitude
 lon_hcst <- attributes(hcst)$Variables$common$longitude
 lat_obs <- attributes(obs)$Variables$common$latitude
 lon_obs <- attributes(obs)$Variables$common$longitude
 
-identical(c(lat_obs), c(lat_hcst))
+identical(lat_obs, lat_hcst)
 [1] TRUE
-identical(c(lon_obs), c(lon_hcst))
+identical(lon_obs, lon_hcst)
 [1] TRUE
 
 time_hcst <- attributes(hcst)$Variables$common$time
-- 
GitLab


From 8f18e513f5976e06bfc3433b17f4d12c64f7f4cb Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Fri, 20 Oct 2023 14:50:01 +0200
Subject: [PATCH 09/12] Finish the questions

---
 .../PATC2023/handson_1-data-loading.md        | 136 +++++----
 .../PATC2023/handson_1-data-loading_ans.md    | 272 ++++++++++++++++++
 2 files changed, 352 insertions(+), 56 deletions(-)
 create mode 100644 inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index eb9a310..bcd171c 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -11,6 +11,7 @@ rm(list = ls())
 # Load package
 library(startR)
 ```
+
 **Data description**:  
 We will use two datasets in the hands-on. The experiment data are Meteo-France System 7 from ECMWF, and the observation ones are ERA5 from ECMWF. The data have been first processed into monthly mean data and stored in our data archive (esarchive).
 
@@ -55,28 +56,26 @@ Understand the following script, run it, and check the result.
 
 **Questions**
 
-(1) What are the dimensions of hcst? Use `dim()` to check.
+(1) What are the dimensions of `hcst`? Use `dim()` to check.
 
 ```r
-dim(hcst)
-#      dat       var     syear  ensemble      time  latitude longitude 
-#        1         1        24        25         2        61        61 
+dim(____)
 ```
    
-(2) What is the structure of hcst? Use `str()` to check.
+(2) What is the structure of `hcst`? Use `str()` to check.
 ```r
-str(hcst, max.level = 1)
-str(hcst, max.level = 2)
-str(hcst, max.level = 3)
+str(hcst, max.level = _____)  # try 1, 2, 3
 ```
 
 (3) The metadata variables are stored in `attr(hcst, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
 ```r
 metadata_attr <- attr(hcst, 'Variables')
 str(metadata_attr)
-metadata_attr$common$time
-metadata_attr$common$latitude
-metadata_attr$common$longitude
+names(metadata_attr$common)
+
+hcst_time <- metadata_attr$common$time
+hcst_lat <- __________
+hcst_lon <- __________
 ```
 
 ### 1.b Forecast data
@@ -86,24 +85,24 @@ Therefore, they share the same data path and strucutre.
 Try to take the Start() call above and modify it to load the forecast data (hint: the start year is 2020.)
 
 ```r
-  sdate_fcst <- '20201101'
+  sdate_fcst <- ____________
 
   fcst <- Start(dat = path_exp,
-                var = 'tas',
+                var = _____,
                 syear = sdate_fcst,
                 ensemble = 'all',
-                time = 1:2, 
-                latitude = values(list(20, 80)),
+                time = _____, 
+                latitude = values(list(____, ____)),
                 latitude_reorder = Sort(),
-                longitude = values(list(-20, 40)),
+                longitude = values(list(____, ____)),
                 longitude_reorder = CircularSort(-180, 180),
                 transform = CDORemapper,
-                transform_params = list(grid = 'r360x181', method = 'bilinear'),
+                transform_params = list(grid = _____, method = 'bilinear'),
                 transform_vars = c('latitude', 'longitude'),
                 synonims = list(syear = c('syear', 'sdate'),
                                 latitude = c('lat', 'latitude'),
                                 longitude = c('lon', 'longitude')),
-                return_vars = list(time = 'syear',
+                return_vars = list(time = _____,
                                    longitude = NULL, latitude = NULL),
                 retrieve = TRUE)
 ```
@@ -112,28 +111,26 @@ Try to take the Start() call above and modify it to load the forecast data (hint
 
 Check the forecast data by the same methods for hindcast data.
 
-(1) What are the dimensions of fcst? Use `dim()` to check.
+(1) What are the dimensions of `fcst`? Use `dim()` to check.
 
 ```r
-dim(fcst)
-#      dat       var     syear  ensemble      time  latitude longitude 
-#        1         1         1        51         2        61        61 
+dim(____)
 ```
    
-(2) What is the structure of hcst? Use `str()` to check.
+(2) What is the structure of `fcst`? Use `str()` to check.
 ```r
-str(fcst, max.level = 1)
-str(fcst, max.level = 2)
-str(fcst, max.level = 3)
+str(hcst, max.level = _____)  # try 1, 2, 3
 ```
 
 (3) The metadata variables are stored in `attr(fcst, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
 ```r
-metadata_attr <- attr(fcst, 'Variables')
+metadata_attr <- attr(_____, 'Variables')
 str(metadata_attr)
-metadata_attr$common$time
-metadata_attr$common$latitude
-metadata_attr$common$longitude
+names(metadata_attr$common)
+
+fcst_time <- __________
+fcst_lat <- __________
+fcst_lon <- __________
 ```
 
 ### 1.c Observational data
@@ -148,11 +145,7 @@ Check the time attributes of `hcst`: Is it correct?
 
 ```r
 dim(attributes(hcst)$Variables$common$time)
-#syear  time
-#   24     2
-
 str(attributes(hcst)$Variables$common$time)
-# POSIXct[1:48], format: "1993-12-01" "1994-12-01" "1995-12-01" "1996-12-01" "1997-12-01" ...
 ```
 
 The values are not correct since they should start from November, not December. 
@@ -169,7 +162,11 @@ print(sdate_obs)
 
 Now we have the correct date values, we can use them as the selectors of `syear` in the Start() call. In addition, we will use the reshaping feature in startR to get the desired dimensions.
 
-#TODO: Explain split_multiselected_dims
+If the selector is an array, the parameter `split_multiselected_dims` of Start() splits the array by dimensions and we will get those dimensions in the output.
+For example, we will use `sdate_obs` as the selector of "syear" dimension below. 
+`sdate_obs` has two dimensions, "syear" and "time"; 
+so, by `split_multiselected_dims`, the output `obs` will have these two dimensions, 
+even "time" is not explicitly specified in the Start() call.
 
 ```r
   path_obs <- '/esarchive/recon/ecmwf/era5/monthly_mean/$var$_f1h-r1440x721cds/$var$_$syear$.nc'
@@ -180,46 +177,73 @@ Now we have the correct date values, we can use them as the selectors of `syear`
   #----------------------------------------------------------------------
 
   obs <- Start(dat = path_obs,
-               var = 'tas',
+               var = _____,
                syear = sdate_obs,
                split_multiselected_dims = TRUE,
-               latitude = values(list(20, 80)),
+               latitude = values(list(_____, _____)),
                latitude_reorder = Sort(),
-               longitude = values(list(-20, 40)),
+               longitude = values(list(_____, _____)),
                longitude_reorder = CircularSort(-180, 180),
                transform = CDORemapper,
-               transform_params = list(grid = 'r360x181', method = 'bilinear'),
+               transform_params = list(grid = ______, method = 'bilinear'),
                transform_vars = c('latitude', 'longitude'),
                synonims = list(syear = c('syear', 'sdate'),
                                latitude = c('lat', 'latitude'),
                                longitude = c('lon', 'longitude')),
-               return_vars = list(time = 'syear',
+               return_vars = list(time = ______,
                                   longitude = NULL, latitude = NULL),
                retrieve = TRUE)
 ```
 
-## 2. Check if the datasets are consistent
+**Questions**
+
+Check the obsercational data by the same methods above.
+
+(1) What are the dimensions of `obs`? Use `dim()` to check.
 
 ```r
-dim(hcst)
-dim(fcst)
-dim(obs)
+dim(____)
 ```
+   
+(2) What is the structure of `obs`? Use `str()` to check.
+```r
+str(obs, max.level = ____) # try 1, 2, 3
+```
+
+(3) The metadata variables are stored in `attr(obs, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
+```r
+metadata_attr <- attr(____, 'Variables')
+str(metadata_attr)
+names(metadata_attr$common)
+
+obs_time <- __________
+obs_lat <- __________
+obs_lon <- __________
+```
+
 
+## 2. Check if the datasets are consistent
+
+Wrong data, wrong everything afterward. It is important to examine the data and metadata after we load them. 
+
+(1) Compare the dimensions of the three data by `dim()`.
 ```r
-lat_hcst <- attributes(hcst)$Variables$common$latitude
-lon_hcst <- attributes(hcst)$Variables$common$longitude
-lat_obs <- attributes(obs)$Variables$common$latitude
-lon_obs <- attributes(obs)$Variables$common$longitude
 
-identical(lat_obs, lat_hcst)
-[1] TRUE
-identical(lon_obs, lon_hcst)
-[1] TRUE
+```
+(2) Check the summary of the data by `summary()`.
+```r
+summary(hcst)
+summary(fcst)
+summary(obs)
+```
 
-time_hcst <- attributes(hcst)$Variables$common$time
-time_obs <- attributes(obs)$Variables$common$time
+(3) Compare metadata. We have saved the latitude, longitude, and time attributes above after loading each data. 
+Use `identical()` or `all.equal()` to check if the values are consistent.
+```r
+# lat and lon
+identical(____, ____)
+all.equal(____, ____)
 
-identical(format(time_hcst, '%Y%m'), format(time_obs, '%Y%m'))
-[1] TRUE
+# time: only compare year and month
+identical(format(hcst_time, '%Y%m'), format(obs_time, '%Y%m'))
 ```
diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md
new file mode 100644
index 0000000..635be93
--- /dev/null
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md
@@ -0,0 +1,272 @@
+# Hands-on 1: Load data by startR
+
+## Goal
+Use startR to load the data and learn how to adjust data structure while loading data.
+
+## 0. Load required packages
+
+```r
+# Clean the session
+rm(list = ls())
+# Load package
+library(startR)
+```
+
+**Data description**:  
+We will use two datasets in the hands-on. The experiment data are Meteo-France System 7 from ECMWF, and the observation ones are ERA5 from ECMWF. The data have been first processed into monthly mean data and stored in our data archive (esarchive).
+
+We're going to analyze the near-surface temperature (short name: tas) for seasonal forecast. We will focus on the Europe region (roughly 20W-40E, 20N-80N). The hindcast years are 1993 to 2016, and the forecast year is 2020. The initial month is November. To speed up the practice, we will only load the first two forecast time steps, but all the ensemble members are used to give a less biased result.
+
+## 1. Load experimental data from data repository
+
+### 1.a Hindcast data
+
+Understand the following script, run it, and check the result. 
+
+```r
+  # Use this one if on workstation or nord3 (have access to /esarchive)
+  path_exp <- "/esarchive/exp/meteofrance/system7c3s/monthly_mean/$var$_f6h/$var$_$syear$.nc"
+  #----------------------------------------------------------------------
+  # Run these two lines if you're on Marenostrum4 and log in with training account
+  prefix <- '/gpfs/scratch/bsc32/bsc32734/bsc_training_2023/R_handson/'
+  path_exp <- paste0(prefix, path_exp)
+  #----------------------------------------------------------------------
+
+  sdate_hcst <- paste0(1993:2016, '1101')
+
+  hcst <- Start(dat = path_exp,
+                var = 'tas',
+                syear = sdate_hcst,
+                ensemble = 'all',
+                time = 1:2,
+                latitude = values(list(20, 80)),
+                latitude_reorder = Sort(),
+                longitude = values(list(-20, 40)),
+                longitude_reorder = CircularSort(-180, 180),
+                transform = CDORemapper,
+                transform_params = list(grid = 'r360x181', method = 'bilinear'),
+                transform_vars = c('latitude', 'longitude'),
+                synonims = list(syear = c('syear', 'sdate'),
+                                latitude = c('lat', 'latitude'),
+                                longitude = c('lon', 'longitude')),
+                return_vars = list(time = 'syear',
+                                   longitude = NULL, latitude = NULL),
+                retrieve = TRUE)
+```
+
+**Questions**
+
+(1) What are the dimensions of `hcst`? Use `dim()` to check.
+
+```r
+dim(hcst)
+#      dat       var     syear  ensemble      time  latitude longitude 
+#        1         1        24        25         2        61        61 
+```
+   
+(2) What is the structure of `hcst`? Use `str()` to check.
+```r
+str(hcst, max.level = 1)
+str(hcst, max.level = 2)
+str(hcst, max.level = 3)
+```
+
+(3) The metadata variables are stored in `attr(hcst, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
+```r
+metadata_attr <- attr(hcst, 'Variables')
+str(metadata_attr)
+names(metadata_attr$common)
+
+hcst_time <- metadata_attr$common$time
+hcst_lat <- metadata_attr$common$latitude
+hcst_lon <- metadata_attr$common$longitude
+```
+
+### 1.b Forecast data
+
+The forecast data are from the same dataset as hindcast, but with different years. 
+Therefore, they share the same data path and strucutre. 
+Try to take the Start() call above and modify it to load the forecast data (hint: the start year is 2020.)
+
+```r
+  sdate_fcst <- '20201101'
+
+  fcst <- Start(dat = path_exp,
+                var = 'tas',
+                syear = sdate_fcst,
+                ensemble = 'all',
+                time = 1:2, 
+                latitude = values(list(20, 80)),
+                latitude_reorder = Sort(),
+                longitude = values(list(-20, 40)),
+                longitude_reorder = CircularSort(-180, 180),
+                transform = CDORemapper,
+                transform_params = list(grid = 'r360x181', method = 'bilinear'),
+                transform_vars = c('latitude', 'longitude'),
+                synonims = list(syear = c('syear', 'sdate'),
+                                latitude = c('lat', 'latitude'),
+                                longitude = c('lon', 'longitude')),
+                return_vars = list(time = 'syear',
+                                   longitude = NULL, latitude = NULL),
+                retrieve = TRUE)
+```
+
+**Questions**
+
+Check the forecast data by the same methods for hindcast data.
+
+(1) What are the dimensions of `fcst`? Use `dim()` to check.
+
+```r
+dim(fcst)
+#      dat       var     syear  ensemble      time  latitude longitude 
+#        1         1         1        51         2        61        61 
+```
+   
+(2) What is the structure of `fcst`? Use `str()` to check.
+```r
+str(fcst, max.level = 1)
+str(fcst, max.level = 2)
+str(fcst, max.level = 3)
+```
+
+(3) The metadata variables are stored in `attr(fcst, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
+```r
+metadata_attr <- attr(fcst, 'Variables')
+str(metadata_attr)
+names(metadata_attr$common)
+
+fcst_time <- metadata_attr$common$time
+fcst_lat <- metadata_attr$common$latitude
+fcst_lon <- metadata_attr$common$longitude
+```
+
+### 1.c Observational data
+
+We need the corresponding observational data to compare with the experimental data.
+So, the observational data should be loaded as the same dimensions as the experimental ones.
+To achieve this, we can use the metadata of the experimental data as the selectors for observational data. But be careful with the usage! We must verify the correctness and applicability first. 
+
+**Get the time values from hindcast data**
+
+Check the time attributes of `hcst`: Is it correct? 
+
+```r
+dim(attributes(hcst)$Variables$common$time)
+#syear  time
+#   24     2
+
+str(attributes(hcst)$Variables$common$time)
+# POSIXct[1:48], format: "1993-12-01" "1994-12-01" "1995-12-01" "1996-12-01" "1997-12-01" ...
+```
+
+The values are not correct since they should start from November, not December. 
+But the array has the correct dimensions and we can take advantage of it. 
+What we're going to do here is to tune the values one month ahead so we can have the correct dates. 
+(p.s. `lubridate` is a useful R package for time value manipulation!)
+
+```r
+attributes(hcst)$Variables$common$time <- attributes(hcst)$Variables$common$time - lubridate::days(1)
+date_string <- format(attributes(hcst)$Variables$common$time, '%Y%m')
+sdate_obs <- array(date_string, dim = c(syear = 24, time = 2))
+print(sdate_obs)
+```
+
+Now we have the correct date values, we can use them as the selectors of `syear` in the Start() call. In addition, we will use the reshaping feature in startR to get the desired dimensions.
+
+If the selector is an array, the parameter `split_multiselected_dims` of Start() splits the array by dimensions and we will get those dimensions in the output.
+For example, we will use `sdate_obs` as the selector of "syear" dimension below. 
+`sdate_obs` has two dimensions, "syear" and "time"; 
+so, by `split_multiselected_dims`, the output `obs` will have these two dimensions, 
+even "time" is not explicitly specified in the Start() call.
+
+```r
+  path_obs <- '/esarchive/recon/ecmwf/era5/monthly_mean/$var$_f1h-r1440x721cds/$var$_$syear$.nc'
+  #----------------------------------------------------------------------
+  # Run these two lines if you're on Marenostrum4 and log in with training account
+  prefix <- '/gpfs/scratch/bsc32/bsc32734/bsc_training_2023/R_handson/'
+  path_obs <- paste0(prefix, path_obs)
+  #----------------------------------------------------------------------
+
+  obs <- Start(dat = path_obs,
+               var = 'tas',
+               syear = sdate_obs,
+               split_multiselected_dims = TRUE,
+               latitude = values(list(20, 80)),
+               latitude_reorder = Sort(),
+               longitude = values(list(-20, 40)),
+               longitude_reorder = CircularSort(-180, 180),
+               transform = CDORemapper,
+               transform_params = list(grid = 'r360x181', method = 'bilinear'),
+               transform_vars = c('latitude', 'longitude'),
+               synonims = list(syear = c('syear', 'sdate'),
+                               latitude = c('lat', 'latitude'),
+                               longitude = c('lon', 'longitude')),
+               return_vars = list(time = 'syear',
+                                  longitude = NULL, latitude = NULL),
+               retrieve = TRUE)
+```
+
+**Questions**
+
+Check the obsercational data by the same methods above.
+
+(1) What are the dimensions of `obs`? Use `dim()` to check.
+
+```r
+dim(obs)
+#      dat       var     syear       time  latitude longitude 
+#        1         1         24         2        61        61 
+```
+   
+(2) What is the structure of `obs`? Use `str()` to check.
+```r
+str(obs, max.level = 1)
+str(obs, max.level = 2)
+str(obs, max.level = 3)
+```
+
+(3) The metadata variables are stored in `attr(obs, 'Variables')`. What variables do we have? Use `str()` to check the structure first, then try to access the variable values.
+```r
+metadata_attr <- attr(obs, 'Variables')
+str(metadata_attr)
+names(metadata_attr$common)
+
+obs_time <- metadata_attr$common$time
+obs_lat <- metadata_attr$common$latitude
+obs_lon <- metadata_attr$common$longitude
+```
+
+
+## 2. Check if the datasets are consistent
+
+Wrong data, wrong everything afterward. It is important to examine the data and metadata after we load them. 
+
+(1) Compare the dimensions of the three data by `dim()`.
+```r
+dim(hcst)
+dim(fcst)
+dim(obs)
+```
+(2) Check the summary of the data by `summary()`.
+```r
+summary(hcst)
+summary(fcst)
+summary(obs)
+```
+
+(3) Compare metadata. We have saved the latitude, longitude, and time attributes above after loading each data. 
+Use `identical()` or `all.equal()` to check if the values are consistent.
+```r
+identical(obs_lat, hcst_lat)
+[1] TRUE
+identical(obs_lon, hcst_lon)
+[1] TRUE
+identical(fcst_lat, hcst_lat)
+[1] TRUE
+identical(fcst_lon, hcst_lon)
+[1] TRUE
+
+identical(format(hcst_time, '%Y%m'), format(obs_time, '%Y%m'))
+[1] TRUE
+```
-- 
GitLab


From b9de2964ddb6ad83a8f92cea589a9d8d72e58c6c Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Fri, 20 Oct 2023 14:52:09 +0200
Subject: [PATCH 10/12] Run if the commit doesn't end with -draft

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index db7b631..7706518 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,6 +5,7 @@ workflow:
   rules:
     - if: $CI_COMMIT_TITLE =~ /-draft$/
       when: never
+    - when: always
 #    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 #    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
-- 
GitLab


From 71396fca4c8ff5863fa9daeb7f84cbf31608960d Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Mon, 23 Oct 2023 12:29:47 +0200
Subject: [PATCH 11/12] remove unecessary elements

---
 inst/doc/tutorial/PATC2023/handson_1-data-loading.md     | 9 +++------
 inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md | 9 +++------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
index bcd171c..1a61862 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading.md
@@ -46,8 +46,7 @@ Understand the following script, run it, and check the result.
                 transform = CDORemapper,
                 transform_params = list(grid = 'r360x181', method = 'bilinear'),
                 transform_vars = c('latitude', 'longitude'),
-                synonims = list(syear = c('syear', 'sdate'),
-                                latitude = c('lat', 'latitude'),
+                synonims = list(latitude = c('lat', 'latitude'),
                                 longitude = c('lon', 'longitude')),
                 return_vars = list(time = 'syear',
                                    longitude = NULL, latitude = NULL),
@@ -99,8 +98,7 @@ Try to take the Start() call above and modify it to load the forecast data (hint
                 transform = CDORemapper,
                 transform_params = list(grid = _____, method = 'bilinear'),
                 transform_vars = c('latitude', 'longitude'),
-                synonims = list(syear = c('syear', 'sdate'),
-                                latitude = c('lat', 'latitude'),
+                synonims = list(latitude = c('lat', 'latitude'),
                                 longitude = c('lon', 'longitude')),
                 return_vars = list(time = _____,
                                    longitude = NULL, latitude = NULL),
@@ -187,8 +185,7 @@ even "time" is not explicitly specified in the Start() call.
                transform = CDORemapper,
                transform_params = list(grid = ______, method = 'bilinear'),
                transform_vars = c('latitude', 'longitude'),
-               synonims = list(syear = c('syear', 'sdate'),
-                               latitude = c('lat', 'latitude'),
+               synonims = list(latitude = c('lat', 'latitude'),
                                longitude = c('lon', 'longitude')),
                return_vars = list(time = ______,
                                   longitude = NULL, latitude = NULL),
diff --git a/inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md b/inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md
index 635be93..d0d4b07 100644
--- a/inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md
+++ b/inst/doc/tutorial/PATC2023/handson_1-data-loading_ans.md
@@ -46,8 +46,7 @@ Understand the following script, run it, and check the result.
                 transform = CDORemapper,
                 transform_params = list(grid = 'r360x181', method = 'bilinear'),
                 transform_vars = c('latitude', 'longitude'),
-                synonims = list(syear = c('syear', 'sdate'),
-                                latitude = c('lat', 'latitude'),
+                synonims = list(latitude = c('lat', 'latitude'),
                                 longitude = c('lon', 'longitude')),
                 return_vars = list(time = 'syear',
                                    longitude = NULL, latitude = NULL),
@@ -103,8 +102,7 @@ Try to take the Start() call above and modify it to load the forecast data (hint
                 transform = CDORemapper,
                 transform_params = list(grid = 'r360x181', method = 'bilinear'),
                 transform_vars = c('latitude', 'longitude'),
-                synonims = list(syear = c('syear', 'sdate'),
-                                latitude = c('lat', 'latitude'),
+                synonims = list(latitude = c('lat', 'latitude'),
                                 longitude = c('lon', 'longitude')),
                 return_vars = list(time = 'syear',
                                    longitude = NULL, latitude = NULL),
@@ -199,8 +197,7 @@ even "time" is not explicitly specified in the Start() call.
                transform = CDORemapper,
                transform_params = list(grid = 'r360x181', method = 'bilinear'),
                transform_vars = c('latitude', 'longitude'),
-               synonims = list(syear = c('syear', 'sdate'),
-                               latitude = c('lat', 'latitude'),
+               synonims = list(latitude = c('lat', 'latitude'),
                                longitude = c('lon', 'longitude')),
                return_vars = list(time = 'syear',
                                   longitude = NULL, latitude = NULL),
-- 
GitLab


From 3a2b89dd5508d0c16c80736d507f340336c512b1 Mon Sep 17 00:00:00 2001
From: aho <an.ho@bsc.es>
Date: Thu, 26 Oct 2023 17:20:17 +0200
Subject: [PATCH 12/12] hide the potential changes

---
 .gitlab-ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7706518..200b32d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,11 +1,11 @@
 stages:
   - build
 
-workflow:
-  rules:
-    - if: $CI_COMMIT_TITLE =~ /-draft$/
-      when: never
-    - when: always
+#workflow:
+#  rules:
+#    - if: $CI_COMMIT_TITLE =~ /-draft$/
+#      when: never
+#    - when: always
 #    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
 #    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
-- 
GitLab