---
author: "Nuria Perez"
date: "`r Sys.Date()`"
output: rmarkdown::html_vignette
vignette: >
  %\VignetteEngine{knitr::knitr}
  %\VignetteIndexEntry{Create a High Resolution Dataset}
  %\usepackage[utf8]{inputenc}
---

Create a High Resolution Dataset
-------------------------------------------

Seasonal Forecast has been proved to be be decisive in sectorial decisions and stakeholders management. However, the data in the desired format of decision makers is not always available. Here, it is presented a step by step document to create your own High Resolution Dataset for your period and area of interest.

### 1.- System requirements

CSTools has a system dependency, the CDO libraries, for interpolation of grid data and retrieval of metadata. Make sure you have these libraries installed in the system or download and install from https://code.zmaw.de/projects/cdo.

A critical step on this process is the storage structure of the datasets you would like to use. The retrieve of data from local files, *CSTools* is based in *s2dverification* R package. It can loaded almost any NetCDF 3/4 data file containing climate data in monthly or daily time resolution in regular or gaussian grids. Find out more information in the s2dverification GitLab project https://earth.bsc.es/gitlab/es/s2dverification/blob/master/vignettes/data_retrieval.md

Open an R session and run the following lines to install and load the required packages:

Library *CSTools*, should be installed from CRAN and loaded:


```r
library(CSTools)
#library(s2dverification)
library(zeallot)
#library(ClimProjDiags)
```

### 2.- Load Data

A detailed explanation of the procedure to download public available data and loaded using *CSTools* would be available soon.

If you want to reproduce this vignette, contact the maintainer to request the sample data used.

To explore that data you can run the following commands to see: the names of the list of the 's2dv_cube' objects.

```
startdates <- sapply(1999:2009, function(x) {
               paste0(x, sprintf('%02d', 6), '01')})
c(exp, obs) %<-% CST_Load(var = 'psl', exp = 'system5c3s',
                 obs = 'era5', leadtimemax = 92,
                 sdates =  startdates,
                 #latmin = 30, latmax = 70, lonmin = -60, lonmax = 40,
                 latmin = 25, latmax = 57, lonmin = -21, lonmax = 51,
                 output = 'lonlat', storefreq = "daily",  nprocs = 1,
                 configfile = "/esarchive/scratch/nperez/BSC.conf")
```

3.- Compute Anomalies for slp or gz large scale fields

```
source("/esarchive/scratch/nperez/git/cstools/R/CST_Anomaly.R")
library(s2dverification) #or source("/esarchive/scratch/nperez/git/s2dverification/R/Clim.R")
c(ano_exp, ano_obs) %<-% CST_Anomaly(exp = exp, obs = obs,
                                     cross = FALSE, memb = FALSE,
                                     filter_span = 1, dim_anom = 3)
```

4.- Load the ERA5 field for the desired variable

```
startdates <- sapply(1999:2009, function(x) {
               paste0(x, sprintf('%02d', 6), '01')})
era5_tmax %<-% CST_Load(var = 'tasmax',
                 obs = 'era5', leadtimemax = 92,
                 sdates =  startdates,
                 latmin = 20, latmax = 52.5, lonmin = -17.5, lonmax = 40,
                 output = 'lonlat', storefreq = "daily",  nprocs = 1,
                 configfile = "/esarchive/scratch/nperez/BSC.conf")
```

5.- Compute the Analogs and obtain the downscalled field

First step obtain a sigle time dimension for observations:


```
source("/esarchive/scratch/nperez/git/cstools/R/CST_MergeDims.R")
library(abind)
ano_obs <- CST_MergeDims(data = ano_obs, merge_dims = c('ftime', 'sdate'),
                         rename_dim = 'time')
```


Then, the downscaled dataset is created by running:


```
source("/esarchive/scratch/nperez/git/cstools/R/CST_Analogs.R")

psl_system5c3s_to_era5res <- CST_Analogs(expL = ano_exp, obsL = ano_obs,
                                         obsVar = era5_tmax,
                                         criteria = 'Large_dist')
```


#--------------------------------------------------------------
## Using old Analogs functions:

# Case 1:
downscaled <- CST_Analogs(expL = ano_exp, obsL = ano_obs) #fails
dim(ano_obs$data)
dim(ano_exp$data)

# Case 2:
source("/esarchive/scratch/nperez/git/cstools/R/CST_MergeDims.R")
library(abind)
ano_obs <- CST_MergeDims(data = ano_obs, merge_dims = c('ftime', 'sdate'), 
                         rename_dim = 'time')
dim(ano_obs$data)
dim(ano_exp$data)
downscaled <- CST_Analogs(expL = ano_exp, obsL = ano_obs) #fails

# Case 3:
ano_exp1 <- ano_exp
ano_exp1$data <- ano_exp1$data[1,1, , , , ]
downscaled <- CST_Analogs(expL = ano_exp1, obsL = ano_obs)

# Case 4:
ano_exp1 <- ano_exp
ano_exp1$data <- ano_exp1$data[1,1, 1, , , ]
downscaled <- CST_Analogs(expL = ano_exp1, obsL = ano_obs) #It works


# Using Analogs()
downscaled2 <- Analogs(expL = ano_exp$data, obsL = ano_obs$data,
                       time_obsL = ano_obs$Dates)# fails
downscaled2 <- Analogs(expL = ano_exp1$data, obsL = ano_obs$data,
                       time_obsL = ano_obs$Dates)# fails

# Using Select
output <- CSTools:::Select(expL = ano_exp$data[1, 1, , , , ], obsL = ano_obs$data) #fails

output <- CSTools:::Select(expL = ano_exp$data[1, 1, 1, , , ], obsL = ano_obs$data) #works
sum((ano_exp$data[1,1,1,1,,] - ano_obs$data[1,1,1,1,,])^2) # correct
PlotEquiMap(ano_exp$data[1, 1, 1, 1, , ], lon = ano_exp$lon, lat = ano_exp$lat,
            filled.continents = FALSE, 
    fileout = "/esarchive/scratch/nperez/Forecast1_1999-t1.png")
PlotEquiMap(ano_obs$data[1, 1, 1, 1, , ], lon = ano_exp$lon, lat = ano_exp$lat,
            filled.continents = FALSE,
    fileout = "/esarchive/scratch/nperez/Obs1_1999-t1.png")


save(ano_exp, ano_obs, 
     file = "/esarchive/scratch/nperez/git/cstools/vignettes/psl_vignette.RData")   
# Correcting CST_Analogs
source("/esarchive/scratch/nperez/git/cstools/R/CST_Analogs.R")
library(multiApply)
library(parallel)
result <- Select(expL = ano_exp$data, obsL = ano_obs$data,
                 ncores = 2)
result2 <- BestAnalog(position = result$position, nAnalogs = 1)


PlotEquiMap(ano_exp$data[1,20, 4, 1, , ], lon = ano_exp$lon,
            lat = ano_exp$lat, filled.continents = FALSE)
dev.new()
PlotEquiMap(ano_obs$data[1, 1, 46, , ], lon = ano_obs$lon,
            lat = ano_obs$lat, filled.continents = FALSE)

sum((ano_obs$data[1, 1, 46, , ] - ano_exp$data[1,20, 4, 1, , ])^2)





output <- CSTools:::Select(expL = ano_exp$data[1, 1, , , , ], obsL = ano_obs$data) #fails



dim(obs$data)
dim(exp$data)
PlotEquiMap(Mean1Dim(Mean1Dim(obs$data, 3), 3), lon = obs$lon, lat = obs$lat)
dev.new()
PlotEquiMap(Mean1Dim(Mean1Dim(Mean1Dim(exp$data, 2), 2), 2), lon = exp$lon, lat = exp$lat)
all(obs$lon == exp$lon)
all(obs$lat == exp$lat)
```

