## TEST ANALOGS (2nd TEST, AFTER FIRST FEEDBACK AND IMPROVEMENTS)

library(s2dv)
library(CSTools)
library(FNN)
library(multiApply)

# after modifications in https://earth.bsc.es/gitlab/es/csdownscale/-/blob/dev-Rpackage/R/Analogs.R
# allabres@bscearth393:/esarchive/scratch/allabres/gitlab_clones/csdownscale> git pull origin
# and test again that everything is working as expected:
    # check previous issues with lat_dim and lon_dim
    # check previous issues with NA
    

source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')

source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')

output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'

# TEST DUMMY DATA
    ##'@examples
        exp <- rnorm(15000)
        dim(exp) <- c(member = 5, lat = 4, lon = 5, sdate = 5, time  = 30)
        lon_exp <- 1:5
        lat_exp <- 1:4
        obs <- rnorm(27000)
        dim(obs) <- c(lat = 12, lon = 15, sdate = 5, time  = 30)
        lon_obs <- seq(0,6, 6/14)
        lat_obs <- seq(0,6, 6/11)
        exp <- s2dv_cube(data = exp, lat = lat_exp, lon = lon_exp)
        obs <- s2dv_cube(data = obs, lat = lat_obs, lon = lon_obs)
        attr(exp$lon,"first_lon") <- 1
        attr(exp$lon,"last_lon") <- 5
        attr(exp$lat,"first_lat") <- 1
        attr(exp$lat,"last_lat") <- 4
        attr(obs$lon,"first_lon") <- 0
        attr(obs$lon,"last_lon") <- 6
        attr(obs$lat,"first_lat") <- 0
        attr(obs$lat,"last_lat") <- 6
        downscaled_field_cstools <- CST_Analogs(exp = exp, obs = obs, grid_exp = 'r360x180') # Analogs to CST_Analogs

        downscaled_field_analogs <- Analogs(exp = exp$data, obs = obs$data, exp_lons = lon_exp, exp_lats = lat_exp, obs_lons = lon_obs, obs_lats = lat_obs, grid_exp = 'r360x180')

        #> str(downscaled_field_cstools)
        #List of 8
        #$ data        : num [1:3, 1:12, 1:15, 1:30, 1:5, 1:5] 2.301 2.301 2.301 -0.936 -0.936 ...
        #$ lon         : num [1:15] 0 0.429 0.857 1.286 1.714 ...
        #..- attr(*, "first_lon")= num 0
        #..- attr(*, "last_lon")= num 6
        #$ lat         : num [1:12] 0 0.545 1.091 1.636 2.182 ...
        #..- attr(*, "first_lat")= num 0
        #..- attr(*, "last_lat")= num 6
        #$ Variable    : NULL
        #$ Datasets    : NULL
        #$ Dates       : NULL
        #$ when        : NULL
        #$ source_files: NULL
        #- attr(*, "class")= chr "s2dv_cube"
        #> dim(downscaled_field_analogs)
        #NULL
        #> str(downscaled_field_analogs)
        #List of 3
        #$ data: num [1:3, 1:12, 1:15, 1:30, 1:5, 1:5] 2.301 2.301 2.301 -0.936 -0.936 ...
        #$ lon : num [1:15] 0 0.429 0.857 1.286 1.714 ...
        #$ lat : num [1:12] 0 0.545 1.091 1.636 2.182 ...
        #> dim(downscaled_field_cstools$data - downscaled_field_analogs$data)
        #analogs     lat     lon    time   sdate  member
        #    3      12      15      30       5       5
        #> max(downscaled_field_cstools$data - downscaled_field_analogs$data)
        #[1] 0
        #> min(downscaled_field_cstools$data - downscaled_field_analogs$data)
        #[1] 0


    #dim(downscaled_field[[1]])
    #lat    lon   time  sdate member # old version
    #    12     15     30      5      5
    #analogs     lat     lon    time   sdate  member # new version
    #  3      12      15      30       5       5

    downscaled_ensemble_mean <- MeanDims(downscaled_field[[1]], 'member')
    original_ensemble_mean <- Reorder(MeanDims(exp[[1]], 'member'), names(dim(downscaled_ensemble_mean))[-1]) # [-1] because first dim of downscaled field is "analogs" (without this [-1]: ERROR Parameter 'order' do not match the dimension names of parameter 'data' )

    PlotEquiMap(downscaled_ensemble_mean[1,,,1,1], lat=lat_obs, lon=lon_obs, toptitle = 'test downscaled ensemble mean time1 sdate1' , file=paste0(output_dir, 'test02_downscaled.png'))
    PlotEquiMap(original_ensemble_mean[,,1,1], lat=lat_exp, lon=lon_exp, toptitle = 'test original ensemble mean time1 sdate1' , file=paste0(output_dir, 'test02_original.png'))

    save(exp, lon_exp, lat_exp, obs, lon_obs, lat_obs, downscaled_field_cstools, downscaled_field_analogs, file = paste0(output_dir, 'dummy_data.RData'))

# TEST different lat/lon dimension names (with dummy data):
    # libaries and functions:
        library(s2dv)
        library(CSTools)
        library(FNN)
        library(multiApply)
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')
        output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'
    # prepare data:
        load(paste0(output_dir, 'dummy_data.RData'))
        names(dim(exp$data))[which(names(dim(exp$data))=='lat')] <- 'latitude_test'
        names(dim(exp$data))[which(names(dim(exp$data))=='lon')] <- 'longitude_test'
        names(dim(obs$data))[which(names(dim(obs$data))=='lat')] <- 'latitude_test'
        names(dim(obs$data))[which(names(dim(obs$data))=='lon')] <- 'longitude_test'
    # call functions:
        downscaled_field_cstools <- CST_Analogs(exp = exp, obs = obs, grid_exp = 'r360x180', lon_dim = 'longitude_test', lat_dim = 'latitude_test') # Analogs to CST_Analogs
        # ERROR: Error in if ((obs_lats[1] > exp_lats[1]) | (obs_lats[length(obs_lats)] <  : argument is of length zero
        downscaled_field_analogs <- Analogs(exp = exp$data, exp_lons = lon_exp, exp_lats = lat_exp, 
                                            obs = obs$data, obs_lons = lon_obs, obs_lats = lat_obs, grid_exp = 'r360x180',
                                            lon_dim = 'longitude_test', lat_dim = 'latitude_test')
        # ERROR: Error in s2dv::CDORemap(data_array = exp, lats = lats, lons = lons, grid = target_grid,  : 
                                #Could not find a known longitude dimension name in the provided 'data_array'.
    


# TEST REAL DATA (daily)

    library(s2dv)
    library(CSTools)
    library(FNN)
    library(multiApply)
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')
    output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'

    library(ClimProjDiags) # to use ArrayToList facilitating the use of PlotLayout
    library(abind) #to join variables read from different files

    # prepared data (done once and then saved as RData):
        # code_directory <- '/home/Earth/allabres/Documents/gitlab_projects/alba/Postprocessing_Workflow/'
        
        # # Configuration:
        # var <- 'tas'
        # year_in <- 2010
        # year_fi <- 2014
        # fcst_year <- 0 # 0 if we are only interested in the hindcast; otherwise: year that we wish to calibrate
        # mm <- '05' # start date month
        # leadtime <- 0 # (forecast date = start date + leadtime; leadtime = 0 -> same forecast date as start date)
        # region <- 'user_defined' #options: 'user_defined' (modify lat/lon min/max in module_conf.R), 'LAC' (Latin America and Caribbean), 'A' (Argentina), 'B' (Brazil), 'C' (Caribbean)
      
        # # forecast and reference systems:
        # ref_system <- 'ERA5-Land'
        # fcst_system <- 'ECMWF-s5'
      
        # aggregation <- 'daily' # 'seasonal', 'monthly', 'daily'
        # interpolate_observations <- FALSE #interpolate at load with startR
        # load_selection <- 'both' #(could be 'exp', 'obs' or 'both')
        # circularlons <- TRUE
      
        # source(paste0(code_directory, 'module_conf.R'))
        # source(paste0(code_directory, 'module_load.R'))
    
        # save(exp, lons.hcst, lats.hcst, obs, lons.ref, lats.ref, lons.min, lons.max, lats.min, lats.max, 
        #       fcst_path, ref_path, var, 
        #    file = paste0(output_dir, 'daily_data_to_test.RData'))
    load(paste0(output_dir, 'daily_data_to_test.RData'))

    # Data configuration for Analogs.R:
        # Obs has NA (the function should work ok) -> YES, it does
        # > length(obs[which(is.na(obs))])
        # [1] 655
        
        # preparation of data as in test1 (except NAs)----------------------------------------------------------------
        names(dim(exp)) <- replace(names(dim(exp)), which(names(dim(exp)) == 'latitude'), 'lat')
        names(dim(exp)) <- replace(names(dim(exp)), which(names(dim(exp)) == 'longitude'), 'lon')
        names(dim(obs)) <- replace(names(dim(obs)), which(names(dim(obs)) == 'latitude'), 'lat')
        names(dim(obs)) <- replace(names(dim(obs)), which(names(dim(obs)) == 'longitude'), 'lon')
        names(dim(lats.hcst)) <- 'lat'
        names(dim(lats.ref)) <- 'lat'
        names(dim(lons.hcst)) <- 'lon'
        names(dim(lons.ref)) <- 'lon'
        exp_test <- s2dv_cube(data = exp, lat = lats.hcst, lon = lons.hcst, Variable = list(varName = var, level = NULL), when = Sys.time(), source_files = fcst_path)
        obs_test <- s2dv_cube(data = obs, lat = lats.ref, lon = lons.ref, Variable = list(varName = var, level = NULL), when = Sys.time(), source_files = ref_path)
        attr(exp_test$lon,"first_lon") <- lons.min
        attr(exp_test$lon,"last_lon") <- lons.max
        attr(exp_test$lat,"first_lat") <- lats.min
        attr(exp_test$lat,"last_lat") <- lats.max
        attr(obs_test$lon,"first_lon") <- lons.min
        attr(obs_test$lon,"last_lon") <- lons.max
        attr(obs_test$lat,"first_lat") <- lats.min
        attr(obs_test$lat,"last_lat") <- lats.max

    # csdownscale:
        grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'
        downscaled_field_cstools <- CST_Analogs(exp = exp_test, obs = obs_test, grid_exp = grid_nc)
        downscaled_field_analogs <- Analogs(exp = exp_test$data, exp_lons = exp_test$lon, exp_lats = exp_test$lat, 
                                            obs = obs_test$data, obs_lons = obs_test$lon, obs_lats = obs_test$lat, 
                                            grid_exp = grid_nc)
        #> min(downscaled_field_cstools$data - downscaled_field_analogs$data, na.rm=T)
        #[1] 0
        #> max(downscaled_field_cstools$data - downscaled_field_analogs$data, na.rm=T)
        #[1] 0
  
    # plot:
        downscaled_data_to_plot <- MeanDims(MeanDims(downscaled_field_cstools$data, 'ensemble'), 'sdate')
        PlotEquiMap(downscaled_data_to_plot[1,,,1,1,1], #first analog
                    lon = downscaled_field_cstools$lon,
                    lat = downscaled_field_cstools$lat,
                    filled.continents = FALSE,
                    file = paste0(output_dir, 'plot_mean_downscaled_daily_data.png'))
        PlotEquiMap(MeanDims(MeanDims(exp_test$data, 'ensemble'), 'sdate'),
                    lon = exp_test$lon,
                    lat = exp_test$lat,
                    filled.continents = FALSE,
                    file = paste0(output_dir, 'plot_mean_exp_test.png'))
        PlotEquiMap(MeanDims(obs_test$data, 'sdate'),
                    lon = obs_test$lon,
                    lat = obs_test$lat,
                    filled.continents = FALSE,
                    file = paste0(output_dir, 'plot_mean_obs_test.png'))

    #--------------------------
    load('interpolation_test.RData')
    PlotEquiMap(MeanDims(MeanDims(downscaled_field$data, 'ensemble'), 'sdate'),
        lon = downscaled_field$lon,
        lat = downscaled_field$lat,
        filled.continents = FALSE,
        file = paste0(output_dir, 'plot_mean_test1_with_NA_to_zero.png'))
        

# TEST REAL DATA (daily) with ARRAYS:

    library(s2dv)
    library(CSTools)
    library(FNN)
    library(multiApply)
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')
    output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'

    library(ClimProjDiags) # to use ArrayToList facilitating the use of PlotLayout
    library(abind) #to join variables read from different files

    load(paste0(output_dir, 'daily_data_to_test.RData'))

    grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'
    
    downscaled_field_analogs_latlon <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)

    # OK

    names(dim(exp))[which(names(dim(exp))=='longitude')] <- 'lon'
    names(dim(exp))[which(names(dim(exp))=='latitude')] <- 'lat'
    names(dim(obs))[which(names(dim(obs))=='longitude')] <- 'lon'
    names(dim(obs))[which(names(dim(obs))=='latitude')] <- 'lat'
    downscaled_field_analogs <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            grid_exp = grid_nc)
    #Error in s2dv::CDORemap(data_array = exp, lats = lats, lons = lons, grid = target_grid,  :
    #    Parameter 'lon' must have the same longitude dimension name as the 'data_array'.
    
    # ------------------- not the solution
    #    attr(lons.hcst,"variables")$longitude$dim[[1]]$name <- 'lon'
    #    attr(lats.hcst,"variables")$latitude$dim[[1]]$name <- 'lat'
    #    attr(lons.ref,"variables")$longitude$dim[[1]]$name # it's already "lon" ¿?
    #    attr(lats.ref,"variables")$latitude$dim[[1]]$name # it's already "lat" ¿?
    #------------------- not the solution

    names(attr(lons.hcst, "variables")) <- 'lon'
    names(attr(lats.hcst, "variables")) <- 'lat'
    names(attr(lons.ref, "variables")) <- 'lon'
    names(attr(lats.hcst, "variables")) <- 'lat'
    downscaled_field_analogs <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            grid_exp = grid_nc)
    #Error in s2dv::CDORemap(data_array = exp, lats = lats, lons = lons, grid = target_grid,  :
    #    Parameter 'lon' must have the same longitude dimension name as the 'data_array'.


# TEST REAL DATA (daily) LON/LAT DIMNAMES:

    library(s2dv)
    library(CSTools)
    library(FNN)
    library(multiApply)
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')
    output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'

    library(ClimProjDiags) # to use ArrayToList facilitating the use of PlotLayout
    library(abind) #to join variables read from different files

    load(paste0(output_dir, 'daily_data_to_test.RData'))
    grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'
    
    exp <- s2dv_cube(data = exp, lat = lats.hcst, lon = lons.hcst)
    obs <- s2dv_cube(data = obs, lat = lats.ref, lon = lons.ref)
    
    downscaled_field_cstools <- CST_Analogs(exp = exp, obs = obs, grid_exp = grid_nc, 
                                            lon_dim = 'longitude', lat_dim = 'latitude')
     
    # ERROR 
    # because my internal s2dv_cube has "lon", "lat" but names(dim()) is "latitude" "longitude"; 
    # so exp[['lon']] ok but exp[['longitude']] is NULL even if the dimname (and therefore lon_dim) is 'longitude'


# TEST REAL DATA (daily) NA:

    library(s2dv)
    library(CSTools)
    library(FNN)
    library(multiApply)
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')
    source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')
    output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'
    load(paste0(output_dir, 'daily_data_to_test.RData'))
    grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'

    # introducing some NA to the observations:
        obs_na <- obs
        obs_na[1,1,,1,80:110, 100:170] <- NA
        #> length(obs_na[which(is.na(obs_na))])
        #[1] 1810
    # introducing some NA to the experiment:
        exp_na <- exp
        exp_na[1,1,,,1,5:20, 10:25] <- NA
    # downscaling:
        downscaled_field_original_data <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)
        downscaled_field_obs_na <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs_na, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)
        downscaled_field_exp_na <- Analogs(exp = exp_na, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)
        downscaled_field_both_na <- Analogs(exp = exp_na, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs_na, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)

    # plot one time step
        library(abind)
        library(ClimProjDiags)

        # prepare data of one time step and ensemble:
            plot_longitudes <- downscaled_field_original_data$lon
            plot_latitudes <- downscaled_field_original_data$lat
            plot_original_data <- drop(downscaled_field_original_data$data)[,,,1,1]
            plot_obs_na <- drop(downscaled_field_obs_na$data)[,,,1,1]
            plot_exp_na <- drop(downscaled_field_exp_na$data)[,,,1,1]
            plot_both_na <- drop(downscaled_field_both_na$data)[,,,1,1]
            data_to_plot <- abind(plot_original_data[1,,], plot_obs_na[1,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_exp_na[1,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_both_na[1,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_original_data[2,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_obs_na[2,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_exp_na[2,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_both_na[2,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_original_data[3,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_obs_na[3,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_exp_na[3,,], along = 3)
            data_to_plot <- abind(data_to_plot, plot_both_na[3,,], along = 3)

        # plot    
            PlotLayout(fun = PlotEquiMap, # plot layout plots from left to right first and then top to bottom
                    plot_dims = c('lon', 'lat'),
                    var = ArrayToList(data_to_plot, 3, names = ''),
                    lon = plot_longitudes,
                    lat = plot_latitudes,
                    colNA = 'black',
                    #brks = seq(-1, 1, 0.1),
                    #bar_scale = 0.5,
                    #triangle_ends_scale = 1,
                    #triangle_ends = c(TRUE,FALSE),
                    filled.continents = FALSE, 
                    toptitle = 'downscaled field (first time and ensemble)',
                    title_scale = 0.7,
                    nrow = 3,
                    row_titles = c('analog 1', 'analog 2', 'analog 3'),
                    ncol = 4,
                    col_titles = c('original data', 'obs with NA', 'exp with NA', 'both with NA'),
                    fileout = paste0(output_dir, 'na_test.png')
            )

# TEST REAL DATA (daily) compare the 3 analogs which are obtained as output:
    # with the data in the test before they are identical:
    #> max(abs(plot_original_data[1,,] - plot_original_data[2,,]), na.rm=T)
    #[1] 0
    #> max(abs(plot_original_data[1,,] - plot_original_data[3,,]), na.rm=T)
    #[1] 0

    # Analogs libraries and functions:
        library(s2dv)
        library(CSTools)
        library(FNN)
        library(multiApply)
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')
        #source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')
        source('/esarchive/scratch/jramon/GitLab_jramon/downscaling/csdownscale/R/Analogs.R') 
        output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'

        library(ClimProjDiags) # to use ArrayToList facilitating the use of PlotLayout
        library(abind) #to join variables read from different files

    #new test data:
        #code_directory <- '/home/Earth/allabres/Documents/gitlab_projects/alba/Postprocessing_Workflow/'
        #    
        ## Configuration:
        #var <- 'tas'
        #year_in <- 1993
        #year_fi <- 2016
        #fcst_year <- 0 # 0 if we are only interested in the hindcast; otherwise: year that we wish to calibrate
        #mm <- '05' # start date month
        #leadtime <- 0 # (forecast date = start date + leadtime; leadtime = 0 -> same forecast date as start date)
        #region <- 'user_defined' #options: 'user_defined' (modify lat/lon min/max in module_conf.R), 'LAC' (Latin America and Caribbean), 'A' (Argentina), 'B' (Brazil), 'C' (Caribbean)
        #
        ## forecast and reference systems:
        #ref_system <- 'ERA5-Land' # ERA5
        #fcst_system <- 'ECMWF-s5'
        #
        #aggregation <- 'daily' # 'seasonal', 'monthly', 'daily'
        #interpolate_observations <- FALSE #interpolate at load with startR
        #load_selection <- 'both' #(could be 'exp', 'obs' or 'both')
        #circularlons <- TRUE
        #
        #source(paste0(code_directory, 'module_conf.R'))
        #source(paste0(code_directory, 'module_load.R'))
        #
        #save(exp, lons.hcst, lats.hcst, obs, lons.ref, lats.ref, lons.min, lons.max, lats.min, lats.max, 
        #       fcst_path, ref_path, var, 
        #    file = paste0(output_dir, 'daily_data_to_test_1993-2016_era5land.RData'))
        
    # downscaled field:
        load(paste0(output_dir, 'daily_data_to_test_1993-2016_era5.RData'))
        grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'
        downscaled_field_1993_2016 <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)
        #Error in Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst,  :
        #   There are not enough data in 'obs'. Please to add more latitudes or longitudes.

        load(paste0(output_dir, 'daily_data_to_test_1993-2016_era5land.RData'))
        grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'
        downscaled_field_1993_2016 <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)
        save(downscaled_field_1993_2016, file = paste0(output_dir, 'downscaled_field_1993_2016-ecmwf_to_era5land.RData'))

    # plot:
        load(paste0(output_dir, 'downscaled_field_1993_2016-ecmwf_to_era5land.RData'))
        plot_longitudes <- downscaled_field_1993_2016$lon
        plot_latitudes <- downscaled_field_1993_2016$lat
        plot_data <- abind(drop(downscaled_field_1993_2016$data)[1,,,1,1], drop(downscaled_field_1993_2016$data)[2,,,1,1], along = 3)
        plot_data <- abind(plot_data, drop(downscaled_field_1993_2016$data)[3,,,1,1], along = 3)
        PlotLayout(fun = PlotEquiMap, # plot layout plots from left to right first and then top to bottom
                plot_dims = c('lon', 'lat'),
                var = ArrayToList(plot_data, 3, names = ''),
                lon = plot_longitudes,
                lat = plot_latitudes,
                colNA = 'black',
                #brks = seq(-1, 1, 0.1),
                #bar_scale = 0.5,
                #triangle_ends_scale = 1,
                #triangle_ends = c(TRUE,FALSE),
                filled.continents = FALSE, 
                toptitle = 'downscaled field (first time and ensemble)',
                title_scale = 0.7,
                nrow = 1,
                ncol = 3,
                col_titles = c('analog 1', 'analog 2', 'analog 3'),
                fileout = paste0(output_dir, 'analogs_comparison_1-2-3_test.png')
        )
    
    # comparison:
        max(abs(drop(downscaled_field_1993_2016$data)[1,,,1,1] - drop(downscaled_field_1993_2016$data)[2,,,1,1]), na.rm = TRUE)
        #[1] 0
        max(abs(drop(downscaled_field_1993_2016$data)[1,,,1,1] - drop(downscaled_field_1993_2016$data)[3,,,1,1]), na.rm = TRUE)
        #[1] 0

# TEST REAL DATA (daily) compare the 3 analogs which are obtained as output LARGER REGION:
    # with the data in the test before they are identical:
    #> max(abs(plot_original_data[1,,] - plot_original_data[2,,]), na.rm=T)
    #[1] 0
    #> max(abs(plot_original_data[1,,] - plot_original_data[3,,]), na.rm=T)
    #[1] 0

    # Analogs libraries and functions:
        library(s2dv)
        library(CSTools)
        library(FNN)
        library(multiApply)
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Utils.R')
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Interpolation.R')
        source('/esarchive/scratch/allabres/gitlab_clones/csdownscale/R/Analogs.R')
        output_dir <- '/esarchive/scratch/allabres/review_csdownscale/'

        library(ClimProjDiags) # to use ArrayToList facilitating the use of PlotLayout
        library(abind) #to join variables read from different files

    #new test data:
        code_directory <- '/home/Earth/allabres/Documents/gitlab_projects/alba/Postprocessing_Workflow/'
            
        # Configuration:
        var <- 'tas'
        year_in <- 1993
        year_fi <- 2016
        fcst_year <- 0 # 0 if we are only interested in the hindcast; otherwise: year that we wish to calibrate
        mm <- '05' # start date month
        leadtime <- 0 # (forecast date = start date + leadtime; leadtime = 0 -> same forecast date as start date)
        region <- 'LAC' #options: 'user_defined' (modify lat/lon min/max in module_conf.R), 'LAC' (Latin America and Caribbean), 'A' (Argentina), 'B' (Brazil), 'C' (Caribbean)
        
        # forecast and reference systems:
        ref_system <- 'ERA5-Land' # ERA5
        fcst_system <- 'ECMWF-s5'
        
        aggregation <- 'daily' # 'seasonal', 'monthly', 'daily'
        interpolate_observations <- FALSE #interpolate at load with startR
        load_selection <- 'both' #(could be 'exp', 'obs' or 'both')
        circularlons <- TRUE
        
        source(paste0(code_directory, 'module_conf.R'))
        source(paste0(code_directory, 'module_load.R'))
        
    # downscaled field:
        grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'
        downscaled_field_1993_2016 <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)
        #Error in Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst,  :
        #   There are not enough data in 'obs'. Please to add more latitudes or longitudes.
        #> dim(exp)
        #dat       var     sdate  ensemble      time  latitude longitude
        #    1         1        24        25         1        91        71
        #> dim(obs)
        #dat       var     sdate      time  latitude longitude
        #    1         1        24         1       321       249

        load(paste0(output_dir, 'daily_data_to_test_1993-2016_era5land.RData'))
        grid_nc <- '/esarchive/exp/ecmwf/system5c3s/daily_mean/tas_f6h/tas_20160501.nc'
        downscaled_field_1993_2016 <- Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst, 
                                            obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, 
                                            lon_dim = 'longitude', lat_dim = 'latitude',
                                            grid_exp = grid_nc)
        save(downscaled_field_1993_2016, file = paste0(output_dir, 'downscaled_field_1993_2016-ecmwf_to_era5land_lac_region.RData'))


       # *** caught bus error ***
       # address 0xb2f2acb, cause 'non-existent physical address'
       # *** recursive gc invocation
       # *** recursive gc invocation
       # ...
       # *** recursive gc invocation
       # *** recursive gc invocation
       # 
       # Traceback:
       # 1: Apply(list(obs, window), target_dims = "window", fun = function(x,     y) x <- y)
       # 2: Analogs(exp = exp, exp_lons = lons.hcst, exp_lats = lats.hcst,     obs = obs, obs_lons = lons.ref, obs_lats = lats.ref, lon_dim = "longitude",     lat_dim = "latitude", grid_exp = grid_nc)
       # 
       # Possible actions:
       # 1: abort (with core dump, if enabled)
       # 2: normal R exit
       # 3: exit R without saving workspace
       # 4: exit R saving workspace
       # Selection: 1
       # *** recursive gc invocation
       # *** recursive gc invocation
       # ... 
       # Bus error (core dumped)
       
    #####################################################################################################
    # plot:
        load(paste0(output_dir, 'downscaled_field_1993_2016-ecmwf_to_era5land_lac_region.RData'))
        plot_longitudes <- downscaled_field_1993_2016$lon
        plot_latitudes <- downscaled_field_1993_2016$lat
        plot_data <- abind(drop(downscaled_field_1993_2016$data)[1,,,1,1], drop(downscaled_field_1993_2016$data)[2,,,1,1], along = 3)
        plot_data <- abind(plot_data, drop(downscaled_field_1993_2016$data)[3,,,1,1], along = 3)
        PlotLayout(fun = PlotEquiMap, # plot layout plots from left to right first and then top to bottom
                plot_dims = c('lon', 'lat'),
                var = ArrayToList(plot_data, 3, names = ''),
                lon = plot_longitudes,
                lat = plot_latitudes,
                colNA = 'black',
                #brks = seq(-1, 1, 0.1),
                #bar_scale = 0.5,
                #triangle_ends_scale = 1,
                #triangle_ends = c(TRUE,FALSE),
                filled.continents = FALSE, 
                toptitle = 'downscaled field (first time and ensemble)',
                title_scale = 0.7,
                nrow = 1,
                ncol = 3,
                col_titles = c('analog 1', 'analog 2', 'analog 3'),
                fileout = paste0(output_dir, 'analogs_comparison_1-2-3_test_LAC.png')
        )
    
    # comparison:
        max(abs(drop(downscaled_field_1993_2016$data)[1,,,1,1] - drop(downscaled_field_1993_2016$data)[2,,,1,1]), na.rm = TRUE)
        #[1] 
        max(abs(drop(downscaled_field_1993_2016$data)[1,,,1,1] - drop(downscaled_field_1993_2016$data)[3,,,1,1]), na.rm = TRUE)
        #[1] 


