###############################################################################
##################### SCORECARDS MODULE FOR SUNSET SUITE ######################
###############################################################################

##### Load source functions #####
source('modules/Scorecards/R/tmp/LoadMetrics.R')
source('modules/Scorecards/R/tmp/WeightedMetrics.R')
source('modules/Scorecards/R/tmp/Utils.R')
source('modules/Scorecards/R/tmp/SCTransform.R')
source('modules/Scorecards/R/tmp/ScorecardsSingle.R')
source('modules/Scorecards/R/tmp/ScorecardsMulti.R')
source('modules/Scorecards/R/tmp/ScorecardsSystemDiff.R')
source('modules/Scorecards/R/tmp/SCPlotScorecard.R') 


## TODO: Change function name to 'Scorecards'?
## Define function
Scorecards <- function(recipe) {
  
  ## Parameters for loading data
  input.path <- "/esarchive/scratch/nmilders/scorecards_data/test/recipe_scorecards_data_loading_nadia_20231219150442/outputs/" #temp
  skill.input.path <- paste0(input.path, "Skill/") #paste0(recipe$Run$output_dir, "/outputs/Skill/")
  stats.input.path <- paste0(input.path, "Statistics/") #paste0(recipe$Run$output_dir, "/outputs/Statistics/")
  output.path <- paste0(recipe$Run$output_dir, "/plots/Scorecards/")
  dir.create(output.path, recursive = T, showWarnings = F)
  system <- recipe$Analysis$Datasets$System$name
  reference <- recipe$Analysis$Datasets$Reference$name
  var <- recipe$Analysis$Variables$name
  start.year <- as.numeric(recipe$Analysis$Time$hcst_start)
  end.year <- as.numeric(recipe$Analysis$Time$hcst_end)
  forecast.months <- recipe$Analysis$Time$ftime_min : recipe$Analysis$Time$ftime_max
  calib.method <- tolower(recipe$Analysis$Workflow$Calibration$method)
  
  # NOTE (Eva): This condition needs to be checked, in my case 
  #             (recipe$Analysis$Workflow$Scorecards$start_months = NULL)
  start.months <- 1:12 # I added this line
  # Needs to be corrected:
  # if (recipe$Analysis$Workflow$Scorecards$start_months == 'all') {
  #   start.months <- 1:12
  # } else {
  #   start.months <- as.numeric(strsplit(recipe$Analysis$Workflow$Scorecards$start_months,
  #                                       split = ", | |,")[[1]])
  # }
  
  regions <- recipe$Analysis$Workflow$Scorecards$regions  
  for (i in names(regions)){regions[[i]] <- unlist(regions[[i]])}
  
  metric.aggregation <- recipe$Analysis$Workflow$Scorecards$metric_aggregation
  metrics.load <- unlist(strsplit(tolower(recipe$Analysis$Workflow$Skill$metric), ", | |,"))
  metrics.visualize <- unlist(strsplit(tolower(recipe$Analysis$Workflow$Scorecards$metric), ", | |,"))
  na.rm <- FALSE ## Need to define in recipe ?? 
  ncores <- 1 # recipe$Analysis$ncores

  
  ## Parameters for scorecard table
  inf.to.na <-  recipe$Analysis$Workflow$Scorecards$inf_to_na
  table.label <-  recipe$Analysis$Workflow$Scorecards$table_label
  fileout.label <- recipe$Analysis$Workflow$Scorecards$fileout_label
  legend.white.space <- recipe$Analysis$Workflow$Scorecards$legend_white_space 
  col1.width <- recipe$Analysis$Workflow$Scorecards$col1_width
  col2.width <- recipe$Analysis$Workflow$Scorecards$col2_width
  calculate.diff <- recipe$Analysis$Workflow$Scorecards$calculate_diff


  
  ## Load data files
  start.months <- sprintf("%02d", start.months)
  period <- paste0(start.year, "-", end.year)

  ####### SKILL AGGREGATION #######
  if(metric.aggregation == 'skill'){
  
    ## Load data files
    loaded_metrics <- LoadMetrics(system = system,
                                  reference = reference,
                                  var = var,
                                  start.year = start.year,
                                  end.year = end.year,
                                  metrics =  metrics.load,
                                  start.months = start.months,
                                  forecast.months = forecast.months,
                                  inf.to.na = inf.to.na,
                                  input.path = skill.input.path)
    
    ## Spatial Aggregation of metrics
    if('region' %in% names(dim(loaded_metrics[[1]][[1]]))){
      
      ### Convert loaded metrics to array for already aggregated data
      metrics.dim <- attributes(loaded_metrics[[1]][[1]])$metrics 
      forecast.months.dim <- attributes(loaded_metrics[[1]][[1]])$forecast.months 
      start.months.dim <- attributes(loaded_metrics[[1]][[1]])$start.months 
      regions.dim <- regions #list('NAO' = c(lon.min = -80, lon.max = 40, lat.min = 20, lat.max = 80))
      
      aggregated_metrics <- array(dim = c(system = length(loaded_metrics),
                                          reference = length(loaded_metrics[[1]]),
                                          metric = length(metrics.dim), 
                                          time = length(forecast.months.dim), 
                                          sdate = length(start.months.dim), 
                                          region = length(regions.dim)))
      
      
      for (sys in 1:length(names(loaded_metrics))){
        for (ref in 1:length(names(loaded_metrics[[sys]]))){
          aggregated_metrics[sys, ref, , , , ] <- s2dv::Reorder(data = loaded_metrics[[sys]][[ref]], order = c('metric','time','sdate','region'))
        }
      }
      
      ## Add attributes
      attributes(aggregated_metrics)$metrics <- metrics.load
      attributes(aggregated_metrics)$start.months <- attributes(loaded_metrics[[1]][[1]])$start.months
      attributes(aggregated_metrics)$forecast.months <- attributes(loaded_metrics[[1]][[1]])$forecast.months
      attributes(aggregated_metrics)$regions <- regions
      attributes(aggregated_metrics)$system.name <- names(loaded_metrics)
      attributes(aggregated_metrics)$reference.name <- names(loaded_metrics[[1]])
      
      
    } else {
      ## Calculate weighted mean of spatial aggregation
      aggregated_metrics <- WeightedMetrics(loaded_metrics,
                                            regions = regions,
                                            metric.aggregation = metric.aggregation,
                                            ncores = ncores)
    }## close if on region
  } 
  
  ###### SCORE AGGREGATION ######
  if(metric.aggregation == 'score'){
    
    ## Comments (Nadia): 
    ## 1) syear parameter as FALSE instead of NULL for metrics without syear dimension?
    ## 2) output files names of statistics not correct, too many _ in name
    ## 3) how to load multiple systems? - Loop over sys and ref for metric.aggregation == 'score'
    ## Need to test with multiple sdates
    
    lon_dim <- 'longitude'
    lat_dim <- 'latitude'
    time_dim <- 'syear'
    memb_dim <- 'ensemble'

    ## Define arrays to filled with data
    scorecard_metrics <- array(data = NA, 
                               dim = c(system = length(system),
                                       reference = length(reference),
                                       time = length(forecast.months), 
                                       sdate = length(start.months), 
                                       region = length(regions),
                                       metric = length(metrics.visualize)))
    
    scorecard_sign <- array(data = NA, 
                            dim = c(system = length(system),
                                    reference = length(reference),
                                    time = length(forecast.months), 
                                    sdate = length(start.months), 
                                    region = length(regions),
                                    metric = length(metrics.visualize)))
    
    
    for (sys in 1:length(system)){
      # sys_num <- which(system == sys)
      for (ref in 1:length(reference)){
        # ref_num <- which(refence == ref)
        for (met in metrics.visualize) {   
          
          if(met == 'rpss'){
            ## Load data from saved files
            rps_syear <- .loadmetrics(input_path = skill.input.path, system = system[sys],
                                      reference = reference[ref], var = var, 
                                      period = period, start_months = start.months,
                                      forecast_months = forecast.months, 
                                      metrics = 'rps_syear', 
                                      calib_method = calib.method, syear = TRUE)
            
            rps_clim_syear <- .loadmetrics(input_path = skill.input.path, system = system[sys],
                                            reference = reference[ref], var = var, 
                                            period = period, start_months = start.months,
                                            forecast_months = forecast.months, 
                                            metrics = 'rps_clim_syear', 
                                            calib_method = calib.method, syear = TRUE)

            ## Remove dat and var dimensions
            rps_syear <- Subset(rps_syear, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
            rps_clim_syear <- Subset(rps_clim_syear, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
            
            ## Spatially aggregate data
            rps_syear_spatial_aggr <- sapply(X = 1:length(regions), 
                                             FUN = function(X) {
                                               WeightedMean(data = rps_syear,
                                                            region = regions[[X]],
                                                            lon = as.vector(attributes(rps_syear)$Variables$dat1$longitude),
                                                            lat = as.vector(attributes(rps_syear)$Variables$dat1$latitude),
                                                            londim = lon_dim,
                                                            latdim = lat_dim,
                                                            na.rm = F)
                                             }, simplify = 'array')
            
            rps_clim_syear_spatial_aggr <- sapply(X = 1:length(regions), 
                                             FUN = function(X) {
                                               WeightedMean(data = rps_clim_syear,
                                                            region = regions[[X]],
                                                            lon = as.vector(attributes(rps_clim_syear)$Variables$dat1$longitude),
                                                            lat = as.vector(attributes(rps_clim_syear)$Variables$dat1$latitude),
                                                            londim = lon_dim,
                                                            latdim = lat_dim,
                                                            na.rm = F)
                                             }, simplify = 'array')
            
            ## Include name of region dimension
            names(dim(rps_syear_spatial_aggr))[length(dim(rps_syear_spatial_aggr))] <- 'region'
            names(dim(rps_clim_syear_spatial_aggr))[length(dim(rps_clim_syear_spatial_aggr))] <- 'region'
      
            ## Temporally aggregate data
            rps_temp_aggr <- Apply(data = rps_syear_spatial_aggr, 
                                   target_dims = time_dim, 
                                   fun = 'mean', ncores = ncores)$output1
            
            rps_clim_temp_aggr <- Apply(data = rps_clim_syear_spatial_aggr, 
                                        target_dims = time_dim,
                                        fun = 'mean',  ncores = ncores)$output1
            
            ## Calculate RPSS from aggregated RPS and RPS_clim
            rpss <- 1 - rps_temp_aggr / rps_clim_temp_aggr
            
            ## Calculate significance
            sign_rpss <- RandomWalkTest(rps_syear_spatial_aggr, rps_clim_syear_spatial_aggr, 
                                        time_dim = time_dim, test.type = 'two.sided',
                                        alpha = 0.05, pval = FALSE, sign = TRUE, 
                                        ncores = NULL)$sign
          
            ## Save metric result in arrays
            scorecard_metrics[sys, ref, , , ,which(metrics.visualize == met)] <- s2dv::Reorder(data = rpss, order = c('time', 'smonths','region'))
            scorecard_sign[sys, ref, , , , which(metrics.visualize == met)] <- s2dv::Reorder(data = sign_rpss, order = c('time', 'smonths','region'))
            
          } ## close if on rpss
        
          if(met == 'crpss'){
            
            ## Load data from saved files
            crps_syear <- .loadmetrics(input_path = skill.input.path, system = system[sys],
                                      reference = reference[ref], var = var, 
                                      period = period, start_months = start.months,
                                      forecast_months = forecast.months, 
                                      metrics = 'crps_syear', 
                                      calib_method = calib.method, syear = TRUE)
            
            crps_clim_syear <- .loadmetrics(input_path = skill.input.path, system = system[sys],
                                           reference = reference[ref], var = var, 
                                           period = period, start_months = start.months,
                                           forecast_months = forecast.months, 
                                           metrics = 'crps_clim_syear', 
                                           calib_method = calib.method, syear = TRUE)
            
            ## Remove dat and var dimensions
            crps_syear <- Subset(crps_syear, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
            crps_clim_syear <- Subset(crps_clim_syear, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
            
            ## Spatially aggregate data
            crps_syear_spatial_aggr <- sapply(X = 1:length(regions), 
                                             FUN = function(X) {
                                               WeightedMean(data = crps_syear,
                                                            region = regions[[X]],
                                                            lon = as.vector(attributes(crps_syear)$Variables$dat1$longitude),
                                                            lat = as.vector(attributes(crps_syear)$Variables$dat1$latitude),
                                                            londim = lon_dim,
                                                            latdim = lat_dim,
                                                            na.rm = na.rm)
                                             }, simplify = 'array')
            
            crps_clim_syear_spatial_aggr <- sapply(X = 1:length(regions), 
                                                  FUN = function(X) {
                                                    WeightedMean(data = crps_clim_syear,
                                                                 region = regions[[X]],
                                                                 lon = as.vector(attributes(crps_clim_syear)$Variables$dat1$longitude),
                                                                 lat = as.vector(attributes(crps_clim_syear)$Variables$dat1$latitude),
                                                                 londim = lon_dim,
                                                                 latdim = lat_dim,
                                                                 na.rm = na.rm)
                                                  }, simplify = 'array')
            
            ## Include name of region dimension
            names(dim(crps_syear_spatial_aggr))[length(dim(crps_syear_spatial_aggr))] <- 'region'
            names(dim(crps_clim_syear_spatial_aggr))[length(dim(crps_clim_syear_spatial_aggr))] <- 'region'
            
            ## Temporally aggregate data
            crps_temp_aggr <- Apply(data = crps_syear_spatial_aggr, 
                                   target_dims = time_dim, 
                                   fun = 'mean', ncores = ncores)$output1
            
            crps_clim_temp_aggr <- Apply(data = crps_clim_syear_spatial_aggr, 
                                        target_dims = time_dim,
                                        fun = 'mean', ncores = ncores)$output1
            
            ## Calculate CRPSS from aggregated CRPS and CRPS_clim
            crpss <- 1 - crps_temp_aggr / crps_clim_temp_aggr
            
            ## Calculate significance
            sign_crpss <- RandomWalkTest(crps_syear_spatial_aggr, crps_clim_syear_spatial_aggr, 
                                        time_dim = time_dim, test.type = 'two.sided',
                                        alpha = 0.05, pval = FALSE, sign = TRUE, 
                                        ncores = NULL)$sign
            
            ## Save metric result in arrays
            scorecard_metrics[sys, ref, , , , which(metrics.visualize == met)] <- s2dv::Reorder(data = crpss, order = c('time', 'smonths','region'))
            scorecard_sign[sys, ref, , , , which(metrics.visualize == met)] <- s2dv::Reorder(data = sign_crpss, order = c('time', 'smonths','region'))
            
          } ## close if on crpss
      
          if(met == 'enscorr'){
            ## Load data from saved files
            cov <- .loadmetrics(input_path = stats.input.path, system = system[sys],
                                      reference = reference[ref], var = var, 
                                      period = period, start_months = start.months,
                                      forecast_months = forecast.months, 
                                      metrics = 'cov', 
                                      calib_method = calib.method, syear = NULL)
            
            std_hcst <- .loadmetrics(input_path = stats.input.path, system = system[sys],
                                     reference = reference[ref], var = var, 
                                     period = period, start_months = start.months,
                                     forecast_months = forecast.months, 
                                     metrics = 'std_hcst', 
                                     calib_method = calib.method, syear = NULL)
            
            std_obs <- .loadmetrics(input_path = stats.input.path, system = system[sys],
                                     reference = reference[ref], var = var, 
                                     period = period, start_months = start.months,
                                     forecast_months = forecast.months, 
                                     metrics = 'std_obs', 
                                     calib_method = calib.method, syear = NULL)
          
            
            n_eff <- .loadmetrics(input_path = stats.input.path, system = system[sys],
                                  reference = reference[ref], var = var, 
                                  period = period, start_months = start.months,
                                  forecast_months = forecast.months, 
                                  metrics = 'n_eff', 
                                  calib_method = calib.method, syear = NULL)
            
            ## Calculate spatial aggregation
            cov_spatial_aggr <- sapply(X = 1:length(regions), 
                                              FUN = function(X) {
                                                WeightedMean(data = cov,
                                                             region = regions[[X]],
                                                             lon = as.vector(attributes(cov)$Variables$dat1$longitude),
                                                             lat = as.vector(attributes(cov)$Variables$dat1$latitude),
                                                             londim = lon_dim,
                                                             latdim = lat_dim,
                                                             na.rm = na.rm)
                                              }, simplify = 'array')
            
            ## Include name of region dimension
            names(dim(cov_spatial_aggr))[length(dim(cov_spatial_aggr))] <- 'region'
            
    
            std_hcst_spatial_aggr <- sapply(X = 1:length(regions), 
                                       FUN = function(X) {
                                         WeightedMean(data = std_hcst,
                                                      region = regions[[X]],
                                                      lon = as.vector(attributes(std_hcst)$Variables$dat1$longitude),
                                                      lat = as.vector(attributes(std_hcst)$Variables$dat1$latitude),
                                                      londim = lon_dim,
                                                      latdim = lat_dim,
                                                      na.rm = na.rm)
                                       }, simplify = 'array')
            
            names(dim(std_hcst_spatial_aggr))[length(dim(std_hcst_spatial_aggr))] <- 'region'
            
            std_obs_spatial_aggr <- sapply(X = 1:length(regions), 
                                            FUN = function(X) {
                                              WeightedMean(data = std_obs,
                                                           region = regions[[X]],
                                                           lon = as.vector(attributes(std_obs)$Variables$dat1$longitude),
                                                           lat = as.vector(attributes(std_obs)$Variables$dat1$latitude),
                                                           londim = lon_dim,
                                                           latdim = lat_dim,
                                                           na.rm = na.rm)
                                            }, simplify = 'array')
            
            names(dim(std_obs_spatial_aggr))[length(dim(std_obs_spatial_aggr))] <- 'region'
            
            n_eff_spatial_aggr <- sapply(X = 1:length(regions), 
                                           FUN = function(X) {
                                             WeightedMean(data = n_eff,
                                                          region = regions[[X]],
                                                          lon = as.vector(attributes(std_obs)$Variables$dat1$longitude),
                                                          lat = as.vector(attributes(std_obs)$Variables$dat1$latitude),
                                                          londim = lon_dim,
                                                          latdim = lat_dim,
                                                          na.rm = na.rm)
                                           }, simplify = 'array')
            
            names(dim(n_eff_spatial_aggr))[length(dim(n_eff_spatial_aggr))] <- 'region'
            n_eff_spatial_aggr <- Subset(n_eff_spatial_aggr, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
            
            ## Calculate correlation
            enscorr <- cov_spatial_aggr / (std_hcst_spatial_aggr * std_obs_spatial_aggr)
            
            ## Drop unwanted dimensions
            enscorr <- Subset(enscorr, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
    
            ## Calculate significance of corr
            alpha <- 0.05  # Apply 95% confidence
            
            t_alpha2_n2 <- qt(p = alpha/2, df = n_eff_spatial_aggr-2, lower.tail = FALSE)
            t <- abs(enscorr) * sqrt(n_eff_spatial_aggr-2) / sqrt(1-enscorr^2)
            
            sign_corr<- array(data = NA, 
                              dim = c(time = length(forecast.months), 
                                      smonths = length(start.months), 
                                      region = length(regions)))
            
            
            for (time in 1:dim(sign_corr)[['time']]){
              for (mon in 1:dim(sign_corr)[['smonths']]){
                for (reg in 1:dim(sign_corr)[['region']]){
            
                  if (anyNA(c(t[time, mon, reg], t_alpha2_n2[time, mon, reg])) == FALSE 
                      && t[time, mon, reg] >= t_alpha2_n2[time, mon, reg]){
                    sign_corr[time, mon, reg] <- TRUE
                  } else {
                    sign_corr[time, mon, reg] <- FALSE
                  }
            
                }
              }
            }  
    
            ## Save metric result in arrays
            scorecard_metrics[sys, ref, , , , which(metrics.visualize == met)] <- s2dv::Reorder(data = enscorr, order = c('time', 'smonths','region'))
            scorecard_sign[sys, ref, , , , which(metrics.visualize == met)] <- s2dv::Reorder(data = sign_corr, order = c('time', 'smonths','region'))
     
          } ## close if on enscorr
         
          if(met == 'mean_bias'){
            
            mean_bias <- .loadmetrics(input_path = skill.input.path, system = system[sys],
                                      reference = reference[ref], var = var, 
                                      period = period, start_months = start.months,
                                      forecast_months = forecast.months, 
                                      metrics = 'mean_bias', 
                                      calib_method = calib.method, syear = NULL)
            
            ## Calculate spatial aggregation
            mean_bias_spatial_aggr <- sapply(X = 1:length(regions), 
                                       FUN = function(X) {
                                         WeightedMean(data = mean_bias,
                                                      region = regions[[X]],
                                                      lon = as.vector(attributes(mean_bias)$Variables$dat1$longitude),
                                                      lat = as.vector(attributes(mean_bias)$Variables$dat1$latitude),
                                                      londim = lon_dim,
                                                      latdim = lat_dim,
                                                      na.rm = na.rm)
                                       }, simplify = 'array')
            
            names(dim(mean_bias_spatial_aggr))[length(dim(mean_bias_spatial_aggr))] <- 'region'
            
            ## Drop unwanted dimensions
            mean_bias_spatial_aggr <- Subset(mean_bias_spatial_aggr, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
            
            ## Save metric result in array
            scorecard_metrics[sys, ref, , , , which(metrics.visualize == met)] <- s2dv::Reorder(data = mean_bias_spatial_aggr, order = c('time', 'smonths','region'))
            
          } ## close on mean_bias
                
          if(met == 'enssprerr'){
            
            enssprerr <- .loadmetrics(input_path = skill.input.path, system = system[sys],
                                      reference = reference[ref], var = var, 
                                      period = period, start_months = start.months,
                                      forecast_months = forecast.months, 
                                      metrics = 'enssprerr', 
                                      calib_method = calib.method, syear = NULL)
            
            ## Calculate spatial aggregation
            enssprerr_spatial_aggr <- sapply(X = 1:length(regions), 
                                             FUN = function(X) {
                                               WeightedMean(data = enssprerr,
                                                            region = regions[[X]],
                                                            lon = as.vector(attributes(enssprerr)$Variables$dat1$longitude),
                                                            lat = as.vector(attributes(enssprerr)$Variables$dat1$latitude),
                                                            londim = lon_dim,
                                                            latdim = lat_dim,
                                                            na.rm = na.rm)
                                             }, simplify = 'array')
            
            names(dim(enssprerr_spatial_aggr))[length(dim(enssprerr_spatial_aggr))] <- 'region'
            
            ## Drop unwanted dimensions
            enssprerr_spatial_aggr <- Subset(enssprerr_spatial_aggr, along = c('dat', 'var'), indices = list(1,1) , drop = 'selected')
            
            ## Save metric result in array
            scorecard_metrics[sys, ref, , , , which(metrics.visualize == met)] <- s2dv::Reorder(data = enssprerr_spatial_aggr, order = c('time', 'smonths','region'))
            
          } ## close on enssprerr
        
        } ## close loop on metric
      } ## close if on reference
    } ## close if on system
        
     aggregated_metrics <- scorecard_metrics
    
  } ## close if on score   
  
  
  
  
  stop()
  

  ## Create simple scorecard tables 
  ## (one system only)
  ## Metrics input must be in the same order as function SC_spatial_aggregation
  scorecard_single <- ScorecardsSingle(data = aggregated_metrics,
                                       system = system,
                                       reference = reference,
                                       var = var,
                                       start.year = start.year,
                                       end.year = end.year,
                                       start.months = start.months,
                                       forecast.months = forecast.months,
                                       region.names = names(regions),
                                       metrics = metrics.visualize,
                                       table.label =  table.label,
                                       fileout.label = fileout.label,
                                       legend.white.space = legend.white.space, 
                                       col1.width = col1.width,
                                       col2.width = col2.width,
                                       output.path = output.path)
  
  ## Create multi system/reference scorecard tables
  ## (multiple systems with one reference or one system with multiple references)
  ## Metrics input must be in the same order as function SC_spatial_aggregation
  if(length(system) >  1 || length(reference) >  1){
    scorecard_multi <- ScorecardsMulti(data = aggregated_metrics,
                                       system = system,
                                       reference = reference,
                                       var = var,
                                       start.year = start.year,
                                       end.year = end.year,
                                       start.months = start.months,
                                       forecast.months = forecast.months,
                                       region.names = attributes(regions)$names,
                                       metrics = metrics.visualize,
                                       table.label =  table.label,
                                       fileout.label = fileout.label,
                                       output.path = output.path)
  } ## close if
  
  
  if(calculate.diff == TRUE){
    if(length(system) ==  2 || length(reference) ==  2){
      scorecard_diff <- ScorecardsSystemDiff(data = aggregated_metrics,
                                             system = system,
                                             reference = reference,
                                             var = var,
                                             start.year = start.year,
                                             end.year = end.year,
                                             start.months = start.months,
                                             forecast.months = forecast.months,
                                             region.names = attributes(regions)$names,
                                             metrics = metrics.visualize,
                                             table.label = table.label,
                                             fileout.label =  fileout.label,
                                             legend.white.space = legend.white.space, 
                                             col1.width = col1.width,
                                             col2.width = col2.width,
                                             output.path = output.path)
    } else {stop ("Difference scorecard can only be computed with two systems or two references.")}
  } ## close if on calculate.diff
  
}

