'''
#################################################################################
This software prototype has been developed under the Copernicus C3S_512 contract.

Institution: BSC
Author: Jesus Peña-Izquierdo
Email: jesus.pena@bsc.es
#################################################################################
'''

import xarray as xr
from pathlib import Path
import os
import numpy as np
import sys
import yaml
import logging
from src.tools import tools
t = tools()


class transferring:   
    
    def __init__(self,
                 conf
                ):
        
        self.dataset_parameters = conf['dataset_parameters']
        
        self.variable = conf['dataset_parameters']['variable']
        self.variable_ref = conf['dataset_parameters']['variable_ref']
        self.originating_centre = conf['dataset_parameters']['originating_centre']
        self.system = conf['dataset_parameters']['system']
        self.years = conf['dataset_parameters']['years']
        self.start_months = conf['dataset_parameters']['start_months']
        
        if not self.variable_ref:
            self.variable_ref = self.variable
        
        self.dataset_sf = conf['dataset_parameters']['dataset_sf']
        self.data_aggregation_sf = conf['transfer_parameters']['data_aggregation_sf']
        
        self.dataset_ref = conf['dataset_parameters']['dataset_ref']
        self.data_aggregation_ref = conf['transfer_parameters']['data_aggregation_ref']

        self.downloaded_data_one_level =  conf['transfer_parameters']['downloaded_data_one_level']
        
        if 'pressure' in self.dataset_sf:
            self.levels = conf['dataset_parameters']['levels']
        else:
            self.levels = [None]
            
        self.force_transfer = conf['transfer_parameters']['force_transfer']
        self.compression = conf['transfer_parameters']['compression']
        
        self.base_dataIN_path = Path(conf['folders']['base_dataIN_path'])
        self.base_dataOUT_path = Path(conf['folders']['base_dataOUT_path'])
        self.logs_path = Path(conf['folders']['logs_path'])
        
        self.file_format_read = conf['transfer_parameters']['file_format_read']
        self.file_format_used = conf['transfer_parameters']['file_format_used']

        self.time_dimname = conf['compute_parameters']['xarray_parameters']['time_dimname']
        self.leadtime_dimname = conf['compute_parameters']['xarray_parameters']['leadtime_dimname']
        self.member_dimname = conf['compute_parameters']['xarray_parameters']['member_dimname']
        self.lon_dimname = conf['compute_parameters']['xarray_parameters']['lon_dimname']
        self.lat_dimname = conf['compute_parameters']['xarray_parameters']['lat_dimname']
        self.pressure_dimname = conf['compute_parameters']['xarray_parameters']['pressure_dimname']
        
        self.downloaded = conf['transfer_parameters']['downloaded']
        # From the download.py
        if self.downloaded: 
            # Only for download option monthly_files. For single_files/startmonth_files is not implemented here  
            self.pathIN_sf = self.base_dataIN_path.joinpath('seasonal', self.dataset_sf, 'monthly_files', self.variable) #, 'time-unification2', 'date')
            self.pathIN_ref = self.base_dataIN_path.joinpath('reanalysis', self.dataset_ref, 'monthly_files', self.variable_ref)
        
        # From datachecker or alternative archive
        else:
            self.pathIN_sf = self.base_dataIN_path.joinpath('seasonal', self.dataset_sf, self.variable)
            self.pathIN_ref = self.base_dataIN_path.joinpath('reanalysis', self.dataset_ref, self.variable_ref) #, 'special_ref') ####### 'special_ref' only for special case (min/max 2m temperature in last 24h)
            
        self.pathOUT_sf = self.base_dataOUT_path.joinpath('seasonal', self.dataset_sf)
        self.pathOUT_ref = self.base_dataOUT_path.joinpath('reanalysis', self.dataset_ref)

    
        formatter = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
        log_error_file = 'log_error_transfer_{}s{}_{}.log'.format(self.originating_centre,
                                                                  self.system,
                                                                  self.variable)
        logging.basicConfig(filename = self.logs_path.joinpath(log_error_file),
                            level = "WARNING",
                            format = formatter,
                            filemode = "w")
        

        
        with open("./src/grib_conventions.yaml") as f:
            self.grib_conventions = yaml.load(f, Loader=yaml.FullLoader)
            
        if self.file_format_used == 'netcdf':
            self.file_formatOUT = 'nc'
            self.engineOUT = 'netcdf4'
        else:
            self.file_formatOUT = 'grib'
            self.engineOUT = 'cfgrib'
            
        if self.file_format_read == 'netcdf':
            self.file_formatIN = 'nc'
            self.engineIN = 'netcdf4'
        else:
            self.file_formatIN = 'grib'
            self.engineIN = 'cfgrib'
        
        
        print('\n#########################')
        print('TRANSFERRING.............')
        print('#########################\n')
        
        t.print_header(self.dataset_parameters)


  
    
    def update_level(self,
                     level
                    ):
        
        # This function update the level information of the class
        
        if 'pressure' in self.dataset_sf:
            self.level_ext = '-p' + str(level)
        else:
            self.level_ext = ''


    def get_sf(self):
        
        # This function extract the required seasonal forecast data from the datachecker archive
                
        print('\nStarting the extraction and transfer of selected data for the Seasonal Forecast dataset...')
        self.pathIN_sf.joinpath("time-unification")
        
        self.pathOUT_m_sf = self.pathOUT_sf.joinpath('monthly_files', self.variable)
        self.transfer_sf_ok = True
        
        if not os.path.exists(self.pathOUT_m_sf):
            os.makedirs(self.pathOUT_m_sf)
        
        #To extract the monthly mean of each member
        gribkey_sf = self.grib_conventions['filters_sf']
        
        for yy in range(self.years[0], self.years[1]+1):
            for mm in range(self.start_months[0], self.start_months[1]+1):

                file_date = '{}{:02d}01'.format(yy, mm)
                
                # From download.py
                if self.downloaded:
                    fileIN = '{}s{}-{}-{}.{}'.format(self.originating_centre,
                                                                 self.system,
                                                                 self.variable,
                                                                 file_date, 
                                                                 self.file_formatIN)
                # From datachecker or alternative archive
                else:
                    fileIN = '{}-{}-{}.{}'.format(self.originating_centre,
                                                     self.system, 
                                                     file_date, 
                                                     self.file_formatIN
                                                    )

                fileIN_path = self.pathIN_sf.joinpath(fileIN)
                
                print('\nTransferring from file: {}...'.format(fileIN_path))
                
                # Check if files already exist
                transfer_files = []
                for level_i in self.levels:
                    self.update_level(level_i)

                    fileOUT = '{}s{}-{}-{}{:02d}01{}.{}'.format(self.originating_centre,
                                                              self.system,
                                                              self.variable,
                                                              yy, mm,
                                                              self.level_ext,  
                                                              self.file_formatOUT  
                                                             )
                    fileOUT_path = self.pathOUT_m_sf.joinpath(fileOUT)
                    
                    transfer_files.append((not os.path.exists(fileOUT_path)) or self.force_transfer)
                print(transfer_files)
                # If any of the files have not been transferred or the force transfer is enabled
                if 'pressure' in self.dataset_sf:
                    if len(self.levels) == 1:
                        if transfer_files[0]:
                            sum_transfer = True
                        else:
                            sum_transfer = False
                    if len(self.levels) == 2:
                        if transfer_files[0] or transfer_files[1]:
                            sum_transfer = True
                        else:
                            sum_transfer = False
                    if len(self.levels) == 3:
                        if transfer_files[0] or transfer_files[1] or transfer_files[2]:
                            sum_transfer = True
                        else:
                            sum_transfer = False
                    if len(self.levels) == 4:
                        if transfer_files[0] or transfer_files[1] or transfer_files[2] or transfer_files[3]:
                            sum_transfer = True
                        else:
                            sum_transfer = False
                    if len(self.levels) == 5:
                        if transfer_files[0] or transfer_files[1] or transfer_files[2] or transfer_files[3] or transfer_files[4]:
                            sum_transfer = True
                        else:
                            sum_transfer = False
                else:
                    if transfer_files[0]:
                        sum_transfer = True
                    else:
                        sum_transfer = False
                if sum_transfer: #np.sum(transfer_files):
                    if True: #try:
                        # Remove .idx files to avoid possible problems
                        removed = [os.remove(self.pathIN_sf.joinpath(f)) for f in os.listdir(self.pathIN_sf) \
                                    if f.startswith(fileIN) and f.endswith('.idx')]
                        # Reading
                        
                        print(fileIN_path)
                        data_fileIN = xr.open_dataarray(fileIN_path,
                                                        engine = self.engineIN,
                                                        backend_kwargs = gribkey_sf
                                                       )
                        
                        # One file per level, year and month
                        for level_i in self.levels:
                            self.update_level(level_i)
                            
                            fileOUT = '{}s{}-{}-{}{:02d}01{}.{}'.format(self.originating_centre,
                                                                      self.system,
                                                                      self.variable,
                                                                      yy, mm,
                                                                      self.level_ext,  
                                                                      self.file_formatOUT  
                                                                     )

                            fileOUT_path = self.pathOUT_m_sf.joinpath(fileOUT)

                            
                            if (not os.path.exists(fileOUT_path)) or self.force_transfer:
                                # Select specific level
                                if 'pressure' in self.dataset_sf: 
                                    if self.downloaded_data_one_level: #if len(self.levels) == 1:
                                        data_fileOUT = data_fileIN
                                    else:
                                        data_fileOUT = data_fileIN.sel({self.pressure_dimname: level_i})
                                else:
                                    data_fileOUT = data_fileIN
                                
                                if self.originating_centre == 'ncep':
                                    data_fileIN_level = data_fileOUT
                                    #if self.variable != "2m_temperature": #if self.variable == "sea_surface_temperature": ##ADD MASKED VARIABLES!!
                                        #fileIN_path_template = Path('/data/alba/suite/downloads/seasonal/seasonal-monthly-single-levels/monthly_files/2m_temperature').joinpath('nceps2-2m_temperature-{}{:02d}01.grib'.format(yy,mm))
                                    fileIN_path_template = Path('/shared/data_for_scorecards_SF/suite/templates_needed_to_transfer_ncep_and_ukmo').joinpath('nceps2-2m_temperature-{}{:02d}01.grib'.format(yy,mm))
                                    data_template = xr.open_dataarray(fileIN_path_template,
                                                        engine = self.engineIN)
                                                                                                            
                                    data_all_mem = []

                                    for mem in range(0, 16):
                                        print('-------- preparing member number: {} ---------'.format(mem))
                                        data_mem = data_fileIN_level.where(data_fileIN_level.number==mem, drop = True)
                                        #if self.variable != "2m_temperature": #if self.variable == "sea_surface_temperature":
                                        data_no_mask = data_template.where(data_template.number==mem, drop=True)
                                        data_mem = data_mem.where(np.isnan(data_no_mask) == False, drop = True)
                                        #else:
                                        #    data_mem = data_mem.where(np.isnan(data_mem) == False, drop = True)
                                        if mem==0:
                                            data_mem0 = data_mem
                                        else:
                                            #print(data_mem) #issue
                                            data_mem = data_mem.assign_coords({'step': data_mem0.step})
                                            data_mem = data_mem.assign_coords({'time': data_mem0.time})
                                        
                                        data_all_mem.append(data_mem)
                                    
                                    print('-------- concatenating all members ----------')
                                    data_fileOUT = xr.concat([data_all_mem[0],
                                                                data_all_mem[1],
                                                                data_all_mem[2],
                                                                data_all_mem[3],
                                                                data_all_mem[4],
                                                                data_all_mem[5],
                                                                data_all_mem[6],
                                                                data_all_mem[7],
                                                                data_all_mem[8],
                                                                data_all_mem[9],
                                                                data_all_mem[10],
                                                                data_all_mem[11],
                                                                data_all_mem[12],
                                                                data_all_mem[13],
                                                                data_all_mem[14],
                                                                data_all_mem[15],
                                                                ], 'number')
                                
                                if self.originating_centre == 'ukmo' and mm <5: 
                                    data_fileIN_level = data_fileOUT
                                    
                                    #if self.variable != "2m_temperature": #if self.variable == "sea_surface_temperature":  ##ADD MASKED VARIABLES!! #Or just do it always because there can be unexpecte NA as in specific_humidity at 850hPa
                                    fileIN_path_template = Path('/shared/data_for_scorecards_SF/suite/templates_needed_to_transfer_ncep_and_ukmo').joinpath('ukmos14-2m_temperature-{}{:02d}01.grib'.format(yy,mm))
                                    data_template = xr.open_dataarray(fileIN_path_template,
                                                        engine = self.engineIN)
                                    
                                    data1 = data_fileIN_level.where(data_fileIN_level.number<7, drop = True)
                                    #if self.variable != "2m_temperature":
                                    data_no_mask = data_template.where(data_template.number<7, drop=True)
                                    data1 = data1.where(np.isnan(data_no_mask) == False, drop = True)
                                    #else:
                                    #    data1 = data1.where(np.isnan(data1) == False, drop = True)
                                    
                                    data2 = data_fileIN_level.where(data_fileIN_level.number<14, drop = True)
                                    data2 = data2.where(data2.number>6, drop = True)
                                    #if self.variable != "2m_temperature":
                                    data_no_mask = data_template.where(data_template.number<14, drop=True)
                                    data_no_mask = data_no_mask.where(data_no_mask.number>6, drop=True)
                                    data2 = data2.where(np.isnan(data_no_mask) == False, drop = True)
                                    #else:
                                    #    data2 = data2.where(np.isnan(data2) == False, drop = True)
                                    data2 = data2.assign_coords({'step': data1.step})
                                    data2 = data2.assign_coords({'time': data1.time})
                                    
                                    data3 = data_fileIN_level.where(data_fileIN_level.number<21, drop = True)
                                    data3 = data3.where(data3.number>13, drop = True)
                                    #if self.variable != "2m_temperature":
                                    data_no_mask = data_template.where(data_template.number<21, drop=True)
                                    data_no_mask = data_no_mask.where(data_no_mask.number>13, drop=True)
                                    data3 = data3.where(np.isnan(data_no_mask) == False, drop = True)
                                    #else:
                                    #    data3 = data3.where(np.isnan(data3) == False, drop = True)
                                    data3 = data3.assign_coords({'step': data1.step})
                                    data3 = data3.assign_coords({'time': data1.time})
                                    
                                    data4 = data_fileIN_level.where(data_fileIN_level.number>20, drop = True)
                                    #if self.variable != "2m_temperature":
                                    data_no_mask = data_template.where(data_template.number>20, drop=True)
                                    data4 = data4.where(np.isnan(data_no_mask) == False, drop = True)
                                    #else:
                                    #    data4 = data4.where(np.isnan(data4) == False, drop = True)
                                    data4 = data4.assign_coords({'step': data1.step})
                                    data4 = data4.assign_coords({'time': data1.time})
                                    
                                    data_fileOUT = xr.concat([data1, data2, data3, data4], 'number')
                                    
                                # Remove problematic coordinates
                                data_fileOUT = t.clean_coordinates(data_fileOUT,
                                                                   coordinates = [self.time_dimname,
                                                                                  self.lon_dimname,
                                                                                  self.lat_dimname,
                                                                                  self.leadtime_dimname,
                                                                                  self.member_dimname
                                                                                 ]
                                                                  )
                                
                                
                                # Add time as a dimension
                                if self.originating_centre != 'ncep' and self.originating_centre != 'ukmo':
                                    data_fileOUT = data_fileOUT.expand_dims('time')
                                if self.originating_centre == 'ukmo' and mm>4:
                                    data_fileOUT = data_fileOUT.expand_dims('time')
                                
                                # Remove if exist to avoid writting permission problems
                                if os.path.exists(fileOUT_path):
                                    os.remove(fileOUT_path)

                                # Writting
                                data_fileOUT.to_netcdf(fileOUT_path, 
                                                      encoding = {data_fileOUT.name:{'zlib': self.compression}}
                                                     )
                                
                                print('...DONE: {} transferred.'.format(fileOUT_path))
                                
                            elif os.path.exists(fileOUT_path):
                                print('...SKIPPED: {} already exists.'.format(fileOUT_path))
                                
                        data_fileIN.close()
                        
                    #except Exception as e:
                    #    print('\n------ERROR------\nReading file: {}'.format(e))
                    #    logging.error(e)
                    #    
                    #    self.transfer_sf_ok = False
                        
                        #print('\nTRANSFER FOR SYSTEM [{}-s{}] and VARIABLE [{}] SKIPPED.\n'format(self.originating_centre,
                        #                                                                    self.system,
                        #                                                                    self.variable))
                        # Missing files lead to stop the transfer
                        #return

                else:
                    print('...SKIPPED: All selected levels already transferred.'.format(fileIN_path))


        # One file per level, year and month
        for level_i in self.levels:
            self.update_level(level_i)
            
            # Checking that all files are transferred.
            ok, files, files_missed = t.check_files_dates(folder_path = self.pathOUT_m_sf,
                                                          months = self.start_months,
                                                          years = self.years,
                                                          filename_start = '{}s{}-{}-'.format(self.originating_centre,
                                                                                           self.system,
                                                                                           self.variable),
                                                          filename_end = self.level_ext + '.' + self.file_formatOUT,
                                                          verbose = True
                                                         )
            
            # if not empty list
            if files:
                self.file_path_sf0 = files[0]


            if self.data_aggregation_sf == 'start_month':

                #Conversion of monthly files into one single files
                print('\nConverting monthly files to one file per start_month to improve reading performance...')

                #Output folder
                pathOUT_s_sf = self.pathOUT_sf.joinpath('startmonth_files', self.variable)           
                if not os.path.exists(pathOUT_s_sf):
                    os.makedirs(pathOUT_s_sf)

                year_limits = '{}to{}'.format(self.years[0], self.years[1])

                #Output files for each start month
                for mm in range(self.start_months[0], self.start_months[1]+1):
                    print('\n...Preloading all monthly files for start month {}...'.format(mm))

                    fileOUT = '{}s{}-{}-{}-s{:02d}{}.{}'.format(self.originating_centre,
                                                              self.system,
                                                              self.variable,
                                                              year_limits,
                                                              mm,
                                                              self.level_ext,
                                                              self.file_formatOUT
                                                             )

                    fileOUT_path = pathOUT_s_sf.joinpath(fileOUT)

                    # Check if file already exists and if force transfer is enabled
                    if (not os.path.exists(fileOUT_path)) or (self.force_transfer):

                        #Checking all dates (files) are located
                        ok, files, files_missed = t.check_files_dates(folder_path = self.pathOUT_m_sf,
                                                                       months = [mm, mm],
                                                                       years = self.years,
                                                                       filename_start = '{}s{}-{}-'.format(self.originating_centre,
                                                                                                   self.system,
                                                                                                   self.variable),
                                                                      filename_end = self.level_ext + '.' + self.file_formatOUT,
                                                                     )

                        if not ok:
                            e = "------ERROR------\nSome start months files missing: {}".format(files_missed)
                            print(e)
                            #sys.exit(e)
                            logging.error(e)   

                        else: 
                            leadtime_coords = xr.open_dataarray(files[0]).coords[self.leadtime_dimname]

                            #Preload dataset
                            data = xr.open_mfdataset(files,
                                                     concat_dim = self.time_dimname,
                                                     combine='nested',
                                                     drop_variables = self.leadtime_dimname
                                                    )
                            #Convert to datarray
                            var = list(data.data_vars)
                            data = data[var[0]]

                            # Reassigning leadtime coordinates
                            data = data.assign_coords({self.leadtime_dimname: leadtime_coords.values})

                            # Remove problematic coordinates
                            #data = data.drop(['valid_time'])

                            # Remove if exist to avoid writting permission problems
                            if os.path.exists(fileOUT_path):
                                os.remove(fileOUT_path)

                            print('...Storing data into one file...')
                            data.to_netcdf(fileOUT_path,
                                           encoding = {data.name:{'zlib': self.compression}}
                                          )   

                            print('...File {} saved.'.format(fileOUT_path))


                    # If file already exists and force transfer is disabled, then it skips the file generation    
                    elif os.path.exists(fileOUT_path):
                        print('File already exists:\n{}\nGeneration skipped.'.format(fileOUT_path))
                                   
        print('\n\nTransfer of Seasonal Forecast files complete.')

            
    def get_ref(self):
        
        # This function extract the required reanalysis data from the datachecker archive and it interpolates
        # the reanalysis data to the coordinates of the SF system.
        
        self.transfer_ref_ok = True
        
        #if self.transfer_sf_ok:
        print('\nStarting the extraction and transfer of selected data for the Reference dataset...')

        self.update_level(self.levels[0])



        # Load first file of SF for interpolation coordinates
        file_sf0 = '{}s{}-{}-{}{:02d}01{}.{}'.format(self.originating_centre,
                                                   self.system,
                                                  self.variable,
                                                   self.years[0],
                                                   self.start_months[0],
                                                     self.level_ext,
                                                   self.file_formatOUT
                                                  )
        donttrydoit = True
        if donttrydoit: #try:
            file_path_sf0 = self.file_path_sf0
            
            #self.pathOUT_m_sf = self.pathOUT_sf.joinpath('monthly_files',self.variable)
            #file_path_sf0 = self.pathOUT_m_sf.joinpath(file_sf0)
        
            coords_like_sf = xr.open_dataarray(file_path_sf0,engine = self.engineOUT)

            # Extract coordinates for reference interpolation
            coords_like_sf = coords_like_sf.isel({self.member_dimname: 0,
                                                  self.leadtime_dimname: 0}) \
                                            .drop([self.time_dimname,
                                                   self.member_dimname,
                                                   self.leadtime_dimname]
                                                 )

            lon0 = str(coords_like_sf.coords[self.lon_dimname][0].values) + 'E'
            lat0 = str(t.lat2NS(coords_like_sf.coords[self.lat_dimname][0].values))
            lonlat0 = lon0 + lat0


            self.pathOUT_m_ref = self.pathOUT_ref.joinpath('monthly_files',self.variable)

            if not os.path.exists(self.pathOUT_m_ref):
                os.makedirs(self.pathOUT_m_ref)

            #To extract the specific product
            gribkey_ref = self.grib_conventions['filters_ref']

            for yy in range(self.years[0], self.years[1] + 2): # Add one more year for validating the forecast
                for mm in range(self.start_months[0], self.start_months[1]+1):
                    file_date = '{}{:02d}01'.format(yy, mm)
                    
                    # From download.py
                    if self.downloaded:
                        #fileIN = '{}-{}-{}.{}'.format(self.variable, ptype, file_date, self.file_formatIN)
                        fileIN = '{}-{}.{}'.format(self.variable, file_date, self.file_formatIN)
                    # From datachecker or alternative archive
                    else:
                        
                        fileIN = '{}-{}.{}'.format(self.variable_ref, file_date, self.file_formatIN)
                        
                    fileIN_path = self.pathIN_ref.joinpath(fileIN)

                    print('\nTransferring from file: {}...'.format(fileIN_path))

                    # Check if files already exist
                    transfer_files = []
                    for level_i in self.levels:
                        self.update_level(level_i)

                        fileOUT = '{}-grid{}-{}{}.{}'.format(self.variable, 
                                                       lonlat0,
                                                       file_date, 
                                                           self.level_ext,
                                                       self.file_formatOUT
                                                          )

                        fileOUT_path = self.pathOUT_m_ref.joinpath(fileOUT)

                        transfer_files.append((not os.path.exists(fileOUT_path)) or self.force_transfer)
                        
                        #print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')
                        #print(transfer_files)
                        #print(fileOUT_path)
                        #print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++')

                    # If any of the files have not been transferred or the force transfer is enabled
                    if np.sum(transfer_files):
                        #try:
                            #print('+++++++++++++++++TRY+++++++++++')
                            ## Remove .idx files to avoid possible problems
                            #removed = [os.remove(self.pathIN_ref.joinpath(f)) for f in os.listdir(self.pathIN_ref) \
                            #        if f.startswith(fileIN) and f.endswith('.idx')]
                            #print('+++++++++++++++++1+++++++++++')
                            #try:
                            #    print(fileIN_path)
                            #    print(self.engineIN)
                            #    data_fileIN = xr.open_dataarray(fileIN_path,
                            #                                engine = self.engineIN,
                            #                                backend_kwargs = gribkey_ref['instant']
                            #                                   )
                            #    print('+++++++++++++++++2+++++++++++')
                            #except:
                            #    print('++++++++++++++++++++3++++++++')
                            #    data_fileIN = xr.open_dataarray(fileIN_path,
                            #                                engine = self.engineIN,
                            #                                backend_kwargs = gribkey_ref['accumulated']
                            #                                   )
                            #print('+++++++++++++++++4+++++++++++')
                            
                            # Remove .idx files to avoid possible problems
                        removed = [os.remove(self.pathIN_ref.joinpath(f)) for f in os.listdir(self.pathIN_ref) \
                            if f.startswith(fileIN) and f.endswith('.idx')]

                        try:
                            data_fileIN = xr.open_dataarray(fileIN_path,
                                                        engine = self.engineIN)#,
                                                        #backend_kwargs = self.grib_conventions['filters_ref']['instant'])
                            

                        except Exception as ex:
                            print('\n------ERROR reading (1)------: {}'.format(ex))
                            
                            for try_i in self.grib_conventions['filters_ref']['stepType_extra']:
                                try:
                                    print('...trying stepType "{}"...'.format(try_i))
                                    # For different temporal aggregation variables defined in stepType_extra.
                                    gribkey_product_type = self.grib_conventions['filters_ref']['accumulated']
                                    gribkey_product_type['filter_by_keys']['stepType'] = try_i
                                        
                                    data_fileIN = xr.open_dataarray(fileIN_path,
                                                                engine = self.engineIN,
                                                                backend_kwargs = gribkey_product_type)
                                        
                                except Exception as ex:
                                    print('...stepType "{}" not working...'.format(try_i))
                                    # Next temporal aggregation variable




                        # One file per level, year and month
                        for level_i in self.levels:
                            self.update_level(level_i)
                            fileOUT = '{}-grid{}-{}{}.{}'.format(self.variable, 
                                                       lonlat0,
                                                       file_date, 
                                                           self.level_ext,
                                                       self.file_formatOUT
                                                          )
                            
                            fileOUT_path = self.pathOUT_m_ref.joinpath(fileOUT)

                            if (not os.path.exists(fileOUT_path)) or self.force_transfer:
                                # Select specific level
                                if 'pressure' in self.dataset_sf: 
                                    data_fileOUT = data_fileIN.sel({self.pressure_dimname: level_i})
                                    # Remove problematic coordinates
                                    data_fileOUT = data_fileOUT.drop([self.pressure_dimname])
                                else:
                                    data_fileOUT = data_fileIN
                                data_fileOUT = data_fileOUT.drop(['valid_time'])

                                data_fileOUT = data_fileOUT.interp_like(coords_like_sf) #\
                                                      #       .drop([self.member_dimname,
                                                      #              self.leadtime_dimname
                                                      #             ])
                                # Remove if exist to avoid writting permission problems
                                if os.path.exists(fileOUT_path):
                                    os.remove(fileOUT_path)

                                # Writting
                                data_fileOUT.to_netcdf(fileOUT_path, 
                                                          encoding = {data_fileOUT.name:{'zlib': self.compression}}
                                                         )


                                print('...DONE: {} interpolated and transferred.'.format(fileOUT_path))

                            elif os.path.exists(fileOUT_path):
                                print('...SKIPPED: {} already exists.'.format(fileOUT_path))
                        data_fileIN.close()
                        #except Exception as e:
                        #    print('\n------ERROR------\nReading file: {}'.format(e))
                        #    logging.error(e)
                        #
                        #    self.transfer_ref_ok = False

                            #print('\nTRANSFER FOR REFERENCE VARIABLE [{}] SKIPPED.\n'format(self.variable))

                            # Reading fail leads to skip transfer of files
                            #return

                    else:
                        print('...SKIPPED: All selected levels already transferred.'.format(fileIN_path))


             # One file per level, year and month
            for level_i in self.levels:
                self.update_level(level_i)

                # Checking that all files are transferred.
                ok, files, files_missed = t.check_files_dates(folder_path = self.pathOUT_m_ref,
                                                              months = self.start_months,
                                                              years = [self.years[0], self.years[1]+1],
                                                              filename_start = '{}-grid{}-'.format(self.variable,
                                                                                                   lonlat0
                                                                                                  ),
                                                              filename_end = self.level_ext + '.' + self.file_formatOUT,
                                                             )


                if (self.data_aggregation_ref == 'single_file') and self.transfer_ref_ok:

                    #Conversion of monthly files into one single files
                    print('\nConverting monthly files to one single file to improve reading performance...')

                    #Output folder
                    pathOUT_s_ref = self.pathOUT_ref.joinpath('single_file', self.variable)
                    if not os.path.exists(pathOUT_s_ref):
                        os.makedirs(pathOUT_s_ref)

                    #Output files
                    fileOUT = '{}-grid{}-{}to{}{}.{}'.format(self.variable, 
                                                           lonlat0,
                                                           self.years[0], self.years[1]+1,
                                                           self.level_ext,
                                                           self.file_formatOUT
                                                          )

                    fileOUT_path = pathOUT_s_ref.joinpath(fileOUT)

                    # Check if file already exists and if force transfer is enabled
                    if (not os.path.exists(fileOUT_path)) or (self.force_transfer):

                        #Checking all dates (files) are located
                        ok, files, files_missed = t.check_files_dates(folder_path = self.pathOUT_m_ref,
                                                               months = self.start_months,
                                                               years = [self.years[0], self.years[1]+1],
                                                               filename_start = '{}-grid{}-'.format(self.variable,
                                                                                                   lonlat0
                                                                                                  ),
                                                                      filename_end = self.level_ext + '.' + self.file_formatOUT,
                                                                            )


                        if not ok:
                            e = "------ERROR------\nSome reference files missing: {}".format(files_missed)
                            sys.exit(e)
                            logging.error(e)   

                        else: 
                            print('...Preloading monthly files...')

                            #Preload dataset
                            data = xr.open_mfdataset(files,
                                                     concat_dim = self.time_dimname,
                                                     combine='nested')
                            #Convert to datarray
                            var = list(data.data_vars)
                            data = data[var[0]]

                            # Remove problematic coordinates
                            #data = data.drop(['valid_time'])

                            print('...Storing data into one single file...')

                            # Remove if exist to avoid writting permission problems
                            if os.path.exists(fileOUT_path):
                                os.remove(fileOUT_path)
                           
                            data.to_netcdf(fileOUT_path,
                                           encoding = {data.name:{'zlib': self.compression}}
                                          )   

                            print('...File {} saved.'.format(fileOUT_path))


                    # If file already exists and force transfer is disable, then it skips the single file generation    
                    elif os.path.exists(fileOUT_path):
                        print('\nSingle file already exists: \n{}\nGeneration skipped.'.format(fileOUT_path))

            print('\n\nTransfer of ERA5 files complete.')
            
        #except:
        #    print("------ERROR------\nNo seasonal forecast files transferred.\nTransfer of reference data SKIPPED.")

    def special(self):
        self.transfer_ref_ok = True
        print('\nStarting the download of selected data for the Reference dataset...')

        self.update_level(self.levels[0])
        self.pathIN_ref = self.pathIN_ref.joinpath('special_ref')
        
        #self.pathOUT_m_ref = self.pathOUT_ref.joinpath('monthly_files',self.variable_ref)

        if not os.path.exists(self.pathIN_ref):
            os.makedirs(self.pathIN_ref)

        for yy in range(self.years[0], self.years[1] + 2): # Add one more year for validating the forecast
            for mm in range(self.start_months[0], self.start_months[1]+1):
                file_date = '{}{:02d}01'.format(yy, mm)
                fileIN = '{}-{}.{}'.format(self.variable_ref, file_date, self.file_formatIN)
                fileIN_path = self.pathIN_ref.joinpath(fileIN)
                #fileOUT_m = '{}-mean-{}.{}'.format(self.variable_ref, file_date, self.file_formatIN)
                #fileOUT_x = '{}-max-{}.{}'.format(self.variable_ref, file_date, self.file_formatIN)
                #fileOUT_n = '{}-min-{}.{}'.format(self.variable_ref, file_date, self.file_formatIN)

                download_path = fileIN_path

                print('\nTransferring from file: {}...'.format(fileIN_path))
                
                if os.path.exists(download_path):
                    print('')
                    print('download skipped')
                else:
                    download_form ={'product_type': 'reanalysis',
                                'variable': self.variable_ref,
                                'year': [str(yy)],
                                'month': ['{:02d}'.format(mm)],
                                'time': '00:00',
                                'format': 'grib'
                                }
            
                    download_form['day'] = ['{:02d}'.format(di) for di in range(1,31+1)]
                    download_form['time'] = ['{:02d}:00'.format(hi) for hi in range(0,24,1)]
            
                    print("\nDownload request: {}...".format(download_form))
                    
                    from C3S512.cds import downloader
                    downloader('reanalysis-era5-single-levels',
                            download_form,
                            download_path
                            )
                    print('...File {} downloaded.'.format(download_path))
                
            #    data_fileIN = xr.open_dataarray(fileIN_path, engine = self.engineIN)
            #    data_fileOUT = data_fileIN
            #
            #    print(data_fileOUT)
            #
            #    #compute max/min daily
            #    for t in range(0, data_fileIN.shape[0]-1):
            #        if 'maximum' in self.variable_ref: 
            #            data_fileOUT[t,:] = max(data_fileIN[t,:].max('step'),data_fileIN[t+1,:].max('step'))
            #        elif 'minimum' in self.variable_ref:
            #            data_fileOUT[t,:] = min(data_fileIN[t,:].min('step'),data_fileIN[t+1,:].min('step'))
            #    
            #    #compute monthly mean + save
            #    data_fileOUT_m = data_fileOUT.mean('time')
            #    pathOUT = self.pathOUT_m_ref.joinpath(fileOUT_m)
            #    data_fileOUT_m.to_netcdf(pathOUT, engine = 'netcdf4')
            #
            #    #compute monthly absolute max/min + save
            #    if 'maximum' in self.variable_ref: 
            #        data_fileOUT_x = data_fileOUT.max('time')
            #        pathOUT = self.pathOUT_m_ref.joinpath(fileOUT_x)
            #        data_fileOUT_x.to_netcdf(pathOUT, engine = 'netcdf4')
            #    elif 'minimum' in self.variable_ref: 
            #        data_fileOUT_n = data_fileOUT.min('time')
            #        pathOUT = self.pathOUT_m_ref.joinpath(fileOUT_n)
            #        data_fileOUT_n.to_netcdf(pathOUT, engine = 'netcdf4')
            #    print(data_fileOUT_m)
            #    print(data_fileOUT_x)
            #
            #    #remove hourly file

     
