import datetime
import html
import re
import time

import dateutil
tzinfos = {"GMT": dateutil.tz.gettz("GMT")}
import numpy as np

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import standard_parameters, get_standard_data, get_standard_metadata, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseAERONETFile
from standardisation_mapping_functions import map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp


class StandardiseNetworkData:

    '''Convert daily provider original files in a month to GHOST standard numpy station specific files'''

    def __init__(self, month, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.month = month
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #set data level from network name
        self.data_level = self.run_details['network'].split('_')[-1]
        
        #iterate through relevant aeronet matrices for parameter
        for current_aeronet_matrix_ii in range(len(self.parameter_details['aeronet_matrix'])):
        
            #get current aeronet matrix/parameter name to parameter details
            self.parameter_details['current_aeronet_matrix'] = self.parameter_details['aeronet_matrix'][current_aeronet_matrix_ii]
            self.parameter_details['current_aeronet_parameter_name'] = self.parameter_details['aeronet_parameter_name'][current_aeronet_matrix_ii]

            #gather data files to parse
            data_files = self.gather_data_files(current_aeronet_matrix_ii)

            #parse all data files, writing standard files out per station
            self.parse_files(data_files)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self, current_aeronet_matrix_ii):

        '''Function to gather data files to read per data level'''

        #get specific data files for current month
        data_files = get_data_files(self.run_details, self.parameter_details, specific_month=self.month)
    
        #define select variable fields (also define date/time variable names)
        if self.parameter_details['current_aeronet_matrix'] == 'oneill':
            self.station_ref_var = 'AERONET_Site'
            self.date_var = 'Date_(dd:mm:yyyy)'
            self.time_var = 'Time_(hh:mm:ss)'
            self.dq_varname = 'Data_Quality_Level'
            self.station_name_var = 'AERONET_Site_Name'
            self.lon_var = 'Site_Longitude(Degrees)'
            self.lat_var = 'Site_Latitude(Degrees)'
            self.alt_var = 'Site_Elevation(m)'
            self.req_vars = [self.station_ref_var,self.date_var,self.time_var,self.parameter_details['current_aeronet_parameter_name'],standard_parameters['FINE_MODE_FRAC_500nm']['aeronet_parameter_name'][current_aeronet_matrix_ii],self.dq_varname,self.station_name_var,self.lon_var,self.lat_var,self.alt_var,'contact_name','contact_institution','contact_email_address']
        elif self.parameter_details['current_aeronet_matrix'] == 'directsun':   
            self.station_ref_var = 'AERONET_Site'
            self.date_var = 'Date(dd:mm:yyyy)'
            self.time_var = 'Time(hh:mm:ss)'
            self.dq_varname = 'Data_Quality_Level'
            self.station_name_var = 'AERONET_Site_Name' 
            self.lon_var = 'Site_Longitude(Degrees)'
            self.lat_var = 'Site_Latitude(Degrees)'
            self.alt_var = 'Site_Elevation(m)'
            if self.run_details['parameter'] == 'AOD_550nm':
                self.req_vars = [self.station_ref_var,self.date_var,self.time_var,standard_parameters['AOD_440nm']['aeronet_parameter_name'][current_aeronet_matrix_ii],standard_parameters['AOD_675nm']['aeronet_parameter_name'][current_aeronet_matrix_ii],standard_parameters['AOD_870nm']['aeronet_parameter_name'][current_aeronet_matrix_ii],standard_parameters['AE_440-870nm']['aeronet_parameter_name'][current_aeronet_matrix_ii],self.dq_varname,self.station_name_var,self.lon_var,self.lat_var,self.alt_var,'contact_name','contact_institution','contact_email_address']
            else:
                self.req_vars = [self.station_ref_var,self.date_var,self.time_var,self.parameter_details['current_aeronet_parameter_name'],self.dq_varname,self.station_name_var,self.lon_var,self.lat_var,self.alt_var,'contact_name','contact_institution','contact_email_address']
        elif self.parameter_details['current_aeronet_matrix'] == 'almucantar':
            self.station_ref_var = 'AERONET_Site'
            self.date_var = 'Date(dd:mm:yyyy)'
            self.time_var = 'Time(hh:mm:ss)'
            self.dq_varname = 'Inversion_Data_Quality_Level'
            self.station_name_var = 'AERONET_Site'
            self.lon_var = 'Longitude(Degrees)'
            self.lat_var = 'Latitude(Degrees)'
            self.alt_var = 'Elevation(m)'
            self.req_vars = [self.station_ref_var,self.date_var,self.time_var,self.parameter_details['current_aeronet_parameter_name'],self.dq_varname,self.lon_var,self.lat_var,self.alt_var,'contact_name','contact_institution','contact_email_address','If_AOD_is_L2','If_Retrieval_is_L2(without_L2_0.4_AOD_440_threshold)']

        return data_files


    def parse_files(self, data_files):

        '''Function to parse data files in a month for a data level to GHOST standard numpy station specific files'''
        
        self.run_details['log_file_str'] += 'PROCESSING DATA LEVEL: {}\n'.format(self.data_level) 

        #define dictionary for storing all parsed data (per specific data level)
        all_data = {k:[] for k in self.req_vars}
    
        #iterate through daily data files (for specific data level)
        for data_file in data_files:
            self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 

            #create object for parsing AERONET daily file
            parsing_obj = ParseAERONETFile()

            #open and read file contents into memory
            f = open(data_file, "r", encoding='latin1')
            page = f.read()
            f.close()

            #replace erroneously used less than character to break lines correctly
            page = re.sub("to_<", "to_lessthan", page)

            #parse read file contents 
            parsing_obj.feed(page)

            #check if read file actually has some data, if not then continue 
            if hasattr(parsing_obj, 'var_data'):
                if len(parsing_obj.var_data[self.station_ref_var]) == 0:
                    continue
            else:    
                continue
            
            #iterate through read object dictionary keys
            for var in list(parsing_obj.var_data.keys()):
                #is read object key one of required vars?
                if var in self.req_vars:
                    #if so --> append to all data dictionary
                    all_data[var] = np.append(all_data[var], np.array(parsing_obj.var_data[var], dtype=object))

            #add parsed contact information to all_data dictionary
            all_data['contact_name'] = np.append(all_data['contact_name'], np.array([parsing_obj.contact_name]*len(parsing_obj.var_data[self.station_ref_var]), dtype=object))
            all_data['contact_institution'] = np.append(all_data['contact_institution'], np.array([parsing_obj.contact_institution]*len(parsing_obj.var_data[self.station_ref_var]), dtype=object)) 
            all_data['contact_email_address'] = np.append(all_data['contact_email_address'], np.array([parsing_obj.contact_email_address]*len(parsing_obj.var_data[self.station_ref_var]), dtype=object)) 
            
        #get all unique station reference variables 
        unique_station_references = np.unique(all_data[self.station_ref_var])

        #get current AERONET data level in format as reported in file
        data_level_mod = self.data_level.replace(".","") 
        #get mask of all data of correct data level (for sanity)
        data_level_bool = np.array([True if var == data_level_mod else False for var in all_data[self.dq_varname]])

        #iterate through unique stations references in data
        for orig_station_reference in unique_station_references:

            #strip underscore from original unique station reference string
            station_reference = orig_station_reference.replace('_','')

            self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(station_reference)

            #create site instances of metadata/data dictionaries
            site_data =     get_standard_data(self.parameter_details)
            site_metadata = get_standard_metadata(self.parameter_details)

            #isolate variable data specific for station
            unique_station_bool = all_data[self.station_ref_var] == orig_station_reference
            #further isolate variable data to be only that associated with a data quality level that is currently being processed 
            unique_station_bool = data_level_bool & unique_station_bool
            station_var_data = {req_var:all_data[req_var][unique_station_bool] for req_var in self.req_vars}
            if self.run_details['parameter'] == 'AOD_550nm':
                station_var_data['AOD_550nm'] = []

            #------------------------------------------------------------------------#
            #PARSE TIME
            #------------------------------------------------------------------------#
        
            #parse datetime for each reported timepoint 
            #first create a unified date/time string (yyyy-mm-dd hh:mm:ss Z, of the measurement --> z is always GMT)
            #secondly parse into datetime (taking into account timezone information). then convert to UTC datetime, then round to nearest minute    
            start_dt = [round_to_minute(dateutil.parser.parse('{}-{}-{} {} GMT'.format(date[6:],date[3:5],date[:2],time), tzinfos=tzinfos).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date,time in zip(station_var_data[self.date_var],station_var_data[self.time_var])] 
            #convert each start measurement datetime to minutes from reference time
            frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

            #------------------------------------------------------------------------#
            #PARSE METADATA
            #------------------------------------------------------------------------#

            #GHOST version
            site_metadata['GHOST_version']['value'] = [self.run_details['ghost_version']]*len(frac_time)

            #initialise string which will store all GHOST process warnings as empty            
            site_metadata['process_warnings']['value'] = ['']*len(frac_time)

            #write measurement position metadata to metadata/data dictionaries
            for meta_ii in range(len(frac_time)):
                site_metadata['longitude']['value'].append(station_var_data[self.lon_var][meta_ii])
                site_metadata['latitude']['value'].append(station_var_data[self.lat_var][meta_ii])
                site_metadata['altitude']['value'].append(station_var_data[self.alt_var][meta_ii])

            #save meta_update_stamp, set as copy of time array
            site_metadata['meta_update_stamp']['value'] = frac_time

            #write known metadata  
            site_metadata['network']['value'] = [self.run_details['network']]*len(frac_time)
            site_metadata['station_name']['value'] = station_var_data[self.station_name_var]
            site_metadata['data_level']['value'] = [self.data_level.replace("lev","")]*len(frac_time)
            site_metadata['contact_name']['value'] = station_var_data['contact_name']
            site_metadata['contact_institution']['value'] = station_var_data['contact_institution']
            site_metadata['contact_email_address']['value'] = station_var_data['contact_email_address']
            site_metadata['principal_investigator_name']['value'] = station_var_data['contact_name']
            site_metadata['principal_investigator_institution']['value'] = station_var_data['contact_institution']
            site_metadata['principal_investigator_email_address']['value'] = station_var_data['contact_email_address']
            site_metadata['data_licence']['value'] = ['Public domain: https://aeronet.gsfc.nasa.gov/new_web/data_usage.html']*len(frac_time)
            if self.parameter_details['current_aeronet_matrix'] == 'directsun':
                site_metadata['retrieval_algorithm']['value'] = ['directsun algorithm']*len(frac_time) 
            elif self.parameter_details['current_aeronet_matrix'] == 'oneill':
                site_metadata['retrieval_algorithm']['value'] = ['spectral deconvolution algorithm']*len(frac_time)
            elif self.parameter_details['current_aeronet_matrix'] == 'almucantar':
                site_metadata['retrieval_algorithm']['value'] = ['inversion code algorithm']*len(frac_time)
            #assume datum information
            site_metadata['ellipsoid']['value'] = ['WGS 84']*len(frac_time)
            site_metadata['horizontal_datum']['value'] = ['World Geodetic System 1984']*len(frac_time)
            site_metadata['vertical_datum']['value'] = ['tidal - mean sea level']*len(frac_time)
            site_metadata['projection']['value'] = ['geographic']*len(frac_time)

            #process measurement information
            standard_measurement_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(self.parameter_details['current_aeronet_matrix'], site_metadata, self.run_details, self.parameter_details, write_to_metadata_array=False)
            for key, value in standard_measurement_metadata.items():
                site_metadata[key]['value'] = [value] * len(frac_time)
            site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
            measurement_qa_flag_codes = []
            #set measurement qa codes which are True as joined metadata str
            for standard_flag_name in measurement_qa_flags.keys():
                if measurement_qa_flags[standard_flag_name]:
                    measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
            site_metadata['measurement_qa_flags']['value'] = [','.join(measurement_qa_flag_codes)]*len(frac_time)

            #append station_reference now that we have the standard method abbreviation
            site_metadata['station_reference']['value'] = ['{}_{}'.format(station_reference, standard_method_abbreviation)]*len(frac_time)

            #iterate through metadata variables, if any columns have not been appended to add a NaN
            for column_header in list(site_metadata.keys()):
                n_column = len(site_metadata[column_header]['value'])
                #if column is empty, then append NaN
                if n_column < 1:
                    site_metadata[column_header]['value'] = [np.NaN] * len(frac_time)

            #get data file download timestamp
            site_metadata['data_download_stamp']['value'] = [get_datafile_download_timestamp(data_file)] * len(frac_time)

            #add data revision timestamp, set as 0001-01-01 00:00 
            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
            site_metadata['data_revision_stamp']['value'] = [data_revision_timestamp] * len(frac_time) 

            #------------------------------------------------------------------------#
            #PARSE FLAGS/QA
            #------------------------------------------------------------------------#

            #create empty list for standard data flags associated with each measurement
            flags = []
            qa = []
 
            #process data flags (some artificially created)
            for val_ii, val in enumerate(station_var_data[self.date_var]):

                #lists to append qa/data flags per measurement
                flags_per_measurement = []
                qa_per_measurement = []

                #set GHOST/network decreed validity as initially 'V'
                ghost_decreed_validity = 'V'
                network_decreed_validity = 'V'

                #when calculating AOD 550nm from AERONET directsun measurements, artificially create QA flag variable where one of used components is outside valid bounds
                if (self.parameter_details['current_aeronet_matrix'] == 'directsun') & (self.run_details['parameter'] == 'AOD_550nm'): 
                
                    #Is AOD_440nm, AOD675nm, AOD870nm or AE_440-870nm outside valid bounds?
                    #calculate AOD_550nm (if all needed component values in valid ranges)
                    float_aod440 = float(station_var_data['AOD_440nm'][val_ii]) 
                    float_aod675 = float(station_var_data['AOD_675nm'][val_ii]) 
                    float_aod870 = float(station_var_data['AOD_870nm'][val_ii]) 
                    float_ae440_870 = float(station_var_data['440-870_Angstrom_Exponent'][val_ii]) 
                    aod440_valid = False
                    aod675_valid = False
                    aod870_valid = False
                    ae440_870_valid = False

                    #AE_440-870nm outside valid bounds?
                    if (float_ae440_870 >= standard_parameters['AE_440-870nm']['extreme_lower_limit']) & (float_ae440_870 <= standard_parameters['AE_440-870nm']['extreme_upper_limit']):
                        ae440_870_valid = True
                        #AOD_440nm outside valid bounds?
                        if (float_aod440 >= standard_parameters['AOD_440nm']['extreme_lower_limit']) & (float_aod440 <= standard_parameters['AOD_440nm']['extreme_upper_limit']):
                            aod_550_adjust1 = (float_aod440/(440**(-float_ae440_870)))*(550**(-float_ae440_870))
                            aod440_valid = True
                        #AOD_675nm outside valid bounds?
                        if (float_aod675 >= standard_parameters['AOD_675nm']['extreme_lower_limit']) & (float_aod675 <= standard_parameters['AOD_675nm']['extreme_upper_limit']):
                            aod_550_adjust2 = (float_aod675/(675**(-float_ae440_870)))*(550**(-float_ae440_870))
                            aod675_valid = True
                        #AOD_870nm outside valid bounds?
                        if (float_aod870 >= standard_parameters['AOD_870nm']['extreme_lower_limit']) & (float_aod870 <= standard_parameters['AOD_870nm']['extreme_upper_limit']):   
                            aod_550_adjust3 = (float_aod870/(870**(-float_ae440_870)))*(550**(-float_ae440_870))
                            aod870_valid = True

                    #add qa flags to reflect that data is product
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Data Product'])

                    #if one of needed values is outside of valid bounds then set qa flag, and also set AOD 550nm value as NaN
                    if (not aod440_valid) or (not aod675_valid) or (not aod870_valid) or (not ae440_870_valid):
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Insufficient Data to Calculate Data Product'])
                        ghost_decreed_validity = 'I'
                        network_decreed_validity = 'I'
                        station_var_data['AOD_550nm'].append(np.NaN)
                    #append valid calculated AOD 500nm
                    else:
                        station_var_data['AOD_550nm'].append(np.mean([aod_550_adjust1,aod_550_adjust2,aod_550_adjust3]))
            
                #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                if ghost_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                if network_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])           

                #add 'Un-Flagged Data' flag for all measurements as have no flags
                flags_per_measurement.append(standard_data_flag_name_to_data_flag_code['Un-Flagged Data'])

                #append flags/qa codes to list to write out
                flags.append(flags_per_measurement)
                qa.append(qa_per_measurement)

            #----------------------------------------------------------------------------#
            #WRITE OUT DATA
            #----------------------------------------------------------------------------#
            site_data['time']['value'] = frac_time
            site_data[self.parameter_details['bsc_parameter_name']]['value'] = station_var_data[self.parameter_details['current_aeronet_parameter_name']]
            site_data['temporal_resolution']['value'] = [0]*len(frac_time)   
            site_data['flag']['value'] = flags
            site_data['qa']['value'] = qa
            #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
            for column_header in list(site_data.keys()):
                n_column = len(site_data[column_header]['value'])
                #if column is empty, then append NaN
                if n_column == 0:
                     if site_data[column_header]['data_type'] != np.uint8:
                         site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                     else:
                         site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
            #------------------------------------------------------
 
            #standardise metadata/data formats and write out station data to numpy file
            write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
            if write_obj.written == False:
                self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(station_reference)