import bisect
import copy
import time

import numpy as np
import pandas as pd

from crossnetwork_processing_functions import determine_key_changing_metadata, write_station_specific_numpy
from GHOST_standards import standard_QA_name_to_QA_code, standard_data_flag_name_to_data_flag_code
from geographic_functions import check_measurement_positions
import unit_converter


class NetworkFileStandardise:

    '''Class to standardise read metadata/data to GHOST formats'''

    def __init__(self, metadata, data, run_details, parameter_details):
        
        #add passed instance variables to self
        self.metadata = metadata
        self.data = data 
        self.run_details = run_details
        self.parameter_details = parameter_details

        #variable that sets if have written file
        self.written = False

        #check if all measurement data is missing (i.e. NaN), if so do not write file
        all_nan = self.check_if_all_nan()

        if not all_nan:
            #remove duplicated metadata columns read from file
            self.resolve_duplicate_metadata_from_file()

            #convert empty string metadata to NaN
            self.remove_empty_string_metadata()

            #check validity of saved measurement positions
            self.metadata = check_measurement_positions(self.metadata)

            #standardise data formats - convert to numpy dataframes
            self.metadata = self.convert_to_numpy_dataframe(self.metadata)
            self.data = self.convert_to_numpy_dataframe(self.data)

            #check for duplicated/overlapping times across whole time record
            self.process_overlapping_and_duplicate_times()

            #convert temporal resolutions of 0 to 1 for convenience
            self.convert_instantaneous_temporal_resolutions()

            #normalise measured concentrations in units of mass density/mole fraction to normal temperature and pressure 
            #do not do for certain matrices
            if self.parameter_details['matrix'] not in ['precip','meteo','aod','extaod','absaod','absaod-bc','absaod-brc','scataod','ssa','asy','rin','vconc','size','schuster']:
                self.normalise_concentrations()

            #write out to temporary station specific numpy file
            write_station_specific_numpy(self.metadata,self.data,self.run_details,self.parameter_details,update_file=False)
            self.written = True

    def check_if_all_nan(self):

        '''function that checks if all measurements are NaN, if so do not write file'''

        return all(pd.isnull(np.array(self.data[self.parameter_details['bsc_parameter_name']]['value'])))

    def resolve_duplicate_metadata_from_file(self):

        '''function that removes duplicate metadata read from a data file.
           this happens when a network reports metadata line by line, and therefore metadata is collected and processed line by line.
           a consequence of this is therefore multiple columns are duplicated needlessly, where no key fields are changing between columns. 
      
           this function checks column by column and if all key metadata metadata values are identical to the last column, the column is removed.
        '''

        #if number of metadata column is already 1, do not need to run function
        if len(self.metadata['station_reference']['value']) == 1:
            return 
        else:
            
            #initialise list to store metadata columns to remove
            metadata_column_indices_to_remove = []

            #iterate through metadata columns, check if one of key fields are different from previous column
            previous_metadata_column = {var:self.metadata[var]['value'][0] for var in self.metadata.keys()}
            for metadata_column_ii in range(1, 1+len(self.metadata['station_reference']['value'][1:])):
                metadata_column = {var:self.metadata[var]['value'][metadata_column_ii] for var in self.metadata.keys()}

                #has one of key metadata values changed between columns?
                has_column_changed = determine_key_changing_metadata(previous_metadata_column, metadata_column)
            
                #if not append column index to remove
                if not has_column_changed:
                    metadata_column_indices_to_remove.append(metadata_column_ii)
                #otherwise take current column as previous column to test against
                else:
                    previous_metadata_column = copy.deepcopy(metadata_column)

            #if have metadata columns to remove, do so now
            if len(metadata_column_indices_to_remove) > 0:
                for metadata_column_index_to_remove in sorted(metadata_column_indices_to_remove, reverse=True):
                    for var in self.metadata.keys():
                        self.metadata[var]['value'] = list(self.metadata[var]['value'])
                        del self.metadata[var]['value'][metadata_column_index_to_remove]


    def remove_empty_string_metadata(self):

        '''function that removes empty string metadata from a data file, by converting them to NaNs'''

        #iterate through metadata variables
        for var in self.metadata.keys():
            if (self.metadata[var]['data_type'] != np.uint32) & (var != 'process_warnings'):
                #iterate through metadata values
                for ii, val in enumerate(self.metadata[var]['value']):
                    #strip leading/trailing spaces from string 
                    if (self.metadata[var]['data_type'] == object) & (not pd.isnull(val)):
                        strip_val = np.str_(val).strip()
                    else:
                        strip_val = copy.deepcopy(val)
                    #if is empty string then convert it to be NaN
                    if strip_val == '':
                        self.metadata[var]['value'][ii] = np.NaN 
                    #else, rewrite stripped string
                    else:
                        self.metadata[var]['value'][ii] = strip_val


    def convert_to_numpy_dataframe(self, data_dict):

        '''function that processes data dictionary (i.e. metadata/data) processed from raw data files to numpy dataframe
           this process also standardises the data formats of the raw data
           also does string formatting (i.e. upper, lower case strings).
           all missing elements are conserved as np.NaN values throughout processing, regardless of type
        '''

        #create lists for standardising data formats of the input data dictionary
        standard_dtype = [(key,data_dict[key]['data_type']) for key in data_dict]

        #create numpy dataframe to store standardised forms of variables inside data dictionary
        standardised_dataframe = np.empty(len(data_dict[list(data_dict.keys())[0]]['value']), dtype=standard_dtype)

        #fill numpy dataframe with data
        for key in standardised_dataframe.dtype.names:

            #cut data for data_key
            param_array = data_dict[key]['value']

            #do string formatting for relevant fields before filling dataframe
            if (standardised_dataframe[key].dtype == np.dtype('O')) & (key not in ['flag', 'qa']):
    
                #upper case string formatting
                if 'upper' in data_dict[key]['string_format']:
                    param_array = [np.str_(x).upper() if pd.isnull(x) == False else x for x in param_array]

                #lower case string formatting
                elif 'lower' in data_dict[key]['string_format']:
                    param_array = [np.str_(x).lower() if pd.isnull(x) == False else x for x in param_array]

                #titalling string formatting
                elif 'title' in data_dict[key]['string_format']:
                    param_array = [np.str_(x).title() if pd.isnull(x) == False else x for x in param_array]

                #standard string formatting
                else:
                    param_array = [np.str_(x) if pd.isnull(x) == False else x for x in param_array]

            #fill dataframe
            standardised_dataframe[key] = param_array

        return standardised_dataframe


    def process_overlapping_and_duplicate_times(self):

        '''Function to check if any times are duplicate or overlapping across the time record from a specific data file.
           Start times are first sorted in ascending order to resolve potential overlapping times (also metadata is sorted by timestamp).
           Firstly, if a temporal window is identically duplicated (i.e. same start and end time), data which is flagged as being not maximum data quality level is screened out. If this does not resolve the duplication, data which is flagged by providers as preliminary is screened out. If the duplication is not resolved then data which is flagged as GHOST decreed invalid is then screened out. If this also does not resolve the duplication, then the first remaining temporal window is kept preferentially.
           Secondly, after removing the duplicate windows, it is checked if any window end times overlap with the next start time. If an overlap is found, again data which is flagged as being not maximum data quality level is screened out. If this does not resolve the duplication, data which is flagged by providers as preliminary is screened out. If this does not resolve the duplication, data which is flagged as GHOST decreed invalid is then screened out. If this also does not resolve the overlap, the remaining windows with a temporal resolution equal to the finest temporal resolution across the remaining windows are kept. If this finally does not resolve the duplication, then the first remaining temporal window is kept preferentially.
           These checks are done recursively until each window has no other windows which are duplicates or overlap with it.
           Function also deals with re-ordering/modification of all variables which are time variant
        '''

        #get QA flag code of Not Maximum Data Quality Level
        not_maximum_data_level_qa_flag = standard_QA_name_to_QA_code['Not Maximum Data Quality Level']

        #get data flag code of network decreed preliminary data
        preliminary_data_flag = standard_data_flag_name_to_data_flag_code['Preliminary Data']

        #get QA flag code of invalid GHOST decreed data
        invalid_GHOST_decreed_data_qa_flag = standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed']

        #sort measurement start time (and associated time variant variables) in ascending order to resolve potential overlapping times

        #get data parameter names
        data_keys = self.data.dtype.names

        #get indices of sorted time array
        sorted_time_indices = np.array(sorted(range(len(self.data['time'])), key=lambda k: self.data['time'][k]))

        #sort time array and other data arrays
        self.data = self.data[sorted_time_indices]

        #get indices of sorted metadata update stamp
        sorted_meta_update_stamp_indices = np.array(sorted(range(len(self.metadata['meta_update_stamp'])), key=lambda k: self.metadata['meta_update_stamp'][k]))

        #sort metadata update stamp array and other metadata arrays
        self.metadata = self.metadata[sorted_meta_update_stamp_indices]    

        #-----------------------
        #Add temporal resolutions to start times to get measurement end times
        #check for measurement windows which are duplicated or overlap iteratively with each individual window

        #iterate through time
        inds_to_remove = np.empty(0, dtype=np.uint32)
        for ii, start_time in enumerate(self.data['time'][:-1]):

            #only test for current time if not in list of bad indices to be removed
            if ii not in inds_to_remove:

                duplicate_inds = np.array([ii], dtype=np.uint32)
                overlap_inds = np.array([ii], dtype=np.uint32)

                #get window end time by adding temporal resolution as timedelta
                timedelta = self.data['temporal_resolution'][ii]

                #get measurement window end time
                end_time = start_time + timedelta

                #1. check if any measurement window is duplicate,
                #2. check if window end time overlaps with next start time,
                #keep checking until there are no overlaps/ duplicates with current window
                for jj in range(ii+1, (ii+1) + len(self.data['time'][ii+1:])):
                    next_start_time = self.data['time'][jj]

                    #only test for next time if not in list of bad indices to be removed
                    if jj not in inds_to_remove:

                        #get window end time by adding temporal resolution as timedelta
                        next_timedelta = self.data['temporal_resolution'][jj]

                        #get measurement window end time
                        next_end_time = next_start_time + next_timedelta

                        #duplicate check
                        if (start_time == next_start_time) & (end_time == next_end_time):
                            duplicate_inds=np.append(duplicate_inds, jj)
                        #overlap check
                        elif end_time > next_start_time:
                            overlap_inds=np.append(overlap_inds, jj)
                        #if no duplicates or overlaps, proceed to next start time
                        else:
                            break

                #resolve duplicate windows
                if len(duplicate_inds) > 1:

                    #first screen out not maximum data level flagged data
                    duplicate_inds_valid = np.array([duplicate_ind for duplicate_ind in duplicate_inds if not_maximum_data_level_qa_flag not in self.data['qa'][duplicate_ind]])
                    
                    #resolved duplication?
                    if len(duplicate_inds_valid) == 1:
                        #get only remaining index of valid duplicate inds
                        chosen_ind = duplicate_inds_valid[0]
                    #else if removing not maximum data level flagged data fails to resolve duplication, try to next resolve duplication using preliminary data
                    else:                   
                        if len(duplicate_inds_valid) == 0:
                            #restore original duplicate inds
                            duplicate_inds_valid = copy.deepcopy(duplicate_inds)

                        #secondly screen out preliminary data
                        duplicate_inds_valid_2 = np.array([duplicate_ind for duplicate_ind in duplicate_inds_valid if preliminary_data_flag not in self.data['flag'][duplicate_ind]])

                        #resolved duplication?
                        if len(duplicate_inds_valid_2) == 1:
                            #get only remaining index of valid duplicate inds 2
                            chosen_ind = duplicate_inds_valid_2[0]
                        #else if preliminary data flag fails to resolve duplication, try to resolve finally using GHOST decreed invalid data
                        else:
                            if len(duplicate_inds_valid_2) == 0:
                                #restore duplicate inds valid
                                duplicate_inds_valid_2 = copy.deepcopy(duplicate_inds_valid)

                            #thirdly screen out GHOST decreed invalid flagged data
                            duplicate_inds_valid_3 = np.array([duplicate_ind for duplicate_ind in duplicate_inds_valid_2 if invalid_GHOST_decreed_data_qa_flag not in self.data['qa'][duplicate_ind]])

                            #if removing GHOST decreed flagged data fails to resolve duplication, proceed to keep first remaining duplicate value preferentially
                            if len(duplicate_inds_valid_3) == 0:
                                #get first index of valid duplicate inds 2 
                                chosen_ind = duplicate_inds_valid_2[0]
                            elif len(duplicate_inds_valid_3) > 1:
                                #get first index of valid duplicate inds 3
                                chosen_ind = duplicate_inds_valid_3[0]
                            else:
                                #get only remaining index of valid duplicate inds 3
                                chosen_ind = duplicate_inds_valid_3[0]

                    #get bad inds to drop
                    bad_inds = np.array([duplicate_ind for duplicate_ind in duplicate_inds if duplicate_ind != chosen_ind], dtype=np.uint32)

                    #save out indices to remove
                    inds_to_remove = np.append(inds_to_remove, bad_inds)

                    #remove bad inds from overlap inds
                    overlap_inds = np.array([overlap_ind for overlap_ind in overlap_inds if overlap_ind not in bad_inds])


                #resolve overlapping windows
                if len(overlap_inds) > 1:

                    #first screen out not maximum data level flagged data
                    overlap_inds_valid = np.array([overlap_ind for overlap_ind in overlap_inds if not_maximum_data_level_qa_flag not in self.data['qa'][overlap_ind]])

                    #resolved duplication?
                    if len(overlap_inds_valid) == 1:
                        #get only remaining index of valid overlap inds
                        chosen_ind = overlap_inds_valid[0]
                    #else if removing not maximum data level flagged data fails to resolve overlap, try to next resolve duplication using preliminary data
                    else:
                        if len(overlap_inds_valid) == 0:
                            #restore original overlap inds
                            overlap_inds_valid = copy.deepcopy(overlap_inds)
                            
                        #secondly screen out preliminary data
                        overlap_inds_valid_2 = np.array([overlap_ind for overlap_ind in overlap_inds_valid if preliminary_data_flag not in self.data['flag'][overlap_ind]])

                        #resolved duplication?
                        if len(overlap_inds_valid_2) == 1:
                            #get only remaining index of valid overlap inds 2
                            chosen_ind = overlap_inds_valid_2[0]
                        #else if removing preliminary flagged data fails to resolve overlap, try to next resolve duplication using GHOST decreed invalid data
                        else:
                            if len(overlap_inds_valid_2) == 0:
                                #restore original overlap inds
                                overlap_inds_valid_2 = copy.deepcopy(overlap_inds_valid)

                            #thirdly screen out GHOST decreed invalid flagged data
                            overlap_inds_valid_3 = np.array([overlap_ind for overlap_ind in overlap_inds_valid_2 if invalid_GHOST_decreed_data_qa_flag not in self.data['qa'][overlap_ind]])

                            #resolved duplication?
                            if len(overlap_inds_valid_3) == 1:
                                #get only remaining index of valid overlap inds 3
                                chosen_ind = overlap_inds_valid_3[0]
                            #else if removing GHOST decreed flagged data fails to resolve duplication, proceed to attempt to resolve it by keeping finest temporal resolution data
                            else:
                                if len(overlap_inds_valid_3) == 0:
                                    #restore valid overlap inds
                                    overlap_inds_valid_3 = copy.deepcopy(overlap_inds_valid_2)

                                #get temporal resolutions of overlapping time periods
                                overlapping_window_resolutions = self.data['temporal_resolution'][overlap_inds_valid_3]

                                #filter overlap inds, only keeping indices == finest temporal resolution
                                overlap_inds_valid_4 = overlap_inds_valid_3[np.where(overlapping_window_resolutions == np.min(overlapping_window_resolutions))[0]]

                                #the first remaining window is preferentially kept (even if there is > 1 remaining window)
                                #get first index of overlap inds
                                chosen_ind = overlap_inds_valid_4[0]

                    #get bad inds to drop
                    bad_inds = np.array([overlap_ind for overlap_ind in overlap_inds if overlap_ind != chosen_ind], dtype=np.uint32)

                    #save out indices to remove
                    inds_to_remove = np.append(inds_to_remove, bad_inds)


        #if have data indices to remove (either duplicates/or overlaps), do so now
        #if metadata is reported line by line, then also delete metadata columns
        if len(inds_to_remove) > 0:
            self.data = np.delete(self.data, inds_to_remove)
            if len(self.metadata) > 1:
                self.metadata = np.delete(self.metadata, inds_to_remove)


    def convert_instantaneous_temporal_resolutions(self):

        '''convert instantaneous temporal resolutions of 0 minutes to 1 minute for convenience
           of processing through rest of pipeline.
        ''' 

        self.data['temporal_resolution'] = np.array([1 if res == 0 else res for res in self.data['temporal_resolution'][:]], dtype=self.data['temporal_resolution'].dtype)


    def normalise_concentrations(self):
    
        '''normalise measured concentrations in units of mass density/mole fraction to normal temperature and pressure: 
           temperature = 293.15 K, pressure = 1013.25 hPa

           make assumption that concentrations in mole fraction have been first measured as a mass/number density 
           (i.e. network conversion to mole fraction depends on temperature/pressure)

           the conversion is only done when are confident in reported temperature/pressure associated with measurements 
           (i.e. have explicitly stated temperature/pressure associated with measurements OR have known network standard temperature/pressure for component)
        '''

        #get network provided standard temperature/pressures
        network_provided_volume_standard_temperatures = self.metadata['network_provided_volume_standard_temperature']
        network_provided_volume_standard_pressures = self.metadata['network_provided_volume_standard_pressure']
        update_metadata_stamps = self.metadata['meta_update_stamp']

        #get standard measured unit quantity
        if self.parameter_details['standard_units'] != 'unitless':
            conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
            measured_unit_quantity = conv_obj.output_represented_quantity

        #iterate through pairs of standard temperature/pressures
        for ii in range(len(update_metadata_stamps)):
            current_volume_standard_temperature = network_provided_volume_standard_temperatures[ii] 
            current_volume_standard_pressure = network_provided_volume_standard_pressures[ii] 
 
            #have standard temperature AND pressure then normalise concentrations?
            if (not pd.isnull(current_volume_standard_temperature)) & (not pd.isnull(current_volume_standard_pressure)):
                
                #get relevant measurement indices
                #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period 
                start_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii])
                if ii == (len(update_metadata_stamps)-1):
                    end_ind = len(self.data['time'])
                else:
                    end_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii+1])
                if (start_ind == end_ind) & (start_ind == len(self.data['time'])):
                    continue
                elif start_ind == end_ind: 
                    relevant_inds = np.array([start_ind])
                else:
                    relevant_inds = np.arange(start_ind,end_ind) 

                #standard units are mass density?
                if measured_unit_quantity == 'mass_density':
                    #get conversion factor for normalising concentrations
                    conversion_factor = (current_volume_standard_temperature/293.15) * (1013.25/current_volume_standard_pressure)
                    #do conversion for all relevant measurements
                    self.data[self.parameter_details['bsc_parameter_name']][relevant_inds] = self.data[self.parameter_details['bsc_parameter_name']][relevant_inds] * conversion_factor
                    
                #standard units are mole fraction?
                if measured_unit_quantity == 'mole_fraction':
                    #get conversion factor for normalising concentrations
                    conversion_factor = (293.15/current_volume_standard_temperature) * (current_volume_standard_pressure/1013.25)
                    #do conversion for all relevant measurements
                    self.data[self.parameter_details['bsc_parameter_name']][relevant_inds] = self.data[self.parameter_details['bsc_parameter_name']][relevant_inds] * conversion_factor