import copy
import datetime
import itertools
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from geographic_functions import local_timezone_name
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseUSNADPAMNetFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert 1 provided data file to GHOST standard numpy station specific files'''

    def __init__(self, null, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #get data file to parse
        data_file = self.gather_data_files()

        #parse station reference data files
        self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get data files specific to station
        data_file = get_data_files(self.run_details, self.parameter_details)[0]   

        #load external metadata into memory 
        self.external_metadata = np.load(self.run_details['data_root']+'US_NADP_AMNet/metadata/processed/US_NADP_AMNet_META.npy', allow_pickle=True)

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_file

    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 

        #create object for parsing US NADP AMNet annual file, and read file
        parsing_obj = ParseUSNADPAMNetFile(data_file)
        parsing_obj.read_data()

        #get unique station references
        unique_station_references = np.unique(parsing_obj.parsed_data['SiteID']) 

        #iterate through unique station references
        for unique_station_reference in unique_station_references:

            self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_reference)

            #create site instances of metadata/data dictionaries
            site_data =     get_standard_data(self.parameter_details)
            site_metadata = get_standard_metadata(self.parameter_details)

            #isolate variable data specific for station
            valid_inds = parsing_obj.parsed_data['SiteID'] == unique_station_reference
            start_dt = parsing_obj.parsed_data['collStart'][valid_inds]
            end_dt = parsing_obj.parsed_data['collEnd'][valid_inds]
            measurements = parsing_obj.parsed_data['PBM'][valid_inds]
            qr = parsing_obj.parsed_data['PBMVal'][valid_inds]

            #------------------------------------------------------------------------#
            #AVERAGE DUPLICATE MEASUREMENTS
            #------------------------------------------------------------------------#
            #US NADP AMNet measurements are done in duplicates and should be averaged to get a more representative sample measurement
            #before doing any averaging, ensure the QR codes are all 'A' or 'B' (valid data), and are non-NaN
            #for QR codes, take the lowest quality code as been the code for the whole set
            
            valid_start_dt = []
            valid_end_dt = []
            valid_measurements = []
            valid_qr = []

            #get all unique datetime pairs
            pair_dts = np.array([[sdt, edt] for sdt,edt in zip(start_dt, end_dt)])
            unique_pair_dts = [list(pair_dt) for pair_dt in dict.fromkeys(tuple(pair_dt) for pair_dt in pair_dts)]

            #iterate through unique datetime pairs
            for unique_pair_dt in unique_pair_dts:
                #get relevant inds
                relevant_inds = np.where((start_dt == unique_pair_dt[0]) & (end_dt == unique_pair_dt[1]))[0]

                #drop indices associated with blank qr code
                relevant_inds = np.delete(relevant_inds, np.where(qr[relevant_inds] == '')[0])
             
                #if only have travel blank, then continue to next sample ID
                if len(relevant_inds) == 0:
                    continue 

                #get relevant measurements/QR
                relevant_measurements = measurements[relevant_inds]
                relevant_qr = qr[relevant_inds]
                
                # get valid inds to average, where qr codes != 'C', and measurement values are non-NaN
                valid_inds_to_average = [ii for ii, (q, m) in enumerate(zip(relevant_qr,relevant_measurements)) if (q != 'C') & (~np.isnan(m))]
                #if have 0 valid inds to average, then take average from all inds where have non-NaN measurements 
                #if all NaN, then keep all relevant inds   
                have_non_nan_data = True
                if len(valid_inds_to_average) > 0:
                    relevant_inds = relevant_inds[valid_inds_to_average]                
                else:
                    nan_measurements = np.isnan(relevant_measurements)
                    if not np.all(nan_measurements):
                        relevant_inds = relevant_inds[~nan_measurements] 
                    else:
                        have_non_nan_data = False    

                #get rest of vars for relevant inds
                relevant_start_dt = start_dt[relevant_inds]
                relevant_end_dt = end_dt[relevant_inds]
                relevant_measurements = measurements[relevant_inds]
                relevant_qr = qr[relevant_inds]

                #check there is only 1 unique start dt and end dt, if not throw error
                if len(np.unique(relevant_start_dt)) > 1:
                    self.run_details['log_file_str'] += '> 1 START DATE PER UNIQUE SAMPLE, SHOULD NOT BE THE CASE!'
                    create_GHOST_stage_logfile(2,self.run_details) 
                else:
                    valid_start_dt.append(relevant_start_dt[0])
                if len(np.unique(relevant_end_dt)) > 1:
                    self.run_details['log_file_str'] += '> 1 END DATE PER UNIQUE SAMPLE, SHOULD NOT BE THE CASE!'
                    create_GHOST_stage_logfile(2,self.run_details) 
                else:
                    valid_end_dt.append(relevant_end_dt[0])

                #take average measurements (if have non-NaN measurements)
                if have_non_nan_data:
                     valid_measurements.append(np.average(relevant_measurements)) 
                else:
                     valid_measurements.append(np.NaN)

                #append lowest QR code in set
                if 'C' in relevant_qr:
                    valid_qr.append('C')
                elif 'B' in relevant_qr:
                    valid_qr.append('B')
                elif 'A' in relevant_qr:
                    valid_qr.append('A')

            #if have no valid measurements saved after all that continue to next station
            if len(valid_measurements) == 0:
                self.run_details['log_file_str'] += '{} -- NO VALID MEASUREMENTS\n'.format(unique_station_reference)
                continue
            
            #------------------------------------------------------------------------#
            #PARSE METADATA
            #------------------------------------------------------------------------#

            #GHOST version
            site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

            #initialise string which will store all GHOST process warnings as empty            
            site_metadata['process_warnings']['value'].append('')

            #write known metadata  
            site_metadata['network']['value'].append(self.run_details['network'])
            site_metadata['data_level']['value'].append('none')  
            site_metadata['contact_name']['value'].append('David Gay')
            site_metadata['contact_institution']['value'].append('National Atmospheric Deposition Program')      
            site_metadata['contact_email_address']['value'].append('David.Gay@slh.wisc.edu')    
            site_metadata['principal_investigator_name']['value'].append('David Gay')
            site_metadata['principal_investigator_institution']['value'].append('National Atmospheric Deposition Program')      
            site_metadata['principal_investigator_email_address']['value'].append('David.Gay@slh.wisc.edu')  
            site_metadata['data_licence']['value'].append('Public domain: https://edg.epa.gov/EPA_Data_License.html')   
            #assume datum information
            site_metadata['ellipsoid']['value'].append('WGS 84')
            site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
            site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
            site_metadata['projection']['value'].append('geographic')   

            #add reported measurement unit
            #convert raw reported units given to standard name
            reported_units = 'pg/m3' 
            conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
            site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

            #add external metadata
            cut_external_metadata = self.external_metadata[np.where(self.external_metadata['siteId'] == unique_station_reference)[0]][0]
            site_metadata['longitude']['value'].append(cut_external_metadata['longitude'])
            site_metadata['latitude']['value'].append(cut_external_metadata['latitude'])
            site_metadata['altitude']['value'].append(cut_external_metadata['elevation'])
            site_metadata['station_name']['value'].append(cut_external_metadata['siteName'])
           
            #parse station classifications (from external metadata)
            native_classifications = {'area_classification':cut_external_metadata['siteClass']}
            site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)
           
            #process measurement process information
            site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information('', site_metadata, self.run_details, self.parameter_details)
            #set measurement qa codes which are True as joined metadata str
            site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
            measurement_qa_flag_codes = []
            for standard_flag_name in measurement_qa_flags.keys():
                if measurement_qa_flags[standard_flag_name]:
                    measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
            site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

            #iterate through metadata variables, if any columns have not been appended to add a NaN
            for column_header in list(site_metadata.keys()):
                n_column = len(site_metadata[column_header]['value'])
                #if column is empty, then append NaN
                if n_column < 1:
                    site_metadata[column_header]['value'].append(np.NaN)
    
            #set station_reference (provided network ID + standard measurement method abbreviation)
            site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

            #get data file download timestamp
            site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

            #add data revision timestamp, set as 0001-01-01 00:00 for all files
            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
            site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

            #------------------------------------------------------------------------#
            #PARSE TIME
            #------------------------------------------------------------------------#
            #from all information, timezone is believed to be local standard time, i.e. local time without applying daylight savings
            #get timezone for station from longitude and latitude, and then get base offset when daylights saving is not active
            us_tzs = {'Asia/Taipei':-28800,'America/Glace_Bay':14400,'America/Goose_Bay':14400,'America/Halifax':14400,'America/Coral_Harbour':18000,'America/Detroit':18000,'America/Fort_Wayne':18000,'America/Indiana/Indianapolis':18000,'America/Indianapolis':18000,'America/Louisville':18000,'America/New_York':18000,'America/Thunder_Bay':18000,'America/Chicago':21600,'America/Indiana/Knox':21600,'America/Winnipeg':21600,'America/Dawson':25200,'America/Dawson_Creek':25200,'America/Denver':25200,'America/Edmonton':25200,'America/Fort_Nelson':25200,'America/Phoenix':25200,'America/Yellowknife':25200,'America/Los_Angeles':28800,'America/Vancouver':28800,'America/Anchorage':32400,'Pacific/Honolulu':36000}
            local_timezone = local_timezone_name(longitude=float(site_metadata['longitude']['value'][-1]), latitude=float(site_metadata['latitude']['value'][-1]))
            tz_offset = us_tzs[local_timezone]

            #parse start, end and modification datetime for each reported timepoint, and then convert datetimes to UTC
            start_dt = [round_to_minute(dateutil.parser.parse(date)) + datetime.timedelta(seconds=tz_offset) for date in valid_start_dt] 
            end_dt = [round_to_minute(dateutil.parser.parse(date)) + datetime.timedelta(seconds=tz_offset) for date in valid_end_dt] 
        
            #convert each start measurement datetime to minutes from reference time
            frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

            #get timedeltas between start and end of measurement window
            temporal_resolutions = []
            for start, end in zip(start_dt,end_dt):
                dt_window = end - start
                #convert hours to minutes
                dt_window_minutes = int(dt_window.total_seconds()/60.)
                temporal_resolutions.append(dt_window_minutes) 

            #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
            site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

            #------------------------------------------------------------------------#
            #PARSE QA/FLAGS
            #------------------------------------------------------------------------#

            flags = []
            qa = []

            #get dictionary necessary for mapping of US NADP AMNet file data flags to standard data flag codes
            data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['US_NADP_AMNet']                

            #iterate through network qr flags
            for qr_flag in valid_qr:

                #lists to append qa/data flags per measurement
                flags_per_measurement = []
                qa_per_measurement = []
                #set GHOST decreed validity/network decreed validity variables initially as 'V'
                ghost_decreed_validity = 'V'
                network_decreed_validity = 'V'
                             
                # map qr flag and append
                standard_data_flag_dict = data_flag_mapping_dictionary[qr_flag]
                #get mapped standard data flags
                standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                #iterate through standard data flags 
                for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                    #check if standard data flag is empty (i.e. unmapped as non-useful)
                    if standard_data_flag_name != '': 
                        #get corresponding data flag code for standard data flag
                        flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                        #append standard data flag (if not already appended)
                        if flag_code not in flags_per_measurement:
                            flags_per_measurement.append(flag_code)
                        #determine if GHOST/network decreed valid
                        if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                            ghost_decreed_validity = 'I'
                        if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                            network_decreed_validity = 'I'

                #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                if ghost_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                if network_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])

                # add timezone doubt qa flag for each measurement, as get timezone from longitude/latitude
                qa_per_measurement.append(standard_QA_name_to_QA_code['Timezone Doubt'])
        
                #append qa_per_measurement, flags_per_measurement to lists to write out
                flags.append(flags_per_measurement)
                qa.append(qa_per_measurement)

            #------------------------------------------------------------------------#
            #PARSE MEASUREMENTS
            #------------------------------------------------------------------------#

            #measurement values do need converting (given as mass density)
            #measurement temperature and pressure are set as 273.15 K and 1013.25 hPa

            measurements = []

            standard_network_gas_volume = True
            unknown_gas_volume = False
            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '273.15'
            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.25'
            
            #get standard reported units/quantity
            standard_reported_units = conv_obj.output_standard_units
            measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

            #set flags for writing QA flags associated with assumptions in unit conversion
            unit_conversion_standard_network = False
            unit_conversion_educated_guess = False

            #check if reported standard units, match required standard units
            if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                #mass density to mixing ratio?
                if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                    #get conversion factor
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')

                #mixing ratio to mass density?
                elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

                #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                else:
                    conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                #process through each measurement and convert value using derived conversion factor
                for n, meas in enumerate(valid_measurements):
                    #only do conversion if measurement value is above 0 and not infinite
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    
                    if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                        measurements.append(float_meas*conv_obj.conversion_factor)
                    else:
                        measurements.append(float_meas)

                    #append sample gas volume/unit_conversion assumption QA flags if necessary
                    if standard_network_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                    if unknown_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                    if unit_conversion_standard_network:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                    if unit_conversion_educated_guess:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

            #otherwise do nothing to measured values
            else:
                for n, meas in enumerate(valid_measurements):
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    measurements.append(float_meas)

            #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
            valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
            if len(valid_temporal_resolution_inds) != len(frac_time):
                frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                flags = [flags[x] for x in valid_temporal_resolution_inds]
                qa = [qa[x] for x in valid_temporal_resolution_inds]
                measurements = [measurements[x] for x in valid_temporal_resolution_inds]

            #----------------------------------------------------------------------------#
            #WRITE OUT DATA
            #----------------------------------------------------------------------------# 
            site_data['time']['value'] = frac_time
            site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
            site_data['temporal_resolution']['value'] = temporal_resolutions   
            site_data['flag']['value'] = flags
            site_data['qa']['value'] = qa
            #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
            for column_header in list(site_data.keys()):
                n_column = len(site_data[column_header]['value'])
                #if column is empty, then append NaN
                if n_column == 0:
                    if site_data[column_header]['data_type'] != np.uint8:
                        site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                    else:
                        site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
            #------------------------------------------------------

            #standardise metadata/data formats and write out station data to numpy file
            write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
            if write_obj.written == False:
                self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)