import copy
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd
import json

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseUKAIRFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current station reference
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        # self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        # create_GHOST_stage_logfile(0,self.run_details)

        #if stage has completed successfully, with some valid data, write a valid process log file
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #else, if no data has been written for station write an error log file to inform station is not continuing in the pipeline
        elif not self.written_data:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get specific data files for current station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)

        #load external metadata into memory 
        with open(self.run_details['data_root']+'UK_AIR/metadata/{}/processed/UK_AIR_META.json'.format(self.run_details['ghost_version']), 'r', encoding='utf-8') as f:
            self.external_metadata = json.loads(f.read())

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        #set flag to know when have written data for station    
        self.written_data = False

        return data_files

    def parse_file(self, data_file):

        '''Function to process a specific data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 

        #create object for parsing UK AIR file, and read file
        parsing_obj = ParseUKAIRFile(data_file, self.run_details, self.parameter_details)
        parsing_obj.read_data()

        # check if there is data for given station and parameter
        if parsing_obj.parsed_data == {}:
            self.run_details['log_file_str'] += 'NO DATA FOR {} IN {}. SKIPPING.\n'.format(self.run_details['parameter'], data_file)
            

        #self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_name)

        #if no valid data then continue to next station reference
        #if len(parsing_obj.parsed_data['start_dt']) == 0:
        #    exit()

        #get cut data variables for station
        # start_dt = parsing_obj.parsed_data['start_dt'][unique_station_name]
        # end_dt = parsing_obj.parsed_data['end_dt'][unique_station_name]
        # measurements = parsing_obj.parsed_data['measurements'][unique_station_name]
        # units = parsing_obj.parsed_data['units'][unique_station_name]
        # flags = parsing_obj.parsed_data['flags'][unique_station_name]
        # methods = parsing_obj.parsed_data['methods'][unique_station_name]

        # correct sampling points so they match metadata
        if len(parsing_obj.parsed_metadata['SamplingPoint']) == 1:
            sampling_point = 'int_'+parsing_obj.parsed_metadata['SamplingPoint']
        elif len(parsing_obj.parsed_metadata['SamplingPoint']) == 2:
            sampling_point = 'nt_'+parsing_obj.parsed_metadata['SamplingPoint']
        elif len(parsing_obj.parsed_metadata['SamplingPoint']) == 3:
            sampling_point = 't_'+parsing_obj.parsed_metadata['SamplingPoint']
        else:
            sampling_point = parsing_obj.parsed_metadata['SamplingPoint']

        #cut external metadata for sampling point
        station_metadata = self.external_metadata[sampling_point]
        # if metadata is cannot be matched, continue to next station
        if len(station_metadata) == 0:
            self.run_details['log_file_str'] += 'METADATA CANNOT BE MATCHED FOR STATION: {}. SKIPPING.\n'.format(self.station_reference)
            exit()

        #create site instances of metadata/data dictionaries
        site_data =     get_standard_data(self.parameter_details)
        site_metadata = get_standard_metadata(self.parameter_details)

        #------------------------------------------------------------------------#
        #PARSE METADATA
        #------------------------------------------------------------------------#

        #GHOST version
        site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

        #initialise string which will store all GHOST process warnings as empty            
        site_metadata['process_warnings']['value'].append('')

        #write known metadata  
        site_metadata['network']['value'].append(self.run_details['network'])
        site_metadata['data_level']['value'].append('none')   
        site_metadata['contact_institution']['value'].append('United Kingdom Department for Environment Food & Rural Affairs')      
        site_metadata['contact_email_address']['value'].append('enquiries@environment-agency.gov.uk')    
        site_metadata['data_licence']['value'].append('Public domain: https://uk-air.defra.gov.uk/about-these-pages')    

        #assume datum information
        site_metadata['ellipsoid']['value'].append('WGS 84') 
        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
        site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
        site_metadata['projection']['value'].append('geographic')    

        #site metadata
        unique_station_reference = station_metadata['Station Code']['values'][-1]
        site_metadata['longitude']['value'].append(station_metadata['Longitude']['values'][-1])
        site_metadata['latitude']['value'].append(station_metadata['Latitude']['values'][-1])
        site_metadata['altitude']['value'].append(station_metadata['Altitude']['values'][-1])
        site_metadata['station_name']['value'].append(station_metadata['Site name']['values'][-1])
        site_metadata['sampling_height']['value'].append(station_metadata['Inlet Height (m)']['values'][-1])

        #associated networks
        # associated_network_string = ''
        # eea_reference = station_meta['EU Site ID']
        # if eea_reference.strip() != '':
        #     associated_network_string = 'EEA AIRBASE:{}'.format(eea_reference)
        # ebas_reference = station_meta['EMEP Site ID']
        # if ebas_reference.strip() != '':
        #     ebas_string = 'EBAS:{}'.format(ebas_reference)
        #     if associated_network_string == '':
        #         associated_network_string = copy.deepcopy(ebas_string)
        #     else:
        #         associated_network_string = '{};{}'.format(associated_network_string, ebas_string)
        # site_metadata['associated_networks']['value'].append(associated_network_string)

        #station description
        # if station_meta['Site Description'].strip() != '':
        #     station_description = 'Station description: {}'.format(station_meta['Site Description'])
        #     site_metadata['network_miscellaneous_details']['value'].append(station_description)

        #save reported units
        #convert raw reported units given to standard name
        reported_raw_units = parsing_obj.parsed_metadata['unit'].replace('.', ' ')
        print('reported units', data_file, reported_raw_units)

        conv_obj = unit_converter.convert_units(reported_raw_units, reported_raw_units, 1, measured_species = self.parameter_details['chemical_formula'])
        site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

        #parse station classifications
        native_classifications = {'area_classification':station_metadata['Environment Type Eionet']['values'][-1], 'station_classification': station_metadata['Station Classification Eionet']['values'][-1]}
        site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

        #process measurement process information
        unique_method = '' # will have to look for methods in metadata ???
        site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(unique_method, site_metadata, self.run_details, self.parameter_details)

        #set measurement qa codes which are True as joined metadata str
        site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
        measurement_qa_flag_codes = []
        for standard_flag_name in measurement_qa_flags.keys():
            if measurement_qa_flags[standard_flag_name]:
                measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
        site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

        #iterate through metadata variables, if any columns have not been appended to add a NaN
        for column_header in list(site_metadata.keys()):
            n_column = len(site_metadata[column_header]['value'])
            #if column is empty, then append NaN
            if n_column < 1:
                site_metadata[column_header]['value'].append(np.NaN)

        #set station_reference (provided network ID + standard measurement method abbreviation)
        site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

        #get data file download timestamp
        site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

        #add data revision timestamp, set as 0001-01-01 00:00 for all files
        data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
        site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

        #------------------------------------------------------------------------#
        #PARSE TIME - always given in GMT/UTC (no conversion neccessary)
        #------------------------------------------------------------------------#   
        #round datetimes to nearest minute
        start_dt_method_unit = [round_to_minute(dt) for dt in parsing_obj.parsed_data['start_dt']]
        end_dt_method_unit = [round_to_minute(dt) for dt in parsing_obj.parsed_data['end_dt']]

        #convert each start measurement datetime to minutes from reference time
        #frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in self.parsed_data['start_dt']]
        frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt_method_unit]
    
        #get timedeltas between start and end of measurement window
        temporal_resolutions = []
        for start, end in zip(parsing_obj.parsed_data['start_dt'],parsing_obj.parsed_data['end_dt']):
            dt_window = end - start
            #convert hours to minutes
            dt_window_minutes = int(dt_window.total_seconds()/60.)
            temporal_resolutions.append(dt_window_minutes)  

        #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
        site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

        #------------------------------------------------------------------------#
        #PARSE QA/FLAGS
        #------------------------------------------------------------------------#
        # validity and verification flags, parse both together

        flags = []
        qa = []

        #get dictionary necessary for mapping of UK AIR file data flags to standard data flag codes
        data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['UK_AIR']  

        # iterate through data flags (can be multiple per measurement)
        for ii,data_flags in enumerate(parsing_obj.parsed_data['Validity']):

            #lists to append qa/data flags per measurement
            flags_per_measurement = []
            qa_per_measurement = []

            #set GHOST decreed validity/network decreed validity variables initially as 'V'
            ghost_decreed_validity = 'V'
            network_decreed_validity = 'V'

            # iterate through flags per measurement
            data_flags_per_meas = {'validity': parsing_obj.parsed_data['Validity'][ii], 'verification': parsing_obj.parsed_data['Verification'][ii]}

            for data_flag in data_flags_per_meas:

                # map data flag/s and append
                standard_data_flag_dict = data_flag_mapping_dictionary[data_flag][data_flags_per_meas[data_flag]]
                #get mapped standard data flags
                standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                #iterate through standard data flags 
                for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                    #check if standard data flag is empty (i.e. unmapped as non-useful)
                    if standard_data_flag_name != '': 
                        #get corresponding data flag code for standard data flag
                        flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                        #append standard data flag
                        if flag_code not in flags_per_measurement:
                            flags_per_measurement.append(flag_code)
                        #determine if GHOST/network decreed valid
                        if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                            ghost_decreed_validity = 'I'
                        if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                            network_decreed_validity = 'I'

            #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
            if ghost_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
            if network_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])

            #append qa_per_measurement, flags_per_measurement to lists to write out
            flags.append(flags_per_measurement)
            qa.append(qa_per_measurement)

        #------------------------------------------------------------------------#
        #PARSE MEASUREMENTS
        #------------------------------------------------------------------------#    
        #measurement temperature and pressure values are never explicitly provided by UK AIR
        #but there is implication that the measurements follow european standards, so assume this:
        #https://uk-air.defra.gov.uk/networks/monitoring-methods?view=eu-standards
        #these are defined as for 293 K temperature, and 1013 hPa for pressure here for all gas-phase compounds: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32008L0050&from=en and here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32004L0107&from=EN 
        #for particulate matter and substances to be analysed in particulate matter (e.g. lead) the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
        #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.

        measurements = []

        standard_network_gas_volume = False
        unknown_gas_volume = False
        if self.parameter_details['matrix'] == 'gas':
            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '293.0'
            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.0'
            standard_network_gas_volume = True
        else:
            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
            unknown_gas_volume = True
            
        #get standard reported units/quantity
        standard_reported_units = conv_obj.output_standard_units
        measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity 

        #set flags for writing QA flags associated with assumptions in unit conversion
        unit_conversion_standard_network = False
        unit_conversion_educated_guess = False

        #check if reported standard units, match required standard units
        if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

            #mass density to mixing ratio?
            if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                #get conversion factor
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume standards from particulates)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                    unit_conversion_standard_network = True
                else:
                    input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                    unit_conversion_educated_guess = True
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')

            #mixing ratio to mass density?
            elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume standards from particulates)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                    unit_conversion_standard_network = True
                else:
                    input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                    unit_conversion_educated_guess = True
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

            elif (measuring_instrument_reported_unit_quantity == 'molarity') & (self.measured_unit_quantity == 'mass_density'):
                input_units = {'molar_mass':'kg mol-1', 'molarity':standard_reported_units}     
                #do conversion 
                input_vals = {'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'molarity':1.0}
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'molarity')

            #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
            else:
                conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

            #process through each measurement and convert value using derived conversion factor
            for n, meas in enumerate(parsing_obj.parsed_data['Value']):
                #only do conversion if measurement value is above 0 and not infinite, and do not have data flag stating the measurement is missing 
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                    measurements.append(float_meas*conv_obj.conversion_factor)
                else:
                    measurements.append(float_meas)

                #append sample gas volume/unit_conversion assumption QA flags if necessary
                if standard_network_gas_volume:
                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                if unknown_gas_volume:
                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                if unit_conversion_standard_network:
                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                if unit_conversion_educated_guess:
                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
    
        #otherwise do nothing to measured/lod values
        else:
            for n, meas in enumerate(parsing_obj.parsed_data['Value']):
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                measurements.append(float_meas)

        #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
        valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
        if len(valid_temporal_resolution_inds) != len(frac_time):
            frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
            temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
            flags = [flags[x] for x in valid_temporal_resolution_inds]
            qa = [qa[x] for x in valid_temporal_resolution_inds]
            measurements = [measurements[x] for x in valid_temporal_resolution_inds]

        #----------------------------------------------------------------------------#
        #WRITE OUT DATA
        #----------------------------------------------------------------------------# 
        site_data['time']['value'] = frac_time
        site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
        site_data['temporal_resolution']['value'] = temporal_resolutions   
        site_data['flag']['value'] = flags
        site_data['qa']['value'] = qa

        #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
        for column_header in list(site_data.keys()):
            n_column = len(site_data[column_header]['value'])
            #if column is empty, then append NaN
            if n_column == 0:
                if site_data[column_header]['data_type'] != np.uint8:
                    site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                else:
                    site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
        #------------------------------------------------------

        #standardise metadata/data formats and write out station data to numpy file
        write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
         #update flag to state that have written data for station (if have done so)
        if write_obj.written:
            self.written_data = True
        else:
            self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)