import copy
import datetime
import importlib
import sys
import time
import re

import dateutil
import numpy as np
import pandas as pd
import json

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code, standard_sampling_types
from geographic_functions import dms2dd, local_timezone_name
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseINDAAFFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific station into a GHOST standard numpy station specific file'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files of station reference to parse
        data_files = self.gather_data_files()

        #parse station reference data files
        for data_file in data_files:
            self.parse_file(data_file)

        #if stage has completed successfully, with some valid data, write a valid process log file
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #else, if no data has been written for station write an error log file to inform station is not continuing in the pipeline
        elif not self.written_data:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get data files specific to station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)

        #load external metadata into memory 
        with open(self.run_details['data_root']+'INDAAF/metadata/{}/processed/INDAAF_META.json'.format(self.run_details['ghost_version']), 'r', encoding='utf-8') as f:
            self.external_metadata = json.loads(f.read())

        #sort files in date order
        data_file_years = [int(data_file.split('/')[-1].replace('-', '_').split('_')[-1][:-4]) for data_file in data_files]
        data_files = np.array([x for _,x in sorted(zip(data_file_years,data_files))]) 
        data_file_years = sorted(data_file_years)

        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        #set flag to know when have written data for station    
        self.written_data = False

        return data_files


    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 

        #create site instances of metadata/data dictionaries
        site_data =     get_standard_data(self.parameter_details)
        site_metadata = get_standard_metadata(self.parameter_details)

        #create object for parsing INDAAF file, and read file
        parsing_obj = ParseINDAAFFile(data_file, self.run_details, self.parameter_details)
        parsing_obj.read_data()

        station_reference = parsing_obj.parsed_metadata['Station code'] #copy.deepcopy(self.station_reference)

        meas_raw = parsing_obj.parsed_data[self.run_details['parameter']]

        #check if have data for parameter, if not skip file
        if self.run_details['parameter'] not in list(parsing_obj.parsed_data.keys()):
            self.run_details['log_file_str'] += 'NO DATA IN FILE FOR PARAMETER: {}. SKIPPING.\n'.format(data_file)
            return

        #check if have some valid data, if not skip file
        if len(meas_raw) == 0:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {}. SKIPPING.\n'.format(data_file)
            return

        #------------------------------------------------------------------------#
        #PARSE METADATA
        #------------------------------------------------------------------------#
        # prioritise metadata from the data file headers (metadata) over external metadata scraped from the website (external_metadata), except for lat,lon,alt because precision is higher in external metadata

        metadata = parsing_obj.parsed_metadata

        # cut external metadata with unique sampling point ID
        if metadata['Sampling point identifier'] in self.external_metadata: # mainly needed for pm10 concentration files, other files provide metadata in data files
            cut_external_metadata = self.external_metadata[metadata['Sampling point identifier']]
        else:
            cut_external_metadata = {"Sampling Point Identifier": {"values": [''],"update_time": [""]},"Site Name": {"values": [""],"update_time": [""]},"Latitude": {"values":[''],"update_time": [""]},"Longitude":{"values": [""],"update_time": [""]},"Altitude": {"values": [''],"update_time": [""]},"Description": {"values": [""],"update_time": [""]},"Measurement Instrument": {"values": [""],"update_time": [""]},"Instrument Manufacturer": {"values": [""],"update_time": [""]}}

        #GHOST version
        site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

        #initialise string which will store all GHOST process warnings as empty            
        site_metadata['process_warnings']['value'].append('')

        #write known metadata  
        site_metadata['network']['value'].append(self.run_details['network'])
        site_metadata['contact_institution']['value'].append("Laboratoire d'Aerologie, LAERO, CNRS-UT3-IRD, 14 avenue Edouard Belin, , 31400, Toulouse, France")      
        site_metadata['contact_email_address']['value'].append('corinne.galy-lacaux@cnrs.fr')    
        site_metadata['data_licence']['value'].append('Rights reserved to ') # ????????? 
        site_metadata['ellipsoid']['value'].append('WGS 84') 
        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
        site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
        site_metadata['projection']['value'].append('geographic') 
        if 'Data level' in metadata:
            site_metadata['data_level']['value'].append(metadata['Data level'])  
        else:
            site_metadata['data_level']['value'].append('none')  

        # add primary_sampling_instrument_name to site metadata from external metadata
        if 'PREC_ACCUM' in self.run_details['parameter']:
            if ('Instrument type rainfall' in list(metadata.keys())):
                    site_metadata['primary_sampling_instrument_name']['value'].append(metadata['Instrument type rainfall'])
        elif ('_PREC' in self.run_details['parameter']) or (self.parameter_details['matrix'] in ['gas', 'pm', 'pm10', 'pm2.5']):
            if ('Instrument type' in list(metadata.keys())):
                    site_metadata['primary_sampling_instrument_name']['value'].append(metadata['Instrument type'])
        else:
            site_metadata['primary_sampling_instrument_name']['value'].append(cut_external_metadata['Measurement Instrument']['values'][-1])
                
        #site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(rain_sampler, site_metadata, self.run_details, self.parameter_details)
        #site_metadata['primary_sampling_instrument_name']['value'].append(rain_sampler)

        # add sampling height
        if ('Measurement height' in list(metadata.keys())):
            site_metadata['sampling_height']['value'].append(metadata['Measurement height'])

        # add detection limit
        if ('lod' in list(metadata.keys())):
            site_metadata['measuring_instrument_reported_lower_limit_of_detection']['value'].append(metadata['lod'])

        # add units
        if ('unit' in list(metadata.keys())):
            reported_units = metadata['unit']
        else:
            reported_units = ''

        if self.run_details['parameter'] == 'PM10': # parsing does not work because of special character
            reported_units = 'ug m-3'
        elif self.parameter_details['matrix'] == 'gas':
            reported_units = 'ppb'

        #skip file if no units given no parameter
        if reported_units == '':            
            self.run_details['log_file_str'] += 'NO GIVEN UNITS IN FILE FOR PARAMETER: {}. SKIPPING.\n'.format(data_file)
            return

        conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
        site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

        if (cut_external_metadata['Longitude']['values'][-1] != '') and (cut_external_metadata['Longitude']['values'][-1] != None):
            site_metadata['longitude']['value'].append(cut_external_metadata['Longitude']['values'][-1])
        else:
            site_metadata['longitude']['value'].append(metadata['Longitude'])

        if (cut_external_metadata['Latitude']['values'][-1] != '') and (cut_external_metadata['Latitude']['values'][-1] != None):
            site_metadata['latitude']['value'].append(cut_external_metadata['Latitude']['values'][-1])
        else:
            site_metadata['latitude']['value'].append(metadata['Latitude'])

        if (cut_external_metadata['Altitude']['values'][-1] != '') and (cut_external_metadata['Altitude']['values'][-1] != None):
            site_metadata['altitude']['value'].append(cut_external_metadata['Altitude']['values'][-1])
        else:
            site_metadata['altitude']['value'].append(metadata['Altitude'])

        if (cut_external_metadata['Site Name']['values'][-1] != '') and (cut_external_metadata['Site Name']['values'][-1] != None):
            site_metadata['station_name']['value'].append(cut_external_metadata['Site Name']['values'][-1])
        else:
            site_metadata['station_name']['value'].append(metadata['Station code'])

        #parse station classifications (from external metadata)
        if ('Land use' in metadata) and ('Station setting' in metadata):
            native_classifications = {'area_classification': metadata['Station setting'], 'land_use': metadata['Land use']}
        else:
            # premap description in external metadata to already existing categories
            description = cut_external_metadata['Description']['values'][-1].lower()
            if ('shrubs' in description) and ('open' in description):
                land_use = 'deciduousshrubs'
                area_classification = ''
            elif 'urban' in description:
                area_classification = 'urban'
                land_use = ''
            elif 'mosaic forest' in description:
                area_classification = 'rural'
                land_use = 'galleryforestmosaic'
            elif 'croplands' in description:
                area_classification = 'rural'
                land_use = ''
            elif 'cultivated shrubland' in description:
                area_classification = 'rural'
                land_use = 'deciduousshrublandwithsparsetrees'
            elif 'cultivated grass' in description:
                area_classification = 'rural'
                land_use = 'sahelianecoclimaticzone,opengrasslandwithsparseshrubandtrees'
            elif 'open grass' in description:
                area_classification = 'rural'
                land_use = 'sahelianecoclimaticzone,opengrasslandwithsparseshrubandtrees'

            native_classifications = {'area_classification': area_classification, 'land_use': land_use}

        site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

        #process measurement process information 
        if 'PREC_ACCUM' in self.run_details['parameter']:
            measurement_information = metadata['Measurement technique rainfall']
        elif ('_PREC' in self.run_details['parameter']) or (self.parameter_details['matrix'] in ['gas', 'pm', 'pm10', 'pm2.5']):
            measurement_information = metadata['Measurement technique']+'_'+metadata['Instrument manufacturer']+'_'+metadata['Instrument']
        else:
            measurement_information = cut_external_metadata['Measurement Instrument']['values'][-1]+'_'+cut_external_metadata['Instrument Manufacturer']['values'][-1]

        site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(measurement_information, site_metadata, self.run_details, self.parameter_details)
        #set measurement qa codes which are True as joined metadata str
        site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
        measurement_qa_flag_codes = []
        for standard_flag_name in measurement_qa_flags.keys():
            if measurement_qa_flags[standard_flag_name]:
                measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
        site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

        #iterate through metadata variables, if any columns have not been appended to add a NaN
        for column_header in list(site_metadata.keys()):
            n_column = len(site_metadata[column_header]['value'])
            #if column is empty, then append NaN
            if n_column < 1:
                site_metadata[column_header]['value'].append(np.NaN)

        #set station_reference (provided network ID + standard measurement method abbreviation)
        site_metadata['station_reference']['value'][-1] = '{}_{}'.format(station_reference,standard_method_abbreviation) 

        #get data file download timestamp
        site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

        #add data revision timestamp, set as 0001-01-01 00:00 for all files
        data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
        site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

        #------------------------------------------------------------------------#
        #PARSE TIME
        #------------------------------------------------------------------------#

        #get local timezone offset from UTC
        local_timezone = local_timezone_name(longitude=float(site_metadata['longitude']['value'][-1].replace(',', '.')), latitude=float(site_metadata['latitude']['value'][-1].replace(',', '.')))
        tzinfo = dateutil.tz.gettz(local_timezone)        

        #convert timezone to UTC
        start_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in parsing_obj.parsed_data['start_dt']]
        end_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in parsing_obj.parsed_data['end_dt']]

        #convert each start measurement datetime to minutes from reference time
        frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]
            
        #get timedeltas between start and end of measurement window
        temporal_resolutions = []
        for start, end in zip(start_dt,end_dt):
            dt_window = end - start
            #convert hours to minutes
            dt_window_minutes = int(dt_window.total_seconds()/60.)
            temporal_resolutions.append(dt_window_minutes)  

        #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
        site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

        #------------------------------------------------------------------------#
        #PARSE MEASUREMENTS
        #------------------------------------------------------------------------#

        #measurement values may need converting (i.e. mass density to mixing ratio)
        #Precipitation concentrations need conversion from ueq l-1 to mg l-1, but no temperature/pressure normalization -> set network provided volume standard pressure/temperature to NaN
        # precip amount do not need converting
        
        measurements = []
        #initialise qa as all having timezone doubt qa flag, as get timezone from longitude/latitude
        qa = [[standard_QA_name_to_QA_code['Timezone Doubt']] for i in range(len(frac_time))]

        site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
        site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN

        #get standard reported units/quantity
        standard_reported_units = conv_obj.output_standard_units
        measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

        #set flags for writing QA flags associated with assumptions in unit conversion
        unit_conversion_standard_network = False
        unit_conversion_educated_guess = False
        standard_network_gas_volume = False
        unknown_gas_volume = False

        #check if reported standard units, match required standard units
        if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

            #mass density to mixing ratio?
            if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                #get conversion factor
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume network standards)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                    unit_conversion_standard_network = True
                else:
                    input_vals = {'temperature':'293.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                    unit_conversion_educated_guess = True
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')

            #mixing ratio to mass density?
            elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume network standards)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                    unit_conversion_standard_network = True
                else:
                    input_vals = {'temperature':'293.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                    unit_conversion_educated_guess = True
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

            elif (measuring_instrument_reported_unit_quantity == 'molarity') & (self.measured_unit_quantity == 'mass_density'):
                input_units = {'molar_mass':'kg mol-1', 'molarity':standard_reported_units}     
                #do conversion 
                input_vals = {'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'molarity':1.0}
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'molarity')

            #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
            else:
                conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

            #process through each measurement and convert value using derived conversion factor
            for n, meas in enumerate(meas_raw):
                #only do conversion if measurement value is above 0 and not infinite
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                
                if (float_meas > 0.0) & (np.isfinite(float_meas)):
                    measurements.append(float_meas*conv_obj.conversion_factor)
                else:
                    measurements.append(float_meas)

                #append sample gas volume/unit_conversion assumption QA flags if necessary
                if standard_network_gas_volume:
                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                if unknown_gas_volume:
                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                if unit_conversion_standard_network:
                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                if unit_conversion_educated_guess:
                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

        #otherwise do nothing to measured values
        else:
            for n, meas in enumerate(meas_raw):
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                measurements.append(float_meas)

        #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
        valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
        if len(valid_temporal_resolution_inds) != len(frac_time):
            frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
            temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
            measurements = [measurements[x] for x in valid_temporal_resolution_inds]
            qa = [qa[x] for x in valid_temporal_resolution_inds]

        #----------------------------------------------------------------------------#
        #WRITE OUT DATA
        #----------------------------------------------------------------------------# 
        site_data['time']['value'] = frac_time
        site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
        site_data['temporal_resolution']['value'] = temporal_resolutions   
        #set data flags as all un-flagged
        site_data['flag']['value'] = [[standard_data_flag_name_to_data_flag_code['Un-Flagged Data']] for i in range(len(site_data['time']['value']))]
        site_data['qa']['value'] = qa

        #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
        for column_header in list(site_data.keys()):
            n_column = len(site_data[column_header]['value'])
            #if column is empty, then append NaN
            if n_column == 0:
                if site_data[column_header]['data_type'] != np.uint8:
                    site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                else:
                    site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
        #------------------------------------------------------

        #print(self.station_reference, len(list(site_metadata.keys())), len(list(site_data.keys())))
        #standardise metadata/data formats and write out station data to numpy file
        write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)

        #update flag to state that have written data for station (if have done so)
        if write_obj.written:
            self.written_data = True
        else:
            self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)