import copy
import datetime
import re
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import standard_parameters, get_standard_data, get_standard_metadata, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from geographic_functions import local_timezone_name
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseBJMEMCFile
from standardisation_mapping_functions import map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping, map_network_classifications_to_standard_classifications
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter

class StandardiseNetworkData:

    '''Convert daily provider original files in a month to GHOST standard numpy station specific files'''

    def __init__(self, month, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.month = month
        self.year = self.month[:4]
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files to parse
        data_files = self.gather_data_files()

        #parse all data files, writing standard files out per station
        self.parse_files(data_files)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to read per data level'''

        #load external metadata into memory, from a couple of files
        #order so that if current start of month is <202101 (start date of second file)
        if int(self.month) < 202101: 
            meta_files = [self.run_details['data_root']+'BJMEMC/metadata/processed/BJMEMC_META_20131201.npy',
                          self.run_details['data_root']+'BJMEMC/metadata/processed/BJMEMC_META_20210123.npy']
        else:
            meta_files = [self.run_details['data_root']+'BJMEMC/metadata/processed/BJMEMC_META_20210123.npy',
                          self.run_details['data_root']+'BJMEMC/metadata/processed/BJMEMC_META_20131201.npy']
                          
        self.external_metadata_all = []
        for meta_file in meta_files:
            try:
                self.external_metadata_all.append(np.load(meta_file, allow_pickle=True))
            except:
                pass 

        #get specific data files for current month
        data_files = get_data_files(self.run_details, self.parameter_details, specific_month=self.month)

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity
    
        return data_files


    def parse_files(self, data_files):

        '''Function to parse data files in a month for a data level to GHOST standard numpy station specific files'''

        #iterate through daily data files (for specific data level)
        n_measurements = 0
        for data_file_ii, data_file in enumerate(data_files):
            self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 

            #create object for parsing BJMEMC daily file
            parsing_obj = ParseBJMEMCFile(data_file,self.run_details)
            parsing_obj.read_data()

            # if have no parsed data then skip file 
            if len(parsing_obj.parsed_data) == 0:
                continue
            elif len(parsing_obj.parsed_data['start_dt']) == 0:
                continue

            #define dictionary for storing all parsed data
            if data_file_ii == 0:    
                all_data = copy.deepcopy(parsing_obj.parsed_data)
            else:
                for var in parsing_obj.parsed_data.keys():
                    if var in all_data:
                        all_data[var] = np.append(all_data[var], parsing_obj.parsed_data[var])
                    else:
                        all_data[var] = copy.deepcopy(parsing_obj.parsed_data[var])
                        all_data[var] = np.insert(all_data[var], 0, np.array(['']*n_measurements, dtype=np.str_))

            #iterate n measurements
            n_measurements += len(parsing_obj.parsed_data['start_dt'])

        #get all unique station reference variables 
        unique_station_references = [var for var in all_data.keys() if var not in ['start_dt','end_dt']]

        #if length of measurements for any stations does not equal that of n_measurements, it is because station
        #at some point stopped reporting. Fill to the end to match length.
        for station_reference in unique_station_references:
            if len(all_data[station_reference]) != n_measurements:
                diff_n = n_measurements - len(all_data[station_reference])
                all_data[station_reference] = np.append(all_data[station_reference], np.array(['']*diff_n, dtype=np.str_))   

        #iterate through unique stations references in data
        for station_reference in unique_station_references:

            self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(station_reference)

            #create site instances of metadata/data dictionaries
            site_data =     get_standard_data(self.parameter_details)
            site_metadata = get_standard_metadata(self.parameter_details)

            #------------------------------------------------------------------------#
            #PARSE METADATA
            #------------------------------------------------------------------------#

            #GHOST version
            site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

            #initialise string which will store all GHOST process warnings as empty            
            site_metadata['process_warnings']['value'].append('')

            #write known metadata  
            site_metadata['network']['value'].append(self.run_details['network'])
            site_metadata['data_level']['value'].append('none')  
            site_metadata['contact_institution']['value'].append('BJMEMC')      
            site_metadata['contact_email_address']['value'].append('jczx@bjmemc.com.cn')    
            site_metadata['data_licence']['value'].append('Rights reserved to the Beijing Municipal Ecological and Environmental Monitoring Center: http://www.bjmemc.com.cn')

            #datum information (assume WGS84)
            site_metadata['ellipsoid']['value'].append('WGS 84') 
            site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
            site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
            site_metadata['projection']['value'].append('geographic') 

            #add reported measurement unit
            #convert raw reported units given to standard name
            if self.run_details['parameter'] == 'CO':
                reported_units = 'mg/m3'
            else:
                reported_units = 'ug/m3'

            #convert units to standard units
            conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
            site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

            #add external metadata   
            #try making join with metadata files iteratively
            #if any fields are empty strings continue to check other annual metadata files to fill gaps
            parsed_external_metadata = {}
            for external_metadata in self.external_metadata_all:
                cut_external_metadata = external_metadata[np.where(external_metadata['station_name'] == station_reference)[0]]
                
                #take first metadata preferentially (if not specific join)
                if len(cut_external_metadata) >= 1:
                    cut_external_metadata = cut_external_metadata[0]
                    #iterate through metadata variables
                    for meta_var in cut_external_metadata.dtype.names:
                        #new meta variable
                        if meta_var not in parsed_external_metadata:
                            parsed_external_metadata[meta_var] = cut_external_metadata[meta_var]
                        #current empty str for variable and new variable is non-empty
                        elif (parsed_external_metadata[meta_var] in ['','-']) & (cut_external_metadata not in ['','-']):
                            parsed_external_metadata[meta_var] = cut_external_metadata[meta_var]

                #if have made no join with station name, then try station reference
                else:
                    if 'station_reference' in external_metadata.dtype.names:
                        cut_external_metadata = external_metadata[np.where(external_metadata['station_reference'] == station_reference)[0]]
                    
                        #take first metadata preferentially (if not specific join)
                        if len(cut_external_metadata) >= 1:
                            cut_external_metadata = cut_external_metadata[0]
                            #iterate through metadata variables
                            for meta_var in cut_external_metadata.dtype.names:
                                #new meta variable
                                if meta_var not in parsed_external_metadata:
                                    parsed_external_metadata[meta_var] = cut_external_metadata[meta_var]
                                #current empty str for variable and new variable is non-empty
                                elif (parsed_external_metadata[meta_var] in ['','-']) & (cut_external_metadata not in ['','-']):
                                    parsed_external_metadata[meta_var] = cut_external_metadata[meta_var]

            #have some parsed metadata, if not then skip station
            if len(parsed_external_metadata) == 0:
                self.run_details['log_file_str'] += 'COULD NOT MAKE METADATA JOIN FOR STATION: {}. SKIPPING.\n'.format(station_reference)
                continue

            #add external metadata
            site_metadata['longitude']['value'].append(parsed_external_metadata['longitude'])
            site_metadata['latitude']['value'].append(parsed_external_metadata['latitude'])
            site_metadata['station_name']['value'].append(parsed_external_metadata['station_name'])

            #check longitude/latitude are numerical, if not then skip
            try:
                float_lon = float(site_metadata['longitude']['value'][-1])
                float_lat = float(site_metadata['latitude']['value'][-1])
            except:
                self.run_details['log_file_str'] += 'NON-NUMERICAL LONGITUDE/LATITUDE FOR STATION: {}. SKIPPING.\n'.format(station_reference)
                continue

            #process classification information
            if 'classification' in parsed_external_metadata:
                native_classifications = {'area_classification':parsed_external_metadata['classification']}
                site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

            # get standard measurement information
            site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information('', site_metadata, self.run_details, self.parameter_details)
            #set measurement qa codes which are True as joined metadata str
            site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
            measurement_qa_flag_codes = []
            for standard_flag_name in measurement_qa_flags.keys():
                if measurement_qa_flags[standard_flag_name]:
                    measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
            site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

            #iterate through metadata variables, if any columns have not been appended to add a NaN
            for column_header in list(site_metadata.keys()):
                n_column = len(site_metadata[column_header]['value'])
                #if column is empty, then append NaN
                if n_column < 1:
                    site_metadata[column_header]['value'].append(np.NaN)

            #set station_reference (ghost created network ID + standard measurement method abbreviation)
            site_metadata['station_reference']['value'][-1] = '{}_{}'.format(parsed_external_metadata['ghost_station_reference'],standard_method_abbreviation) 

            #get data file download timestamp
            site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

            #add data revision timestamp, set as 0001-01-01 00:00 for all files
            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
            site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

            #------------------------------------------------------------------------#
            #PARSE TIME
            #------------------------------------------------------------------------#
        
            #get local timezone offset from UTC
            local_timezone = local_timezone_name(longitude=float(site_metadata['longitude']['value'][-1]), latitude=float(site_metadata['latitude']['value'][-1]))
            tzinfo = dateutil.tz.gettz(local_timezone)        

            #convert timezone to UTC
            start_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in all_data['start_dt']]
            end_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in all_data['end_dt']]

            #convert each start measurement datetime to minutes from reference time
            frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]
                
            #get timedeltas between start and end of measurement window
            temporal_resolutions = []
            for start, end in zip(start_dt,end_dt):
                dt_window = end - start
                #convert hours to minutes
                dt_window_minutes = int(dt_window.total_seconds()/60.)
                temporal_resolutions.append(dt_window_minutes)  

            #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
            site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

            #------------------------------------------------------------------------#
            #PARSE MEASUREMENTS
            #------------------------------------------------------------------------#
            
            #measurement temperature and pressure values are never provided so are always assumed to be the Chinese standard pressure and temperatures
            #these are defined as for 298.15 K temperature, and 1013.25 hPa for pressure here for all gaseous compounds, defined in supplementary material: revised_standards.pdf
            #for PM10/PM2.5 and lead the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.

            measurements = []
            #initialise qa as all having timezone doubt qa flag, as get timezone from longitude/latitude
            qa = [[standard_QA_name_to_QA_code['Timezone Doubt']] for i in range(len(frac_time))]

            standard_network_gas_volume = False
            unknown_gas_volume = False

            if (self.parameter_details['matrix'] == 'pm10') or (self.parameter_details['matrix'] == 'pm2.5'):
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
                unknown_gas_volume = True
            else:
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '298.15'
                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.25'
                standard_network_gas_volume = True

            #get standard reported units/quantity
            standard_reported_units = conv_obj.output_standard_units
            measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

            #set flags for writing QA flags associated with assumptions in unit conversion
            unit_conversion_standard_network = False
            unit_conversion_educated_guess = False

            #check if reported standard units, match required standard units
            if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                #mass density to mixing ratio?
                if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                    #get conversion factor
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')

                #mixing ratio to mass density?
                elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

                #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                else:
                    conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                #process through each measurement and convert value using derived conversion factor
                for n, meas in enumerate(all_data[station_reference]):
                    #only do conversion if measurement value is above 0 and not infinite
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    
                    if (float_meas > 0.0) & (np.isfinite(float_meas)):
                        measurements.append(float_meas*conv_obj.conversion_factor)
                    else:
                        measurements.append(float_meas)

                    #append sample gas volume/unit_conversion assumption QA flags if necessary
                    if standard_network_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                    if unknown_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                    if unit_conversion_standard_network:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                    if unit_conversion_educated_guess:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

            #otherwise do nothing to measured values
            else:
                for n, meas in enumerate(all_data[station_reference]):
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    measurements.append(float_meas)

            #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
            valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
            if len(valid_temporal_resolution_inds) != len(frac_time):
                frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                measurements = [measurements[x] for x in valid_temporal_resolution_inds]
                qa = [qa[x] for x in valid_temporal_resolution_inds]

            #----------------------------------------------------------------------------#
            #WRITE OUT DATA
            #----------------------------------------------------------------------------#
            site_data['time']['value'] = frac_time
            site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
            site_data['temporal_resolution']['value'] = temporal_resolutions   
            #set data flags as all un-flagged
            site_data['flag']['value'] = [[standard_data_flag_name_to_data_flag_code['Un-Flagged Data']] for i in range(len(site_data['time']['value']))]
            site_data['qa']['value'] = qa
            #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
            for column_header in list(site_data.keys()):
                n_column = len(site_data[column_header]['value'])
                #if column is empty, then append NaN
                if n_column == 0:
                     if site_data[column_header]['data_type'] != np.uint8:
                         site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                     else:
                         site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
            #------------------------------------------------------
 
            #standardise metadata/data formats and write out station data to numpy file
            write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
            if write_obj.written == False:
                self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(station_reference)