import copy
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from geographic_functions import dms2dd, local_timezone_name
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseJAPANNIESFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter

#define necessary dicts for mapping metadata
unit_dict = {'PPB':'nmol mol-1','PPM':'umol mol-1','PPHM':'nmol mol-1','PPBC':'nmolC mol-1','PPMC':'umolC mol-1','MG/M3':'mg m-3','MG/HM3':'ug m-3','UG/M3':'ug m-3'}

station_classification_codes = {'1':'general_environmental_atmospheric_measurement_station',
                                '1_0':'general_environmental_atmospheric_measurement_station:environmental_standard_application_bureau', 
                                '1_1':'general_environmental_atmospheric_measurement_station:environmental_standard_exemption_bureau',
                                '2':'vehicle_exhaust_gas_measurement_station', 
                                '2_0':'vehicle_exhaust_gas_measurement_station:roadside_station',  
                                '2_1':'vehicle_exhaust_gas_measurement_station:roadway_station', 
                                '5':'meteorological_3d_station', 
                                '6':'descent_and_dust_station', 
                                '9':'other'}

land_use_codes = {'1':'residential','2':'commercial','3':'semi-construction','4':'engineering','5':'technical_college','6':'unknown','7':'scenic','8':'harbor','9':'other','':''}

measurement_method_codes = {'SO2':        {'0':'','1':'conventional_solution_conductivity','2':'high_sensitivity_solution_conductivity','3':'uv_fluorescence','':''},
                            'NOX':        {'1':'absorptiometry', '2':'chemiluminescence','9':'','':''},
                            'O3':         {'1':'absorptiometry', '2':'coulometry', '3':'uv_absorption','4':'chemiluminescence','':''},
                            'CO':         {'0':'non-dispersive_infrared_absorption','1':'non-dispersive_infrared_absorption','':'non-dispersive_infrared_absorption'},
                            'hydrocarbon':{'1':'direct', '2':'difference_with_propane_conversion', '3':'difference_with_no_propane_conversion', '4':'total_hydrocarbon_automatic_measuring_machine_with_methane_conversion', '5':'total_hydrocarbon_automatic_measuring_machine_with_propane_conversion', '6':'total_hydrocarbon_automatic_measuring_machine_with_ethylene_conversion','':''},
                            'PM10':       {'1':'light_scattering', '2':'piezoelectric_balance_method', '3':'beta-ray_absorption','':''},
                            'PM2.5':      {'11':'beta-ray_absorption', '12':'filter_vibration', '13':'light_scattering', '14':'beta-ray_absorption_light_scattering_hybrid','':''}
                            }

measuring_instrument_codes = {'1':'frm-compliant_sampler', '11':'kimoto_pm-712', '12':'kimoto_pm-717', '13':'toa_dkk_fpm-377', '14':'horiba_apda-3750A', '15':'thermo_fh62c14', '16':'thermo_sharp_5030', '17':'environnement_sa_mp101m', '18':'thermo_5014i', '9999':'other','':''} 


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, year, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.year = year
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current year
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per year'''

        #get specific data files for current year
        data_files = get_data_files(self.run_details, self.parameter_details, specific_year=self.year)
   
        #load external metadata into memory 
        #load metadata from +-5 years (if possible) in case can not make match with current year meta  
        self.external_metadata_all = []
        self.external_metadata_all.append(np.load(self.run_details['data_root']+'JAPAN_NIES/metadata/processed/JAPAN_NIES_META_{}.npy'.format(self.year), allow_pickle=True))
        for prev_year in np.concatenate((np.arange(int(self.year)-1,int(self.year)-6,-1), np.arange(int(self.year)+1,int(self.year)+6))):
            try:
                self.external_metadata_all.append(np.load(self.run_details['data_root']+'JAPAN_NIES/metadata/processed/JAPAN_NIES_META_{}.npy'.format(prev_year), allow_pickle=True))
            except:
                pass 

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_files

    def parse_file(self, data_file):

        '''Function to a specific annual data file to GHOST standard numpy station specific files'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
        
        #determine if hourly/monthly file 
        if '/monthly/' in data_file:
            file_type = 'monthly'
        else:
            file_type = 'hourly'

        #create object for parsing JAPAN NIES annual file, and read file
        parsing_obj = ParseJAPANNIESFile(data_file,file_type)
        parsing_obj.read_data()

        #get unique station references
        unique_station_references = np.unique(parsing_obj.parsed_data['station_reference'])

        #iterate over all unique station references
        for unique_station_reference in unique_station_references:

            self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_reference)

            #get unique JAPAN species names in data
            unique_JAPAN_NIES_species_names = np.unique(parsing_obj.parsed_data['species'])
            #remove unique unique JAPAN species names not relevant for current species
            unique_JAPAN_NIES_species_names = [unique_JAPAN_NIES_species_name for unique_JAPAN_NIES_species_name in unique_JAPAN_NIES_species_names if unique_JAPAN_NIES_species_name in self.parameter_details['japan_nies_name']]

            #iterate through unique JAPAN NIES species names
            for unique_JAPAN_NIES_species_name in unique_JAPAN_NIES_species_names: 

                #cut read data to just that associated with unique station reference & unique_JAPAN_NIES_species_name being processed
                valid_inds = (parsing_obj.parsed_data['station_reference'] == unique_station_reference) & (parsing_obj.parsed_data['species'] == unique_JAPAN_NIES_species_name)

                #get cut data variables for station
                start_dt = parsing_obj.parsed_data['start_dt'][valid_inds]
                end_dt = parsing_obj.parsed_data['end_dt'][valid_inds]
                cut_measurements = parsing_obj.parsed_data['measurement'][valid_inds]
                raw_units = parsing_obj.parsed_data['units'][valid_inds]
                if file_type == 'monthly':
                    measured_hours_percent = parsing_obj.parsed_data['measured_hours_percent'][valid_inds]
                
                #if no valid time then continue to next station reference
                if len(start_dt) == 0:
                    continue

                #create site instances of metadata/data dictionaries
                site_data =     get_standard_data(self.parameter_details)
                site_metadata = get_standard_metadata(self.parameter_details)

                #if more than 1 unique units, throw error as unexpected
                if len(np.unique(raw_units)) > 1:
                    self.run_details['log_file_str'] += 'MULTIPLE UNITS:{}'.format(np.unique(raw_units))
                    create_GHOST_stage_logfile(2,self.run_details) 
                raw_units = raw_units[0]

                #------------------------------------------------------------------------#
                #PARSE METADATA
                #------------------------------------------------------------------------#

                #GHOST version
                site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

                #initialise string which will store all GHOST process warnings as empty            
                site_metadata['process_warnings']['value'].append('')

                #write known metadata  
                site_metadata['network']['value'].append(self.run_details['network'])
                site_metadata['data_level']['value'].append('none')   
                site_metadata['contact_institution']['value'].append('Japan NIES (National Institute for Environmental Studies)')      
                site_metadata['contact_email_address']['value'].append('gis@nies.go.jp')    
                site_metadata['data_licence']['value'].append('Rights reserved to the National Institute for Environmental Studies, Japan: https://www.nies.go.jp/sitepolicy.html') 
                #known datum information
                site_metadata['ellipsoid']['value'].append('WGS 84') 
                site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
                site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                site_metadata['projection']['value'].append('geographic')    

                #add external metadata   
                #try making join with metadata files +- 5 years of file, preferentially current year, then back in time and then forward in time.
                #if any fields are empty strings continue to check other annual metadata files to fill gaps
                parsed_external_metadata = {}
                for external_metadata in self.external_metadata_all:
                    cut_external_metadata = external_metadata[np.where(external_metadata['NIES_area_code'] == unique_station_reference)[0]]
                    #correctly made join    
                    if len(cut_external_metadata) == 1:
                        cut_external_metadata = cut_external_metadata[0]
                        #iterate through metadata variables
                        for meta_var in cut_external_metadata.dtype.names:
                            #new meta variable
                            if meta_var not in parsed_external_metadata:
                                parsed_external_metadata[meta_var] = cut_external_metadata[meta_var]
                            #current empty str for variable and new variable is non-empty
                            elif (parsed_external_metadata[meta_var] == '') & (cut_external_metadata != ''):
                                parsed_external_metadata[meta_var] = cut_external_metadata[meta_var]

                    #non specific join - throw error
                    elif len(cut_external_metadata) == 2:
                        self.run_details['log_file_str'] += 'META FILE NOT SPECIFIC FOR REFERENCE: {}'.format(unique_station_reference)
                        create_GHOST_stage_logfile(2,self.run_details) 

                #have some parsed metadata, if not then skip station
                if len(parsed_external_metadata) == 0:
                    self.run_details['log_file_str'] += 'COULD NOT MAKE METADATA JOIN FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)
                    continue

                #add external metadata
                site_metadata['station_name']['value'].append(parsed_external_metadata['roman_station_name'])
                if (parsed_external_metadata['longitude_degree'] != '') & (parsed_external_metadata['latitude_degree'] != ''):
                    site_metadata['longitude']['value'].append(dms2dd(parsed_external_metadata['longitude_degree'], parsed_external_metadata['longitude_minute'], parsed_external_metadata['longitude_second'], 'E'))
                    site_metadata['latitude']['value'].append(dms2dd(parsed_external_metadata['latitude_degree'], parsed_external_metadata['latitude_minute'], parsed_external_metadata['latitude_second'], 'N'))
                else:
                    self.run_details['log_file_str'] += 'NO LON/LAT FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)
                    continue
                site_metadata['altitude']['value'].append(parsed_external_metadata['altitude(m)'])
                if self.run_details['parameter'] in ['HCl','HF','H2S']:
                    sampling_height = ''
                elif self.run_details['parameter'] == 'HC':
                    sampling_height = parsed_external_metadata['THC_measuring_altitude(m)']
                elif self.run_details['parameter'] == 'PM10':
                    sampling_height = parsed_external_metadata['SPM_measuring_altitude(m)']
                elif self.run_details['parameter'] == 'O3':
                    sampling_height = parsed_external_metadata['OX_measuring_altitude(m)']
                else:
                    sampling_height = parsed_external_metadata['{}_measuring_altitude(m)'.format(self.run_details['parameter'])]
                site_metadata['sampling_height']['value'].append(sampling_height)

                #non-standard comments
                non_standard_fields_to_parse = {'Address':'roman_street_address','Building Name':'roman_building_name'}
                misc_str = ''
                for field, meta_field in non_standard_fields_to_parse.items():
                    if parsed_external_metadata[meta_field] != '':
                        misc_str += '{}:{};'.format(field, parsed_external_metadata[meta_field])
                if misc_str != '':
                    site_metadata['network_miscellaneous_details']['value'].append(misc_str[:-1])

                #convert raw reported units given to standard name
                #some units also require conversion of units by a factor of 10
                if '0.1' in raw_units:
                    raw_units = raw_units[3:].strip()
                    cut_measurements = cut_measurements / 10.0
                elif '10' in raw_units:
                    raw_units = raw_units[2:].strip()
                    cut_measurements = cut_measurements * 10.0
                elif raw_units == 'MG/HM3':
                    cut_measurements = cut_measurements * 10.0
                elif raw_units == 'PPHM':
                    cut_measurements = cut_measurements * 10.0
                reported_units = unit_dict[raw_units]
                conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
                site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

                #parse station classifications (from external metadata)
                class_num = int(unique_station_reference[5:7])
                if class_num < 51:
                    class_num_code = '1'
                elif class_num < 81:
                    class_num_code = '2'
                else:
                    class_num_code = '6'
                class_code = parsed_external_metadata['station_classification']
                type_code = parsed_external_metadata['station_type']
                #if class_code is empty then get it from station_reference code 
                if class_code == '':
                    class_code = copy.deepcopy(class_num_code)
                #double check for clearly incorrect information
                #i.e. class code of >= 5, but with given type code (should not happen!) 
                if (int(class_code) >= 5) & (type_code != ''):
                    if int(class_num_code) >= 5:
                        type_code = ''
                    else:    
                        class_code = copy.deepcopy(class_num_code)
                if type_code != '':
                    class_code = '{}_{}'.format(class_code,type_code)
                native_classifications = {'station_classification':station_classification_codes[class_code],'land_use':land_use_codes[parsed_external_metadata['restricted_zone']]}
                site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

                #process measurement process information
                if self.run_details['parameter'] in ['NO','NO2']: 
                    measurement_method = measurement_method_codes['NOX'][parsed_external_metadata['{}_measuring_method'.format(self.run_details['parameter'])]]
                elif self.run_details['parameter'] in ['HCl','HF','H2S']:
                    measurement_method = ''
                elif self.run_details['parameter'] == 'CH4':
                    measurement_method = measurement_method_codes['hydrocarbon'][parsed_external_metadata['CH4_measuring_method']]
                elif self.run_details['parameter'] == 'NMHC':
                    measurement_method = measurement_method_codes['hydrocarbon'][parsed_external_metadata['NMHC_measuring_method']]
                elif self.run_details['parameter'] == 'HC':
                    measurement_method = measurement_method_codes['hydrocarbon'][parsed_external_metadata['THC_measuring_method']]
                elif self.run_details['parameter'] == 'O3':
                    measurement_method = measurement_method_codes['O3'][parsed_external_metadata['OX_measuring_method']]
                elif self.run_details['parameter'] == 'PM10':
                    measurement_method = measurement_method_codes['PM10'][parsed_external_metadata['SPM_measuring_method']] 
                elif self.run_details['parameter'] == 'PM2.5':
                    #for PM2.5 the method is implied in the unique_JAPAN_NIES_species_name, so set it if measurement method is empty
                    if unique_JAPAN_NIES_species_name == 'PM25':
                        fill_method = 'beta-ray_absorption'
                    elif unique_JAPAN_NIES_species_name == 'PMFL':
                        fill_method = 'filter_vibration'
                    elif unique_JAPAN_NIES_species_name == 'PMBH':
                        fill_method = 'beta-ray_absorption_light_scattering_hybrid'
                    if measurement_method_codes['PM2.5'][parsed_external_metadata['PM2.5_measuring_method']] == '':
                        measurement_method = '{}_{}'.format(fill_method,measuring_instrument_codes[parsed_external_metadata['PM2.5_measuring_equipment']]).strip('_') 
                    else:
                        measurement_method = '{}_{}'.format(measurement_method_codes['PM2.5'][parsed_external_metadata['PM2.5_measuring_method']],measuring_instrument_codes[parsed_external_metadata['PM2.5_measuring_equipment']]).strip('_') 
                else:
                    measurement_method = measurement_method_codes[self.run_details['parameter']][parsed_external_metadata['{}_measuring_method'.format(self.run_details['parameter'])]]
                site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(measurement_method, site_metadata, self.run_details, self.parameter_details)
                #set measurement qa codes which are True as joined metadata str
                site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
                measurement_qa_flag_codes = []
                for standard_flag_name in measurement_qa_flags.keys():
                    if measurement_qa_flags[standard_flag_name]:
                        measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
                site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

                #iterate through metadata variables, if any columns have not been appended to add a NaN
                for column_header in list(site_metadata.keys()):
                    n_column = len(site_metadata[column_header]['value'])
                    #if column is empty, then append NaN
                    if n_column < 1:
                        site_metadata[column_header]['value'].append(np.NaN)
    
                #set station_reference (provided network ID + standard measurement method abbreviation)
                site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

                #get data file download timestamp
                site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

                #add data revision timestamp, set as 0001-01-01 00:00 for all files
                data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
                site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

                #------------------------------------------------------------------------#
                #PARSE TIME
                #------------------------------------------------------------------------#

                #get local timezone offset from UTC
                local_timezone = local_timezone_name(longitude=float(site_metadata['longitude']['value'][-1]), latitude=float(site_metadata['latitude']['value'][-1]))
                tzinfo = dateutil.tz.gettz(local_timezone)        

                #convert timezone to UTC
                start_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in start_dt]
                end_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in end_dt]

                #convert each start measurement datetime to minutes from reference time
                frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]
            
                #get timedeltas between start and end of measurement window
                temporal_resolutions = []
                for start, end in zip(start_dt,end_dt):
                    dt_window = end - start
                    #convert hours to minutes
                    dt_window_minutes = int(dt_window.total_seconds()/60.)
                    temporal_resolutions.append(dt_window_minutes)  

                #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
                site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

                #------------------------------------------------------------------------#
                #PARSE QA/FLAGS
                #------------------------------------------------------------------------#
            
                #no provided data flags
                #for monthly data the amount of measured hours per month is provided, can use this to infer data flags
                flags = []
                qa = []

                #get dictionary necessary for mapping of JAPAN NIES file data flags to standard data flag codes
                data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['JAPAN_NIES']  

                for meas_ii, meas in enumerate(cut_measurements):

                    #lists to append qa/data flags per measurement
                    flags_per_measurement = []
                    qa_per_measurement = []

                    #set GHOST decreed validity/network decreed validity variables initially as 'V'
                    ghost_decreed_validity = 'V'
                    network_decreed_validity = 'V' 

                    #window representativity flags
                    if file_type == 'monthly':
                        if measured_hours_percent[meas_ii] == '':
                            data_flag = ''
                        elif float(measured_hours_percent[meas_ii]) < 90:
                            data_flag = 'MONTHLY_<90'
                        elif float(measured_hours_percent[meas_ii]) < 75:
                            data_flag = 'MONTHLY_<75'
                        elif float(measured_hours_percent[meas_ii]) < 66:
                            data_flag = 'MONTHLY_<66'
                        elif float(measured_hours_percent[meas_ii]) < 50:
                            data_flag = 'MONTHLY_<50'
                        elif float(measured_hours_percent[meas_ii]) < 25:
                            data_flag = 'MONTHLY_<25'   
                        else:
                            data_flag = '' 
                    else:
                        data_flag = ''

                    if data_flag != '':
                        # map data flag/s and append
                        standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                        #get mapped standard data flags
                        standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                        #iterate through standard data flags 
                        for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                            #check if standard data flag is empty (i.e. unmapped as non-useful)
                            if standard_data_flag_name != '': 
                                #get corresponding data flag code for standard data flag
                                flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                #append standard data flag (if not already in list)
                                if flag_code not in flags_per_measurement:
                                    flags_per_measurement.append(flag_code)
                                #determine if GHOST/network decreed valid
                                if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                    ghost_decreed_validity = 'I'
                                if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                    network_decreed_validity = 'I'

                    #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                    if ghost_decreed_validity == 'I':
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                    if network_decreed_validity == 'I':
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])

                    #add 'Un-Flagged Data' flag for all measurements as all flags artificially created 
                    flags_per_measurement.append(standard_data_flag_name_to_data_flag_code['Un-Flagged Data'])

                    # add timezone doubt qa flag for each measurement, as get timezone from longitude/latitude
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Timezone Doubt'])

                    #append qa_per_measurement, flags_per_measurement to lists to write out
                    flags.append(flags_per_measurement)
                    qa.append(qa_per_measurement)

                #------------------------------------------------------------------------#
                #PARSE MEASUREMENTS
                #------------------------------------------------------------------------#
                #measurement temperature and pressure values are never explicitly provided by JAPAN_NIES, and there are no known network standards
                #however through the documentation there is constant reference with comparison with EPA standards and references, so it is assumed it follows US standards.
                #these are defined as for 298.15 K temperature, and 1013.25 hPa for pressure here for all compounds, except PM2.5 and lead: https://www.govinfo.gov/content/pkg/CFR-2011-title40-vol2/pdf/CFR-2011-title40-vol2-chapI.pdf
                #for PM2.5 and lead the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
                #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.
    
                measurements = []

                standard_network_gas_volume = False
                unknown_gas_volume = True
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN

                #get standard reported units/quantity
                standard_reported_units = conv_obj.output_standard_units
                measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

                #set flags for writing QA flags associated with assumptions in unit conversion
                unit_conversion_standard_network = False
                unit_conversion_educated_guess = False

                #check if reported standard units, match required standard units
                if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                    #mass density to mixing ratio?
                    if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                        #get conversion factor
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')

                    #mixing ratio to mass density?
                    elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

                    #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                    else:
                        conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                    #process through each measurement and convert value using derived conversion factor
                    for n, meas in enumerate(cut_measurements):
                        #only do conversion if measurement value is above 0 and not infinite
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                    
                        if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                            measurements.append(float_meas*conv_obj.conversion_factor)
                        else:
                            measurements.append(float_meas)

                        #append sample gas volume/unit_conversion assumption QA flags if necessary
                        if standard_network_gas_volume:
                            qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                        if unknown_gas_volume:
                            qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                        if unit_conversion_standard_network:
                            qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                        if unit_conversion_educated_guess:
                            qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

                #otherwise do nothing to measured values
                else:
                    for n, meas in enumerate(cut_measurements):
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                        measurements.append(float_meas)

                #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
                valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
                if len(valid_temporal_resolution_inds) != len(frac_time):
                    frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                    temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                    flags = [flags[x] for x in valid_temporal_resolution_inds]
                    qa = [qa[x] for x in valid_temporal_resolution_inds]
                    measurements = [measurements[x] for x in valid_temporal_resolution_inds]

                #----------------------------------------------------------------------------#
                #WRITE OUT DATA
                #----------------------------------------------------------------------------# 
                site_data['time']['value'] = frac_time
                site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
                site_data['temporal_resolution']['value'] = temporal_resolutions   
                site_data['flag']['value'] = flags
                site_data['qa']['value'] = qa
                #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
                for column_header in list(site_data.keys()):
                    n_column = len(site_data[column_header]['value'])
                    #if column is empty, then append NaN
                    if n_column == 0:
                        if site_data[column_header]['data_type'] != np.uint8:
                            site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                        else:
                            site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
                #------------------------------------------------------

                #standardise metadata/data formats and write out station data to numpy file
                write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
                if write_obj.written == False:
                    self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)