import codecs
import copy
import datetime
import os
import re
import sys
import time
from collections import namedtuple
Range = namedtuple('Range', ['start', 'end'])

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseEEAAIRBASEFile
from standardisation_mapping_functions import EEA_AIRBASE_external_meta_map, map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific station into a GHOST standard numpy station specific file'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files of station reference to parse
        data_files = self.gather_data_files()

        #parse station reference data files
        for data_file in data_files:
            self.parse_file(data_file)

        #if stage has completed successfully, with some valid data, write a valid process log file
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #else, if no data has been written for station write an error log file to inform station is not continuing in the pipeline
        elif not self.written_data:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get data files specific to station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)
    
        #sort files in date order
        data_files_start_dates = [datetime.datetime(int(data_file.split('.')[-2].split('-')[2]), int(data_file.split('.')[-2].split('-')[1]), int(data_file.split('.')[-2].split('-')[0])) for data_file in data_files]
        data_files = np.array([x for _,x in sorted(zip(data_files_start_dates,data_files))])   

        #load external metadata into memory 
        self.external_metadata = np.load(self.run_details['data_root']+'EEA_AIRBASE/metadata/processed/EEA_AIRBASE_META.npy', allow_pickle=True)
    
        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        #set flag to know when have written data for station    
        self.written_data = False

        return data_files


    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
    
        #create object for parsing EEA AIRBASE file, and read file
        parsing_obj = ParseEEAAIRBASEFile(data_file)
        parsing_obj.read_data()
            
        if len(parsing_obj.parsed_data['measurement']) == 0:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {}. SKIPPING.\n'.format(data_file)
            return

        #create site instances of metadata/data dictionaries
        site_data =     get_standard_data(self.parameter_details)
        site_metadata = get_standard_metadata(self.parameter_details)

        #------------------------------------------------------------------------#
        #PARSE METADATA
        #------------------------------------------------------------------------#

        #make join to external metadata file --> by station_reference, component_code and measurement_european_group_code
        cut_external_metadata = self.external_metadata[(self.external_metadata['station_european_code'] == self.station_reference) & (self.external_metadata['component_code'].astype(int) == int(parsing_obj.component_code)) & (self.external_metadata['measurement_european_group_code'].astype(int) == int(parsing_obj.measurement_european_group_code))]
        #if have 0 matches then try to make join with just station_reference and component_code
        if len(cut_external_metadata) == 0:
            cut_external_metadata = self.external_metadata[(self.external_metadata['station_european_code'] == self.station_reference) & (self.external_metadata['component_code'].astype(int) == int(parsing_obj.component_code))] 
        #if still have 0 matches then retrieve external metadata so skip station
        if len(cut_external_metadata) == 0:
            self.run_details['log_file_str'] += 'UNABLE TO MAKE JOIN WITH EXTERNAL METADATA FILE FOR {}. SKIPPING.\n'.format(data_file)
            return
        #if have more than 1 match then select match by taking into most relevant measurement date, i.e. file measurement range overlaps most 
        if len(cut_external_metadata) > 1:
            #first convert empty strings to a date much later in future (i.e. empty string = no end date)
            measurement_start_dates = copy.deepcopy(cut_external_metadata['measurement_start_date'])
            measurement_end_dates = copy.deepcopy(cut_external_metadata['measurement_end_date'])
            measurement_end_dates[np.where(measurement_end_dates == '')[0]] = '2030-01-01'
            measurement_start_dates = [datetime.datetime(int(measurement_start_date.split('-')[0]), int(measurement_start_date.split('-')[1]), int(measurement_start_date.split('-')[2])) for measurement_start_date in measurement_start_dates] 
            measurement_end_dates = [datetime.datetime(int(measurement_end_date.split('-')[0]), int(measurement_end_date.split('-')[1]), int(measurement_end_date.split('-')[2])) for measurement_end_date in measurement_end_dates] 
            #find n days overlap between file window and measurement windows
            overlap_days = []
            measurement_length_days = []
            for measurement_start_date, measurement_end_date in zip(measurement_start_dates, measurement_end_dates):
                r1 = Range(start=parsing_obj.data_file_start_date, end=parsing_obj.data_file_end_date)
                r2 = Range(start=measurement_start_date, end=measurement_end_date)
                latest_start = max(r1.start, r2.start)
                earliest_end = min(r1.end, r2.end)
                delta = (earliest_end - latest_start).days
                overlap_days.append(max(0, delta))
                measurement_length_days.append((measurement_end_date - measurement_start_date).days)
            #get measurement window with biggest overlap with file window (if have at least non-zero overlap, and overlaps are not all equal)
            if (np.max(overlap_days) > 0) & (len(np.unique(overlap_days)) > 1):
                cut_external_metadata = cut_external_metadata[np.argmax(overlap_days)]
            #if still have not got unique join, then take the metadata with the longest measurement window preferentially
            #if these are equal the first index will be taken preferentially
            else:
                cut_external_metadata = cut_external_metadata[np.argmax(measurement_length_days)]
        else:
            cut_external_metadata = cut_external_metadata[0]

        #map to metadata fields to standard names
        cut_external_metadata_mapped = {}
        #first load in standard metadata variables
        for var in self.external_metadata.dtype.names:                    
            if var in list(EEA_AIRBASE_external_meta_map.keys()):
                #get standard_name of variable
                standard_var = EEA_AIRBASE_external_meta_map[var]
                #get variable value
                var_value = cut_external_metadata[var]
                #do not write if empty string, or 'unknown' or '-999'
                if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                    cut_external_metadata_mapped[standard_var] = var_value

        #GHOST version
        site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

        #initialise string which will store all GHOST process warnings as empty            
        site_metadata['process_warnings']['value'].append('')

        #write known metadata  
        site_metadata['network']['value'].append(self.run_details['network'])
        #data level - none
        site_metadata['data_level']['value'].append('none')
        #data licence
        site_metadata['data_licence']['value'].append('Public domain where not explicitly stated otherwise: https://www.eea.europa.eu/legal/eea-data-policy') 

        #convert raw reported units given to standard name
        reported_units = cut_external_metadata_mapped['measurement_unit']
        #convert micro character (u'\xb5') to 'u'
        reported_units = reported_units.replace(u'\xb5','u')
        #skip file if no valid units
        if reported_units == '':
            self.run_details['log_file_str'] += 'NO UNITS FOR {}. SKIPPING FILE\n'.format(self.station_reference)
            return
        conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
        site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

        #define list of variable names that require no processing that can be simply added to site_metadata
        metadata_to_add = ['latitude','longitude','altitude','sampling_height','distance_to_kerb',
                          'distance_to_junction','distance_to_source','street_type','street_width','daytime_traffic_speed',
                          'daily_passing_vehicles','station_name','representative_radius','principal_investigator_institution',
                          'principal_investigator_email_address','contact_institution','contact_email_address']

        #iterate through all metadata variable names and put values into standard dictionary
        for variable_name in metadata_to_add:
            #check variable is in unified metadata dict
            if variable_name in list(cut_external_metadata_mapped.keys()):
                site_metadata[variable_name]['value'].append(cut_external_metadata_mapped[variable_name].replace('"',''))

        #parse associated networks 
        associated_network_string = ''
        #first add EBAS, if have 'emep_station' variable and is 'yes'
        if 'emep_station' in list(cut_external_metadata_mapped.keys()):
            if cut_external_metadata_mapped['emep_station'] == 'yes':
                associated_network_string = 'EBAS:{}'.format(self.station_reference)
        #secondly, add local network with local code
        if ('local_network_code' in list(cut_external_metadata_mapped.keys())) & ('local_reference' in list(cut_external_metadata_mapped.keys())):
            local_string = '{}:{}'.format(cut_external_metadata_mapped['local_network_code'],cut_external_metadata_mapped['local_reference'])
            if associated_network_string == '':   
                associated_network_string = copy.deepcopy(local_string)
            else:
                associated_network_string = '{};{}'.format(associated_network_string, local_string)
        #append associated_network_string if not empty
        if associated_network_string != '':
            site_metadata['associated_networks']['value'].append(associated_network_string)

        #assume spatial reference system is same as EEA AQ e-Reporting (EPSG:4979)
        site_metadata['ellipsoid']['value'].append('WGS 84')  
        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984') 
        site_metadata['vertical_datum']['value'].append('Ellipsoidal - WGS 84') 
        site_metadata['projection']['value'].append('geographic')

        #parse population (need to check to remove non-numeric characters)
        if 'population' in list(cut_external_metadata_mapped.keys()):
            population = cut_external_metadata_mapped['population']
            population = re.sub("[^0-9]", "", population)
            site_metadata['population']['value'].append(population) 

        #parse 'principal_investigator_name' (concatenate title, first name and last name, if given)
        principal_investigator_name = ''
        if 'principal_investigator_title' in list(cut_external_metadata_mapped.keys()):
            principal_investigator_name = principal_investigator_name + cut_external_metadata_mapped['principal_investigator_title']
        if 'principal_investigator_first_name' in list(cut_external_metadata_mapped.keys()):
            if principal_investigator_name == '':
                principal_investigator_name = principal_investigator_name + cut_external_metadata_mapped['principal_investigator_first_name']
            else:
                principal_investigator_name = '{} {}'.format(principal_investigator_name, cut_external_metadata_mapped['principal_investigator_first_name'])
        if 'principal_investigator_last_name' in list(cut_external_metadata_mapped.keys()):
            if principal_investigator_name == '':
                principal_investigator_name = principal_investigator_name + cut_external_metadata_mapped['principal_investigator_last_name']
            else:
                principal_investigator_name = '{} {}'.format(principal_investigator_name, cut_external_metadata_mapped['principal_investigator_last_name'])
        if principal_investigator_name != '':
            site_metadata['principal_investigator_name']['value'].append(principal_investigator_name) 
 
        #parse 'contact_name' (concatenate title, first name and last name, if given)
        contact_name = ''
        if 'contact_title' in list(cut_external_metadata_mapped.keys()):
            contact_name = contact_name + cut_external_metadata_mapped['contact_title']
        if 'contact_first_name' in list(cut_external_metadata_mapped.keys()):
            if contact_name == '':
                contact_name = contact_name + cut_external_metadata_mapped['contact_first_name']
            else:
                contact_name = '{} {}'.format(contact_name, cut_external_metadata_mapped['contact_first_name'])
        if 'contact_last_name' in list(cut_external_metadata_mapped.keys()):
            if contact_name == '':
                contact_name = contact_name + cut_external_metadata_mapped['contact_last_name']
            else:
                contact_name = '{} {}'.format(contact_name, cut_external_metadata_mapped['contact_last_name'])
        if contact_name != '':
            site_metadata['contact_name']['value'].append(contact_name) 

        #parse miscellaneous details
        #deal with miscellaneous sampling details
        if 'sample_location_miscellaneous' in list(cut_external_metadata_mapped.keys()):
            miscellaneous_sampling = 'Sampling location: {}'.format(cut_external_metadata_mapped['sample_location_miscellaneous'])
            site_metadata['network_sampling_details']['value'].append(miscellaneous_sampling) 

        #deal with miscellaneous sampling inlet information
        if 'length_sampling_line' in list(cut_external_metadata_mapped.keys()):
            inlet_info = 'Length: {}'.format(cut_external_metadata_mapped['length_sampling_line'])
            site_metadata['measuring_instrument_inlet_information']['value'].append(inlet_info)
                
        #deal with miscellaneous maintenance details
        miscellaneous_maintenance = ''
        if 'calibration_method' in list(cut_external_metadata_mapped.keys()):
            string_to_add = 'Calibration methodology: {}'.format(cut_external_metadata_mapped['calibration_method'])
            miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
        if 'calibration_frequency' in list(cut_external_metadata_mapped.keys()):
            string_to_add = 'Calibration frequency: {}'.format(cut_external_metadata_mapped['calibration_frequency'])
            if miscellaneous_maintenance == '':
                miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
            else:
                miscellaneous_maintenance = '{};{}'.format(miscellaneous_maintenance, string_to_add) 
        if 'calibration_description' in list(cut_external_metadata_mapped.keys()):
            string_to_add = 'Calibration description: {}'.format(cut_external_metadata_mapped['calibration_description'])
            if miscellaneous_maintenance == '':
                miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
            else:
                miscellaneous_maintenance = '{};{}'.format(miscellaneous_maintenance, string_to_add) 
        if miscellaneous_maintenance != '':
            site_metadata['network_maintenance_details']['value'].append(miscellaneous_maintenance)

        #deal with other miscellaneous details
        if 'station_description' in list(cut_external_metadata_mapped.keys()):
            station_description = 'Station description: {}'.format(cut_external_metadata_mapped['station_description'])
            site_metadata['network_miscellaneous_details']['value'].append(station_description) 

        #parse measurement information (from external metadata) - use all possible variables of: 'measurement_equipment', 'measurement_method', 'measurement_automatic'
        if 'measurement_equipment' in list(cut_external_metadata_mapped.keys()):
            #remove 'unnecessary long string from variable
            measurement_equipment = cut_external_metadata_mapped['measurement_equipment'].split('measurementequipment/')[-1]
        else:
            measurement_equipment = ''
        if 'measurement_method' in list(cut_external_metadata_mapped.keys()):
            #remove 'unnessary long string from variable
            measurement_method = cut_external_metadata_mapped['measurement_method'].split('measurementmethod/')[-1]
        else:
            measurement_method = ''
        #parse 'measurement_automatic' 
        if 'measurement_automatic' in list(cut_external_metadata_mapped.keys()):
            measurement_automatic = cut_external_metadata_mapped['measurement_automatic']
            if measurement_automatic == 'yes':
                measurement_type = 'automatic'
            elif measurement_automatic == 'no':
                measurement_type = 'not-automatic'
        else:
            measurement_type = ''
        #concatenate measurement information together
        measurement_information = '{} {} {}'.format(measurement_equipment, measurement_method, measurement_type)
        #strip leading/trailing white spaces, and change multiple white spaces to be singular
        measurement_information = measurement_information.strip()
        measurement_information = ' '.join(measurement_information.split())
        site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(measurement_information, site_metadata, self.run_details, self.parameter_details)
        #set measurement qa codes which are True as joined metadata str
        site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
        measurement_qa_flag_codes = []
        for standard_flag_name in measurement_qa_flags.keys():
            if measurement_qa_flags[standard_flag_name]:
                measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
        site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

        #parse station classifications (from external metadata) - use all possible variables of: 'area_classification', 'station_classification', 'main_emission_source', 'land_use', 'area_classification_rural_subdivision', 'representative_radius'
        if 'area_classification' in list(cut_external_metadata_mapped.keys()):
            area_classification = cut_external_metadata_mapped['area_classification']
        else:
            area_classification = ''
        if 'station_classification' in list(cut_external_metadata_mapped.keys()):
            station_classification = cut_external_metadata_mapped['station_classification']
        else:
            station_classification = ''
        if 'main_emission_source' in list(cut_external_metadata_mapped.keys()):
            main_emission_source = cut_external_metadata_mapped['main_emission_source']
        else:
            main_emission_source = ''
        if 'land_use' in list(cut_external_metadata_mapped.keys()):
            land_use = cut_external_metadata_mapped['land_use']
        else:
            land_use = ''
        if 'area_classification_rural_subdivision' in list(cut_external_metadata_mapped.keys()):
            area_classification_rural_subdivision = cut_external_metadata_mapped['area_classification_rural_subdivision']
        else:
            area_classification_rural_subdivision = ''   
        if 'representative_radius' in list(cut_external_metadata_mapped.keys()):
            representative_radius = cut_external_metadata_mapped['representative_radius']
        else:
            representative_radius = ''   
        native_classifications = {'area_classification':area_classification, 'station_classification':station_classification, 'main_emission_source':main_emission_source, 'land_use':land_use, 'area_classification_rural_subdivision':area_classification_rural_subdivision, 'representative_radius':representative_radius}
        site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

        #iterate through metadata variables, if any columns have not been appended to add a NaN
        for column_header in list(site_metadata.keys()):
            n_column = len(site_metadata[column_header]['value'])
            #if column is empty, then append NaN
            if n_column < 1:
                site_metadata[column_header]['value'].append(np.NaN)
        
        #set station_reference (provided network ID + standard measurement method abbreviation)
        site_metadata['station_reference']['value'][-1] = '{}_{}'.format(self.station_reference,standard_method_abbreviation) 

        #get data file download timestamp
        site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

        #add data revision timestamp, set as 0001-01-01 00:00 for all files
        data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
        site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

        #------------------------------------------------------------------------#
        #PARSE TIME
        #------------------------------------------------------------------------#
        tzinfos = {'CET':3600, 'EET':7200, 'UTC':dateutil.tz.gettz('UTC')}
            
        if cut_external_metadata_mapped['network_time_reference_basis'] not in list(tzinfos.keys()):                    
            self.run_details['log_file_str'] += 'UNCHECKED TIMEZONE = {}\n'.format(cut_external_metadata_mapped['network_time_reference_basis'])
            create_GHOST_stage_logfile(2,self.run_details)

        #round to nearest minute to avoid issues due to floating point estimation
        start_dt = [round_to_minute(dateutil.parser.parse('{} {}'.format(date,cut_external_metadata_mapped['network_time_reference_basis']), tzinfos=tzinfos).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in parsing_obj.parsed_data['start_dt']]
        end_dt = [round_to_minute(dateutil.parser.parse('{} {}'.format(date,cut_external_metadata_mapped['network_time_reference_basis']), tzinfos=tzinfos).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in parsing_obj.parsed_data['end_dt']]

        #convert each start measurement datetime to minutes from reference time
        frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

        #get timedeltas between start and end of measurement window
        temporal_resolutions = []
        for start, end in zip(start_dt,end_dt):
            dt_window = end - start
            #convert hours to minutes
            dt_window_minutes = int(dt_window.total_seconds()/60.)
            temporal_resolutions.append(dt_window_minutes)  

        #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
        site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

        #------------------------------------------------------------------------#
        #PARSE QA/FLAGS
        #------------------------------------------------------------------------#

        flags = []
        qa = []

        #get dictionary necessary for mapping of EEA AQ eReporting file data flags to standard data flag codes
        data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['EEA_AIRBASE']

        for data_flag in parsing_obj.parsed_data['flag']:

            #if data_flag <=0, set it as -1, if >=1, set as 1
            if int(data_flag) <= 0:
                data_flag = '-1'
            elif int(data_flag) >= 1:
                data_flag = '1'

            #lists to append qa/data flags per measurement
            flags_per_measurement = []
            qa_per_measurement = []

            #set GHOST decreed validity/network decreed validity variables initially as 'V'
            ghost_decreed_validity = 'V'
            network_decreed_validity = 'V'

            # map data flag/s and append
            standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
            #get mapped standard data flags
            standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
            #iterate through standard data flags 
            for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                #check if standard data flag is empty (i.e. unmapped as non-useful)
                if standard_data_flag_name != '': 
                    #get corresponding data flag code for standard data flag
                    flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                    #append standard data flag
                    if flag_code not in flags_per_measurement:
                        flags_per_measurement.append(flag_code)
                    #determine if GHOST/network decreed valid
                    if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                        ghost_decreed_validity = 'I'
                    if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                        network_decreed_validity = 'I'

            #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
            if ghost_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
            if network_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])
        
            #append qa_per_measurement, flags_per_measurement to lists to write out
            flags.append(flags_per_measurement)
            qa.append(qa_per_measurement)

        #------------------------------------------------------------------------#
        #PARSE MEASUREMENTS
        #------------------------------------------------------------------------#
        #measurement values may need converting (i.e. mass density to mixing ratio)

        #measurement temperature and pressure values are never provided by EEA AIRBASE so are always assumed to be the network standard pressure and temperatures
        #these are defined as for 293 K temperature, and 1013 hPa for pressure here for all gas-phase compounds: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32008L0050&from=en and here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32004L0107&from=EN 
        #for particulate matter and substances to be analysed in particulate matter (e.g. lead) the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
        #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.
        
        measurements = []

        standard_network_gas_volume = False
        unknown_gas_volume = False

        if self.parameter_details['matrix'] == 'gas':
            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '293.0'
            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.0'
            standard_network_gas_volume = True
        else:
            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
            unknown_gas_volume = True

        #get standard reported units/quantity
        standard_reported_units = conv_obj.output_standard_units
        measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

        #set flags for writing QA flags associated with assumptions in unit conversion
        unit_conversion_standard_network = False
        unit_conversion_educated_guess = False

        #check if reported standard units, match required standard units
        if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

            #mass density to mixing ratio?
            if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                #get conversion factor
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume network standards)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                    unit_conversion_standard_network = True
                else:
                    input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                    unit_conversion_educated_guess = True
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')

            #mixing ratio to mass density?
            elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume network standards)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                    unit_conversion_standard_network = True
                else:
                    input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                    unit_conversion_educated_guess = True
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

            #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
            else:
                conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

            #process through each measurement and convert value using derived conversion factor
            for n, meas in enumerate(parsing_obj.parsed_data['measurement']):
                #only do conversion if measurement value is above 0 and not infinite
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                
                if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                    measurements.append(float_meas*conv_obj.conversion_factor)
                else:
                    measurements.append(float_meas)

                #append sample gas volume/unit_conversion assumption QA flags if necessary
                if standard_network_gas_volume:
                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                if unknown_gas_volume:
                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                if unit_conversion_standard_network:
                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                if unit_conversion_educated_guess:
                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

        #otherwise do nothing to measured values
        else:
            for n, meas in enumerate(parsing_obj.parsed_data['measurement']):
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                measurements.append(float_meas)

        #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
        valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
        if len(valid_temporal_resolution_inds) != len(frac_time):
            frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
            temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
            flags = [flags[x] for x in valid_temporal_resolution_inds]
            qa = [qa[x] for x in valid_temporal_resolution_inds]
            measurements = [measurements[x] for x in valid_temporal_resolution_inds]

        #----------------------------------------------------------------------------#
        #WRITE OUT DATA
        #----------------------------------------------------------------------------# 
        site_data['time']['value'] = frac_time
        site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
        site_data['temporal_resolution']['value'] = temporal_resolutions   
        site_data['flag']['value'] = flags
        site_data['qa']['value'] = qa
        #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
        for column_header in list(site_data.keys()):
            n_column = len(site_data[column_header]['value'])
            #if column is empty, then append NaN
            if n_column == 0:
                if site_data[column_header]['data_type'] != np.uint8:
                    site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                else:
                    site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
        #------------------------------------------------------

        #standardise metadata/data formats and write out station data to numpy file
        write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)

        #update flag to state that have written data for station (if have done so)
        if write_obj.written:
            self.written_data = True
        else:
            self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)
                     