import codecs
import copy
import csv
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd
import json

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from standardisation_mapping_functions import EEA_external_meta_map, map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter
from geographic_functions import local_timezone_name
from network_parsers import ParseEEAFile


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific station into a GHOST standard numpy station specific file'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files of station reference to parse
        data_files = self.gather_data_files()

        #parse station reference data files
        for data_file in data_files:
            self.parse_file(data_file)

        #if stage has completed successfully, with some valid data, write a valid process log file
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #else, if no data has been written for station write an error log file to inform station is not continuing in the pipeline
        elif not self.written_data:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get data files specific to station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)

        #load external metadata into memory
        country_code = data_files[0].split('/')[-2]
        print('country_code', country_code)
        with open(self.run_details['data_root']+'EEA/metadata/{}/processed/EEA_{}_META.json'.format(self.run_details['ghost_version'], country_code), 'r', encoding='utf-8') as f:
            self.external_metadata = json.loads(f.read())
    
        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        #set flag to know when have written data for station    
        self.written_data = False

        return data_files


    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 

        #create object for parsing EEA file, and read file
        parsing_obj = ParseEEAFile(data_file,self.run_details)
        parsing_obj.read_data()

        #get data sampling point and aggregation method from filename
        f_split = data_file.split('/')[-1].split('__')
        data_sampling_point = f_split[0]
        aggregation_method = f_split[-1][:-4]

        # parse data
        start_dt = parsing_obj.parsed_data['start_dt']
        end_dt = parsing_obj.parsed_data['end_dt']
        measurements_raw = parsing_obj.parsed_data['measurement']
        verification_flags = parsing_obj.parsed_data['verification']
        validity_flags = parsing_obj.parsed_data['validity']
        units_raw = parsing_obj.parsed_data['unit']
            
        #create site instances of metadata/data dictionaries
        site_data =     get_standard_data(self.parameter_details)
        site_metadata = get_standard_metadata(self.parameter_details)

        #create a dictionary with unique units associated with indices in the file for conversion 
        unique_measurement_units = np.unique(units_raw)
        unit_dict = {}
        for units in unique_measurement_units:
            if units == '':
                continue
            #convert micro character (u'\xb5') to 'u'
            elif u'\xb5' in units:
                changed_units = units.replace(u'\xb5','u')
                unit_dict[changed_units] = np.where(units_raw == units)[0]
            else:
                unit_dict[units] = np.where(units_raw == units)[0]

        #------------------------------------------------------------------------#
        #PARSE EXTERNAL METADATA
        #------------------------------------------------------------------------#

        #STANDARD EXTERNAL METADATA FILE              
        #first try making data/metadata join with standard external meta file using the "SamplingPoint" (if not empty string)
        if data_sampling_point != '':
            #this list might have several entries because sampling point is not unique
            cut_external_metadata_standard = [val for key, val in self.external_metadata.items() if data_sampling_point == key] 
        else:
            cut_external_metadata_standard = []

        cut_unified_metadata = {}

        if len(cut_external_metadata_standard) > 0:
            # if station/sampling point is mobile, do not take it (0 = fixed, 1 = mobile)
            selected_external_metadata = {}
            for meta_spo_match in cut_external_metadata_standard:
                #take the first sampling point that is fixed, as cannot be more selective currently
                if int(meta_spo_match['Mobile']['values'][-1]) == 0: 
                    selected_external_metadata = meta_spo_match
                    break

            #map required metadata
            for var in selected_external_metadata:
                if var in list(EEA_external_meta_map.keys()):
                    #get standard_name of variable
                    standard_var = EEA_external_meta_map[var]
                    #get variable value (pass most recent metadata variable value (-1))
                    var_value = selected_external_metadata[var]['values'][-1] 

                    #remove invalid values that can be floats
                    try:
                        float_var_value = float(var_value)
                        if standard_var == 'altitude':
                            if float_var_value in [9999.0, 99999.0, 99991.0, 999.0]:
                                continue
                        elif standard_var == 'measuring_instrument_reported_lower_limit_of_detection':
                            if (float_var_value <= 0) or (float_var_value in [9999.0, 99999.0, 99991.0, 999.0]):
                                continue
                        else:
                            if (float_var_value < 0) or (float_var_value in [9999.0, 99999.0, 99991.0, 999.0]):
                                continue

                    #remove invalid values that are strings
                    except:
                        if var_value.strip() in ['', 'unknown']:
                            continue
                    
                    # if valid, write in metadata dictionary
                    cut_unified_metadata[standard_var] = var_value

        #Check have some valid metadata
        if len(cut_unified_metadata) == 0:
            self.run_details['log_file_str'] += 'NO VALID EXTERNAL METADATA FOUND FOR {} SAMPLING POINT. SKIPPING\n'.format(data_sampling_point)
            return

        #make sure have some valid units from data file
        if unit_dict == {}:
            no_units = True       
            #if don't, check if have units in metadata
            #assume detection limit unit is same as measurement unit
            if 'Detection Limit Unit' in list(selected_external_metadata.keys()):
                units = selected_external_metadata['Detection Limit Unit']['values'][-1] 
                if units.strip() != '':
                    #convert micro character (u'\xb5') to 'u'
                    units = units.replace(u'\xb5','u')
                    #get format correct for later conversion
                    units = units.replace('.', ' ').replace('year', 'yr')
                    #add units to unit dict
                    unit_dict[units] = np.arange(len(units_raw))
                    no_units = False
            #otherwise skip file
            if no_units:
                self.run_details['log_file_str'] += 'NO UNITS FOR {} SAMPLING POINT. SKIPPING\n'.format(data_sampling_point)
                return

        #------------------------------------------------------------------------#
        #PARSE TIME/DATA
        #------------------------------------------------------------------------#

        # for hourly/var measurements, the timezone is UTC+01, for daily measurements it is given in the metadata file
        if aggregation_method == 'day':   
            external_time_zone = cut_unified_metadata['time_zone']  
        else:
             external_time_zone = 'UTC+01'
        tzinfo = dateutil.tz.gettz(external_time_zone)  

        #convert timezone to UTC
        start_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in start_dt]
        end_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in end_dt]

        #convert each start measurement datetime to minutes from reference time
        frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

        #if length of time array is 0, skip to next data group
        if len(frac_time) == 0:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {} FOR {} SAMPLING POINT. SKIPPING.\n'.format(data_file,data_sampling_point)
            return

        #get timedeltas between start and end of measurement window
        temporal_resolutions = []
        for start, end in zip(start_dt,end_dt):
            dt_window = end - start
            #convert hours to minutes
            dt_window_minutes = int(dt_window.total_seconds()/60.)
            temporal_resolutions.append(dt_window_minutes)  

       #------------------------------------------------------------------------#
        #PARSE METADATA
        #------------------------------------------------------------------------#

        # get EoI code from external metadata
        eoicode = cut_unified_metadata['eoicode']

        #GHOST version
        site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

        #initialise string which will store all GHOST process warnings as empty            
        site_metadata['process_warnings']['value'].append('')

        #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
        site_metadata['meta_update_stamp']['value'].append(sorted(frac_time)[0])

        #write known metadata  
        site_metadata['network']['value'].append(self.run_details['network'])
        site_metadata['contact_institution']['value'].append('EEA')      
        site_metadata['contact_email_address']['value'].append('helpdesk@eionet.europa.eu')   

        #get data level from file path 
        site_metadata['data_level']['value'].append(data_file.split('original_files/{}/'.format(self.run_details['ghost_version']))[-1].split('/')[0])
        #data licence
        site_metadata['data_licence']['value'].append('Public domain where not explicitly stated otherwise: https://www.eea.europa.eu/legal/eea-data-policy') 

        #assume datum for spatial reference system
        site_metadata['ellipsoid']['value'].append('WGS 84')  
        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984') 
        site_metadata['vertical_datum']['value'].append('Ellipsoidal - WGS 84') 
        site_metadata['projection']['value'].append('geographic')

        #parse unique reported measurement unit
        #can be instances where have multiple units in a file, in this case write raw units as modal occurring unit (that is not blank or cannot be converted to standard units)
        #all raw units are taken into account when converting concentrations however
        unit_n = -1
        for unit_name in list(unit_dict.keys()):
            if len(unit_dict[unit_name]) > unit_n:
                reported_units = unit_name   
                unit_n = len(unit_dict[unit_name])        
        #convert raw reported units given to standard name
        conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
        site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

        #parse associated networks - local network with local code
        if ('local_network_code' in list(cut_unified_metadata.keys())) & ('local_reference' in list(cut_unified_metadata.keys())):
            associated_network_string = '{}:{}'.format(cut_unified_metadata['local_network_code'],cut_unified_metadata['local_reference'])
            site_metadata['associated_networks']['value'].append(associated_network_string)

        #parse measurement information (from external metadata) - use all possible variables of: 'sampling_method', 'measurement_equipment', 'analytical_technique', 'measurement_method', 'measurement_type'
        if 'sampling_method' in list(cut_unified_metadata.keys()):
            sampling_method = cut_unified_metadata['sampling_method']
        else:
            sampling_method = ''
        if 'measurement_equipment' in list(cut_unified_metadata.keys()):
            #remove 'unnessary long string from variable
            measurement_equipment = cut_unified_metadata['measurement_equipment'].split('measurementequipment/')[-1]
        else:
            measurement_equipment = ''
        if 'analytical_technique' in list(cut_unified_metadata.keys()):
            analytical_technique = cut_unified_metadata['analytical_technique']
        else:
            analytical_technique = ''
        if 'measurement_method' in list(cut_unified_metadata.keys()):
            #remove 'unnessary long string from variable
            measurement_method = cut_unified_metadata['measurement_method'].split('measurementmethod/')[-1]
        else:
            measurement_method = ''
        if 'measurement_type' in list(cut_unified_metadata.keys()):
            measurement_type = cut_unified_metadata['measurement_type']
        else:
            measurement_type = ''

        #concatenate measurement information together
        measurement_information = '{} {} {} {} {}'.format(sampling_method, measurement_equipment, analytical_technique, measurement_method, measurement_type)
        #strip leading/trailing white spaces, and change multiple white spaces to be singular
        measurement_information = measurement_information.strip()
        measurement_information = ' '.join(measurement_information.split())
        site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(measurement_information, site_metadata, self.run_details, self.parameter_details)
        #set measurement qa codes which are True as joined metadata str
        site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
        measurement_qa_flag_codes = []
        for standard_flag_name in measurement_qa_flags.keys():
            if measurement_qa_flags[standard_flag_name]:
                measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
        site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

        #parse station classifications (from external metadata) - use all possible variables of: 'area_classification', 'station_classification', 'main_emission_source', 'dispersion_local', 'dispersion_regional'
        if 'area_classification' in list(cut_unified_metadata.keys()):
            area_classification = cut_unified_metadata['area_classification']
        else:
            area_classification = ''
        if 'station_classification' in list(cut_unified_metadata.keys()):
            station_classification = cut_unified_metadata['station_classification']
        else:
            station_classification = ''
        if 'main_emission_source' in list(cut_unified_metadata.keys()):
            main_emission_source = cut_unified_metadata['main_emission_source']
        else:
            main_emission_source = ''
        if 'dispersion_local' in list(cut_unified_metadata.keys()):
            dispersion_local = cut_unified_metadata['dispersion_local']
        else:
            dispersion_local = ''
        if 'dispersion_regional' in list(cut_unified_metadata.keys()):
            dispersion_regional = cut_unified_metadata['dispersion_regional']
        else:
            dispersion_regional = ''

        native_classifications = {'area_classification':area_classification, 'station_classification':station_classification, 'main_emission_source':main_emission_source, 'dispersion_local':dispersion_local, 'dispersion_regional':dispersion_regional}
        site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

        #set standard temperature and pressure needed for conversion
        #measurement temperature and pressure values are never provided by EEA so are always assumed to be the network standard pressure and temperatures
        #these are defined as for 293 K temperature, and 1013 hPa for pressure here for all gas-phase compounds: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32008L0050&from=en and here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32004L0107&from=EN 
        #for particulate matter and substances to be analysed in particulate matter (e.g. lead) the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
        #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.
        #Precipitation concentrations are reported in mg l-1, mgS l-1 or mgN l-1. Only simple unit conversion needed.
        standard_network_gas_volume = False
        unknown_gas_volume = False
        if self.parameter_details['matrix'] == 'gas':
            site_metadata['network_provided_volume_standard_temperature']['value'].append('293.0')
            site_metadata['network_provided_volume_standard_pressure']['value'].append('1013.0')
            standard_network_gas_volume = True
        else:
            site_metadata['network_provided_volume_standard_temperature']['value'].append(np.NaN)
            site_metadata['network_provided_volume_standard_pressure']['value'].append(np.NaN)
            unknown_gas_volume = True

        #define list of variable names that require no processing that can be simply added to site_metadata
        metadata_to_add = ['latitude','longitude','height_facades','street_width','daytime_traffic_speed',
                           'daily_passing_vehicles','station_name']
        
        #iterate through all metadata variable names and put values into standard dictionary
        for variable_name in metadata_to_add:
            #check variable is in unified metadata dict
            if variable_name in list(cut_unified_metadata.keys()):
                site_metadata[variable_name]['value'].append(cut_unified_metadata[variable_name].replace('"',''))
        
        # define list of variable names that have units given in meta file
        metadata_with_unit_provided = ['altitude', 'sampling_height', 'distance_to_building', 'distance_to_kerb', 
                                       'distance_to_source', 'distance_to_junction', 'heating_emissions', 
                                       'traffic_emissions', 'industrial_emissions', 
                                       'measuring_instrument_reported_lower_limit_of_detection']
        
        #iterate through metadata variable names with units and convert units to GHOST standards as appropriate
        for variable_name in metadata_with_unit_provided:

            #check variable is in unified metadata dict
            if variable_name in list(cut_unified_metadata.keys()):

                #get metadata unit
                raw_meta_unit = selected_external_metadata[list(EEA_external_meta_map.keys())[list(EEA_external_meta_map.values()).index(variable_name)]+' Unit']['values'][-1].replace('.', ' ').replace('year', 'yr')
                
                #manually convert from pH to H+ conc
                if raw_meta_unit == 'pHunits':
                    raw_meta_unit = 'mol l-1'

                #for distances if unit is missing, assume metres
                if variable_name in ['distance_to_building', 'distance_to_kerb', 'distance_to_source', 'distance_to_junction']:
                    if raw_meta_unit.strip() == '':
                        raw_meta_unit = 'm'

                # get measured unit quantity
                # if no unit given, continue to next meta variable
                if raw_meta_unit.strip() == '': 
                    continue
                else:
                    conv_obj = unit_converter.convert_units(raw_meta_unit,raw_meta_unit,1, measured_species=self.parameter_details['chemical_formula'])
                    meta_unit_quantity_reported = conv_obj.output_represented_quantity
                    meta_unit_reported = conv_obj.output_standard_units

                # get GHOST standard unit quantity
                conv_obj = unit_converter.convert_units(site_metadata[variable_name]['units'],site_metadata[variable_name]['units'],1)
                meta_unit_quantity_standard = conv_obj.output_represented_quantity
                meta_unit_standard = conv_obj.output_standard_units

                #check if reported standard units, match required standard units
                if meta_unit_reported != meta_unit_standard:

                    # detection limit: should have same unit as parameter: self.measured_unit_quantity
                    #mass density to mixing ratio?
                    if (meta_unit_quantity_reported == 'mass_density') & (meta_unit_quantity_standard == 'mole_fraction'):
                        #get conversion factor
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':meta_unit_reported}
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards for gases)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,meta_unit_standard,input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
                    
                    #mixing ratio to mass density?
                    elif (meta_unit_quantity_reported == 'mole_fraction') & (meta_unit_quantity_standard == 'mass_density'):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':meta_unit_reported}     
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,meta_unit_standard,input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
                    
                    # molarity to mass density? (for H+ conc precip)
                    elif (meta_unit_quantity_reported == 'molarity') & (meta_unit_quantity_standard == 'mass_density'):
                        input_units = {'molar_mass':'kg mol-1', 'molarity':meta_unit_reported}     
                        #do conversion 
                        input_vals = {'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'molarity':1.0}
                        conv_obj = unit_converter.convert_units(input_units,meta_unit_standard,input_vals,measured_species=self.parameter_details['chemical_formula'],conversion_input_quantity = 'molarity')

                    #otherwise simple unit conversion, this should cover all lengths
                    else:
                        conv_obj = unit_converter.convert_units(meta_unit_reported,meta_unit_standard,1.0, measured_species = self.parameter_details['chemical_formula'])

                    # apply conversion factor to meta variable and add the site_metadata
                    try:
                        float_meta = float(cut_unified_metadata[variable_name].replace('"',''))
                    except:
                        float_meta = np.NaN
                    if np.isfinite(float_meta):
                        site_metadata[variable_name]['value'].append(float_meta*conv_obj.conversion_factor)
                    else:
                        site_metadata[variable_name]['value'].append(float_meta)

                else:
                    try:
                        float_meta = float(cut_unified_metadata[variable_name].replace('"',''))
                    except:
                        float_meta = np.NaN
                    site_metadata[variable_name]['value'].append(float_meta)

        #iterate through metadata variables, if any columns have not been appended to add a NaN
        for column_header in list(site_metadata.keys()):
            n_column = len(site_metadata[column_header]['value'])
            #if column is empty, then append NaN
            if n_column < 1:
                site_metadata[column_header]['value'].append(np.NaN)
        
        #set station_reference (provided network ID + standard measurement method abbreviation)
        site_metadata['station_reference']['value'][-1] = '{}_{}'.format(cut_unified_metadata['eoicode'],standard_method_abbreviation) 

        #get data file download timestamp
        site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

        #add data revision timestamp, set as 0001-01-01 00:00 for all files
        data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
        site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

        #------------------------------------------------------------------------#
        #PARSE DATA FLAGS
        #------------------------------------------------------------------------#
        flags = []
        qa = []

        #get dictionary necessary for mapping of EEA file data flags to standard data flag codes
        data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['EEA']                

        #get flags from data file - Validity
        data_flags = validity_flags

        #get verification information - informs on data verification
        verification_data = verification_flags

        #iterate through EEA file data flags and map to standard flag codes
        for data_flag, verification_flag in zip(data_flags,verification_data):
            #lists to append qa/data flags per measurement
            flags_per_measurement = []
            qa_per_measurement = []

            #set GHOST decreed validity/network decreed validity variables initially as 'V'
            ghost_decreed_validity = 'V'
            network_decreed_validity = 'V'

            # map data flag/s and append
            standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
            #get mapped standard data flags
            standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
            #iterate through standard data flags 
            for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                #check if standard data flag is empty (i.e. unmapped as non-useful)
                if standard_data_flag_name != '': 
                    #get corresponding data flag code for standard data flag
                    flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                    #append standard data flag
                    if flag_code not in flags_per_measurement:
                        flags_per_measurement.append(flag_code)
                    #determine if GHOST/network decreed valid
                    #print(flags_per_measurement)
                    if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                        ghost_decreed_validity = 'I'
                    if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                        network_decreed_validity = 'I'
        
            #write preliminary data flags if necessary (if "Verification" variable informs that data is preliminary)
            #1 = Verified, 2 = Preliminary verified, 3 = Not verified
            if (verification_flag == '2') or (verification_flag == '3'):
                if standard_data_flag_name_to_data_flag_code['Preliminary Data'] not in flags_per_measurement:
                    flags_per_measurement.append(standard_data_flag_name_to_data_flag_code['Preliminary Data'])

            #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
            if ghost_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
            if network_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])
    
            #append qa_per_measurement, flags_per_measurement to lists to write out
            flags.append(flags_per_measurement)
            qa.append(qa_per_measurement)

        #------------------------------------------------------------------------#
        #PARSE MEASUREMENTS
        #------------------------------------------------------------------------#
        measurements = []

        #measurement values may need converting (i.e. mass density to mixing ratio)

        #check if reported standard units, match required standard units
        #take care to deal with multiple units in file
        order_n = []
        for reported_units in list(unit_dict.keys()):

            #set flags for writing QA flags associated with assumptions in unit conversion
            unit_conversion_standard_network = False
            unit_conversion_educated_guess = False

            #convert raw reported units given to standard name
            try:
                conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
            #if cannot convert units to standard units, then then a chemical formula is needed for conversion of units but do not have it: ug C/m3 for NMHC
            #therefore do not write concentrations and remove units form unit_dict
            except SystemExit:
                del unit_dict[reported_units]
                continue
            standard_reported_units = conv_obj.output_standard_units
            measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

            #cut concentration data to be that just associated with units
            cut_concentrations = measurements_raw[unit_dict[reported_units]]
            #print(data_file, cut_concentrations)

            #check if reported standard units, match required standard units
            if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:
                #mass density to mixing ratio?
                if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                    #get conversion factor
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards for gases)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
        
                #mixing ratio to mass density?
                elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

                # molarity to mass density? (for H+ conc precip)
                elif (measuring_instrument_reported_unit_quantity == 'molarity') & (self.measured_unit_quantity == 'mass_density'):
                    input_units = {'molar_mass':'kg mol-1', 'molarity':standard_reported_units}     
                    #do conversion 
                    input_vals = {'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'molarity':1.0}
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species=self.parameter_details['chemical_formula'],conversion_input_quantity = 'molarity')

                #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                else:
                    conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                #process through each measurement and convert value using derived conversion factor
                for n, meas in enumerate(cut_concentrations):
                    actual_n = unit_dict[reported_units][n]
                    order_n.append(actual_n)
                    #only do conversion if measurement value is above 0 and not infinite, and do not have data flag stating the measurement is missing 
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[actual_n]):
                        measurements.append(float_meas*conv_obj.conversion_factor)
                    else:
                        measurements.append(float_meas)

                    #append sample gas volume/unit_conversion assumption QA flags if necessary
                    if standard_network_gas_volume:
                        qa[actual_n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                    if unknown_gas_volume:
                        qa[actual_n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                    if unit_conversion_standard_network:
                        qa[actual_n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                    if unit_conversion_educated_guess:
                        qa[actual_n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
            
            #otherwise do nothing to measured values
            else:
                for n, meas in enumerate(cut_concentrations):
                    order_n.append(unit_dict[reported_units][n])
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    measurements.append(float_meas)

        #have 0 measurements after unit conversion? if so skip to next file
        if len(measurements) == 0:
            self.run_details['log_file_str'] += 'NO VALID UNITS IN FILE: {} FOR {} SAMPLING POINT. SKIPPING.\n'.format(data_file,data_sampling_point)
            return

        #resort measurements by original order
        measurements = [x for _,x in sorted(zip(order_n,measurements))]

        #filter frac_time, temporal_resolutions, flags and qa by order_n (in case there are some measurements associated unknown units in file
        order_n = sorted(order_n)
        frac_time = [frac_time[x] for x in order_n]
        temporal_resolutions = [temporal_resolutions[x] for x in order_n]
        flags = [flags[x] for x in order_n]
        qa = [qa[x] for x in order_n]

        #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
        valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
        if len(valid_temporal_resolution_inds) != len(frac_time):
            frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
            temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
            flags = [flags[x] for x in valid_temporal_resolution_inds]
            qa = [qa[x] for x in valid_temporal_resolution_inds]
            measurements = [measurements[x] for x in valid_temporal_resolution_inds]
        
        #----------------------------------------------------------------------------#
        #WRITE OUT DATA
        #----------------------------------------------------------------------------# 
        site_data['time']['value'] = frac_time
        site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
        site_data['temporal_resolution']['value'] = temporal_resolutions   
        site_data['flag']['value'] = flags
        site_data['qa']['value'] = qa
        #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
        for column_header in list(site_data.keys()):
            n_column = len(site_data[column_header]['value'])
            #if column is empty, then append NaN
            if n_column == 0:
                if site_data[column_header]['data_type'] != np.uint8:
                    site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                else:
                    site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
        #------------------------------------------------------

        #standardise metadata/data formats and write out station data to numpy file
        write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)

        #update flag to state that have written data for station (if have done so)
        if write_obj.written:
            self.written_data = True
        else:
            self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)