import copy
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd
import json

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from geographic_functions import local_timezone_name
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseCANADANAPSFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter

#define dict for mapping units
unit_dict = {'ppb':'nmol mol-1','ppbv':'nmol mol-1','ppm':'umol mol-1','ppmv':'umol mol-1','ug/m3':'ug m-3','ng/m3':'ng m-3'}

class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, year, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.year = year
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

         # set flag for precip metals
        self.metals = False
        if self.run_details['parameter'] in ['Al_PREC', 'As_PREC', 'Cd_PREC', 'Co_PREC', 'Cr_PREC', 'Cu_PREC', 'Fe_PREC', 'Mn_PREC', 'Ni_PREC', 'Pb_PREC', 'Se_PREC', 'V_PREC', 'Zn_PREC']:
            self.metals = True
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current year
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per year'''

        #get specific data files for current year
        data_files = get_data_files(self.run_details, self.parameter_details, specific_year=self.year)

        #load all external metadata into memory 
        #self.external_metadata = np.load(self.run_details['data_root']+'CANADA_NAPS/metadata/processed/CANADA_NAPS_META.npy', allow_pickle=True)
        #load external metadata into memory 
        with open(self.run_details['data_root']+'CANADA_NAPS/metadata/{}/processed/CANADA_NAPS_META.json'.format(self.run_details['ghost_version']), 'r', encoding='utf-8') as f:
            self.external_metadata = json.loads(f.read())
   
        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_files

    def parse_file(self, data_file):

        '''Function to a specific annual data file to GHOST standard numpy station specific files'''

        #iterate through NAPS parameter codes until get valid read
        #for continuous .csv files, only use first code (as code is not used in parsing)
        if data_file[-4:] == '.csv':
            all_NAPS_codes = self.parameter_details['naps_code'][0:1]
        else:
            all_NAPS_codes = self.parameter_details['naps_code']

        for NAPS_code in all_NAPS_codes:

            self.run_details['log_file_str'] += 'PROCESSING FILE: {} FOR {} NAPS PARAMETER NAME\n'.format(data_file,NAPS_code) 

            #set all NAPS codes (with current NAPS code first), to parse detection limits and flags (where measurement/dl/flag names are various combinations of NAPS codes)
            # NAPS CODE for precipitation data processing not needed -> NAPS CODE set to parameter name in GHOST standards
            all_NAPS_codes = [NAPS_code]        
            for other_NAPS_code in all_NAPS_codes:
                if other_NAPS_code != NAPS_code:
                    all_NAPS_codes.append(other_NAPS_code)

            #create object for parsing CANADA NAPS annual file, and read file
            parsing_obj = ParseCANADANAPSFile(data_file,self.run_details,NAPS_code,all_NAPS_codes,self.year,self.parameter_details)
            parsing_obj.read_data()

            #check if have some data sheets
            if len(parsing_obj.all_parsed_data) == 0:
                self.run_details['log_file_str'] += 'NO VALID DATA SHEETS FOR {} NAPS PARAMETER NAME IN FILE.\n'.format(NAPS_code)

            #iterate through parsed data sheets
            for parsed_data, parsed_metadata in zip(parsing_obj.all_parsed_data,parsing_obj.all_parsed_metadata):


                #have no data, or all empty strings, then continue to next sheet
                if (len(parsed_data['measurement']) == 0) or (np.all(parsed_data['measurement'] == '')):
                    self.run_details['log_file_str'] += 'NO DATA FOR {} NAPS PARAMETER NAME ON SHEET. SKIPPING\n'.format(NAPS_code)
                    continue 

                #do some setup work for handling variety of PMSpeciation file formats
                if ('/PMSpeciation/' in data_file) & ('_SPEC.' not in data_file):
                    old_parsed_data = copy.deepcopy(parsed_data)
                    parsed_data = {'start_dt':[],'end_dt':[],'measurement':[]}
                    if 'lod' in old_parsed_data:
                        parsed_data['lod'] = []

                #for pm speciated C,EC,OC need to do work to get corrected parameters
                #for C, OC subtract cartridge B (Teflon and Quartz Filter) from cartridge A (Quartz Filter)
                if ('_CARB.xls' in data_file) or ('_CARB.XLS' in data_file):
                    #if have no cartridge data array then continue to next sheet
                    if 'cartridge' not in old_parsed_data:
                        continue

                    if self.run_details['parameter'] in ['C','OC']:
                        #get all unique datetime pairs
                        pair_dts = np.array([[sdt, edt] for sdt,edt in zip(old_parsed_data['start_dt'], old_parsed_data['end_dt'])])
                        unique_pair_dts = [list(pair_dt) for pair_dt in dict.fromkeys(tuple(pair_dt) for pair_dt in pair_dts)]
                        for unique_pair_dt in unique_pair_dts:
                            start_dt = unique_pair_dt[0]
                            end_dt = unique_pair_dt[1]
                            where_a = np.where((old_parsed_data['start_dt'] == start_dt) & (old_parsed_data['end_dt'] == end_dt) & (old_parsed_data['cartridge'] == 'A'))[0]
                            where_b = np.where((old_parsed_data['start_dt'] == start_dt) & (old_parsed_data['end_dt'] == end_dt) & (old_parsed_data['cartridge'] == 'B'))[0]
                            if (len(where_a) == 1) & (len(where_b) == 1):
                                obs_a = old_parsed_data['measurement'][where_a[0]]
                                obs_b = old_parsed_data['measurement'][where_b[0]]
                                if (obs_a == '') or (obs_b == ''):
                                    continue
                                parsed_data['measurement'].append(float(obs_a) - float(obs_b))
                                parsed_data['start_dt'].append(start_dt)
                                parsed_data['end_dt'].append(end_dt)
                                if 'lod' in parsed_data:
                                    parsed_data['lod'].append(old_parsed_data['lod'][where_a[0]])
                                            
                    #for EC just take cartridge A
                    elif self.run_details['parameter'] == 'EC':  
                        where_a = np.where(old_parsed_data['cartridge'] == 'A')[0] 
                        parsed_data['measurement'].extend(old_parsed_data['measurement'][where_a])
                        parsed_data['start_dt'].extend(old_parsed_data['start_dt'][where_a])
                        parsed_data['end_dt'].extend(old_parsed_data['end_dt'][where_a])
                        if 'lod' in parsed_data:
                            parsed_data['lod'].extend(old_parsed_data['lod'][where_a])

                    #set method information per NAPS code
                    if NAPS_code in ['EC_T','OC_T']:
                        parsed_metadata['analytical instrument'] = 'NIOSH thermal-optical analysis'               
                    elif NAPS_code in ['EC_R','OC_R']:
                        parsed_metadata['analytical instrument'] = 'IMPROVE thermal-optical analysis'
                    else:
                        parsed_metadata['analytical instrument'] = 'thermal-optical analysis'

                #for pm speciated IC measurements, just keep measurements associated with Teflon filter
                elif ('_IC.xls' in data_file) or ('_IC.XLS' in data_file):
                    where_t = np.where((old_parsed_data['media'] == 'T') & (old_parsed_data['cartridge'] == 'C'))[0]
                    parsed_data['measurement'].extend(old_parsed_data['measurement'][where_t])  
                    parsed_data['start_dt'].extend(old_parsed_data['start_dt'][where_t])
                    parsed_data['end_dt'].extend(old_parsed_data['end_dt'][where_t])
                    if 'lod' in parsed_data:
                        parsed_data['lod'].extend(old_parsed_data['lod'][where_t])              
                    #met method information
                    parsed_metadata['sample preparation'] = 'teflon filter pack'
                    parsed_metadata['analytical instrument'] = 'ion chromatography'

                #for pm speciated NA and NH4 measurements, set method and sample preparation, and also cut for valid cartridge
                elif ('_NA.xls' in data_file) or ('_NA.XLS' in data_file) or ('_NH4.xls' in data_file) or ('_NH4.XLS' in data_file):
                    where_c = np.where(old_parsed_data['media'] == 'C')[0]
                    parsed_data['measurement'].extend(old_parsed_data['measurement'][where_c])  
                    parsed_data['start_dt'].extend(old_parsed_data['start_dt'][where_c])
                    parsed_data['end_dt'].extend(old_parsed_data['end_dt'][where_c])
                    if 'lod' in parsed_data:
                        parsed_data['lod'].extend(old_parsed_data['lod'][where_c])  
                    #set method information
                    parsed_metadata['sample preparation'] = 'denuder'
                    parsed_metadata['analytical instrument'] = 'ion chromatography'

                #for pm speciated ICPMS measurements, measurements are fine coarse fractions
                #for pm10 speciated measurements combine fine and coarse fractions
                elif ('_ICPMS.xls' in data_file) or ('_ICPMS.XLS' in data_file):

                    #Speciated PM2.5 - take only fine fraction
                    if self.parameter_details['matrix'] == 'pm2.5':
                        where_f = np.where(old_parsed_data['observation_type'] == 'F')[0]
                        parsed_data['measurement'].extend(old_parsed_data['measurement'][where_f])  
                        parsed_data['start_dt'].extend(old_parsed_data['start_dt'][where_f])
                        parsed_data['end_dt'].extend(old_parsed_data['end_dt'][where_f])
                        if 'lod' in parsed_data:
                            parsed_data['lod'].extend(old_parsed_data['lod'][where_f])  

                    #Speciated PM10
                    elif self.parameter_details['matrix'] == 'pm10':
                        #get all unique datetime pairs
                        pair_dts = np.array([[sdt, edt] for sdt,edt in zip(old_parsed_data['start_dt'], old_parsed_data['end_dt'])])
                        unique_pair_dts = [list(pair_dt) for pair_dt in dict.fromkeys(tuple(pair_dt) for pair_dt in pair_dts)]
                        for unique_pair_dt in unique_pair_dts:
                            start_dt = unique_pair_dt[0]
                            end_dt = unique_pair_dt[1]
                            where_c = np.where((old_parsed_data['start_dt'] == start_dt) & (old_parsed_data['end_dt'] == end_dt) & (old_parsed_data['observation_type'] == 'C'))[0]
                            where_f = np.where((old_parsed_data['start_dt'] == start_dt) & (old_parsed_data['end_dt'] == end_dt) & (old_parsed_data['observation_type'] == 'F'))[0]
                            if (len(where_c) == 1) & (len(where_f) == 1):
                                obs_c = old_parsed_data['measurement'][where_c[0]]
                                obs_f = old_parsed_data['measurement'][where_f[0]]
                                if (obs_c == '') or (obs_f == ''):
                                    continue
                                parsed_data['measurement'].append(float(obs_c) + float(obs_f))
                                parsed_data['start_dt'].append(start_dt)
                                parsed_data['end_dt'].append(end_dt)
                                if 'lod' in parsed_data:
                                    parsed_data['lod'].append(old_parsed_data['lod'][where_c[0]])

                    #add method information
                    parsed_metadata['analytical instrument'] = 'icpms'

                #for SPEC file assume thermal-optical analysis for EC/OC, and ion chromatography for the rest
                elif ('_SPEC.xls' in data_file) or ('_SPEC.XLS' in data_file):  
                    if self.run_details['parameter'] in ['EC','OC']:
                        parsed_metadata['analytical instrument'] = 'thermal-optical analysis'
                    else:
                        parsed_metadata['analytical instrument'] = 'ion chromatography'
                     
                #for PartisolPM2.5, assume ion chromatography is the method (unless PM2.5, when assuming gravimetry)
                elif '/PartisolPM2.5/' in data_file:
                    parsed_metadata['sampler'] = 'PartisolPM2.5'
                    if self.run_details['parameter'] == 'PM2.5':
                        parsed_metadata['analytical instrument'] = 'gravimetry'
                    else:
                        parsed_metadata['analytical instrument'] = 'ion chromatography'

                #for DICHOT, for PM10/PM10 speciated iterate through lines adding 'coarse' and 'fine' data
                #for PM2.5/PM2.5 speciated just keep 'fine' data
                #for mass data assume gravimetry, for speciated data assume ion chromatography                
                elif '/PMDICHOT/' in data_file:
                    old_parsed_data = copy.deepcopy(parsed_data)
                    parsed_data = {'start_dt':[],'end_dt':[],'measurement':[]}
                    if 'lod' in old_parsed_data:
                        parsed_data['lod'] = []

                    #concatenate fine and coarse PM for integrated PM10 and PM10 speciated
                    if (self.run_details['parameter'] == 'PM10') or (self.parameter_details['matrix'] == 'pm10'):
                        #get all unique datetime pairs
                        pair_dts = np.array([[sdt, edt] for sdt,edt in zip(old_parsed_data['start_dt'], old_parsed_data['end_dt'])])
                        unique_pair_dts = [list(pair_dt) for pair_dt in dict.fromkeys(tuple(pair_dt) for pair_dt in pair_dts)]
                        for unique_pair_dt in unique_pair_dts:
                            start_dt = unique_pair_dt[0]
                            end_dt = unique_pair_dt[1]
                            where_c = np.where((old_parsed_data['start_dt'] == start_dt) & (old_parsed_data['end_dt'] == end_dt) & (old_parsed_data['observation_type'] == 'C'))[0]
                            where_f = np.where((old_parsed_data['start_dt'] == start_dt) & (old_parsed_data['end_dt'] == end_dt) & (old_parsed_data['observation_type'] == 'F'))[0]
                            if (len(where_c) == 1) & (len(where_f) == 1):
                                obs_c = old_parsed_data['measurement'][where_c[0]]
                                obs_f = old_parsed_data['measurement'][where_f[0]]
                                if (obs_c == '') or (obs_f == ''):
                                    continue
                                parsed_data['measurement'].append(float(obs_c) + float(obs_f))
                                parsed_data['start_dt'].append(start_dt)
                                parsed_data['end_dt'].append(end_dt)
                                if 'lod' in parsed_data:
                                    parsed_data['lod'].append(old_parsed_data['lod'][where_c[0]])

                    #for PM2.5 and PM2.5 speciated cut just for fine data
                    if (self.run_details['parameter'] == 'PM2.5') or (self.parameter_details['matrix'] == 'pm2.5'):
                        where_f = np.where(old_parsed_data['observation_type'] == 'F')[0]
                        parsed_data['measurement'].extend(old_parsed_data['measurement'][where_f])  
                        parsed_data['start_dt'].extend(old_parsed_data['start_dt'][where_f])
                        parsed_data['end_dt'].extend(old_parsed_data['end_dt'][where_f])
                        if 'lod' in parsed_data:
                            parsed_data['lod'].extend(old_parsed_data['lod'][where_f])

                    if self.run_details['parameter'] in ['PM10','PM2.5']:
                        parsed_metadata['analytical instrument'] = 'gravimetry'
                    else:
                        parsed_metadata['analytical instrument'] = 'ion chromatography'
                                  
                    
                #for IntegratedPM2.5-10, merge with Integrated PM2.5 file (where available)
                elif '/IntegratedPM2.5-10/' in data_file:
                    old_parsed_data = copy.deepcopy(parsed_data)
                    parsed_data = {'start_dt':[],'end_dt':[],'measurement':[]}
                    if 'lod' in old_parsed_data:
                        parsed_data['lod'] = []

                    data_file_pm25 = data_file.replace('/IntegratedPM2.5-10/','/IntegratedPM2.5/').replace('_PM25-10_','_PM25_')
                    #test for existence of Integrated PM2.5 file
                    if os.path.exists(data_file_pm25):
                        if NAPS_code == 'PM2.5-10':
                            pm25_NAPS_code = 'PM2.5'
                            pm25_all_NAPS_codes = ['PM2.5']
                            pm25_data_sheet = 'PM2.5'
                        else:
                            pm25_NAPS_code = copy.deepcopy(NAPS_code)
                            pm25_all_NAPS_codes = copy.deepcopy(all_NAPS_codes)
                            pm25_data_sheet = parsed_metadata['data_sheet_name']
                        pm25_parsing_obj = ParseCANADANAPSFile(data_file_pm25,self.run_details,pm25_NAPS_code,pm25_all_NAPS_codes,self.year)
                        pm25_parsing_obj.read_data()
                    #if file does not exist, continue to next sheet
                    else:
                        self.run_details['log_file_str'] += 'NO DATA FOR {} NAPS PARAMETER NAME IN RECIPROCAL PM2.5 FILE.\n'.format(NAPS_code)
                        continue

                    if len(pm25_parsing_obj.all_parsed_data) == 0:
                        self.run_details['log_file_str'] += 'NO VALID DATA SHEETS IN RECIPROCAL PM2.5 FILE.\n'

                    #read PM2.5 data sheet that currently have for PM2.5-10
                    for parsed_data_pm25, parsed_metadata_pm25 in zip(pm25_parsing_obj.all_parsed_data,pm25_parsing_obj.all_parsed_metadata):
                        if parsed_metadata_pm25['data_sheet_name'] == pm25_data_sheet:
                            #get all unique datetime pairs in PM2.5-10 data
                            pair_dts = np.array([[sdt, edt] for sdt,edt in zip(old_parsed_data['start_dt'], old_parsed_data['end_dt'])])
                            unique_pair_dts = [list(pair_dt) for pair_dt in dict.fromkeys(tuple(pair_dt) for pair_dt in pair_dts)]

                            #do quick sanity check for units, break if units are not the same for PM2.5-10 and PM2.5
                            if parsed_metadata['units'] != parsed_metadata_pm25['units']:
                                self.run_details['log_file_str'] += 'PMFINE AND PMCOARSE UNITS DO NOT MATCH: {} and {}'.format(parsed_metadata_pm25['units'], parsed_metadata['units'])
                                create_GHOST_stage_logfile(2,self.run_details)

                            for unique_pair_dt in unique_pair_dts:
                                start_dt = unique_pair_dt[0]
                                end_dt = unique_pair_dt[1]
                                #get matching pm2.5 time data
                                where_pm25_10 = np.where((old_parsed_data['start_dt'] == start_dt) & (old_parsed_data['end_dt'] == end_dt))[0]
                                where_pm25 = np.where((parsed_data_pm25['start_dt'] == start_dt) & (parsed_data_pm25['end_dt'] == end_dt))[0]
                                if (len(where_pm25_10) == 1) & (len(where_pm25) == 1):
                                    obs_pm25_10 = old_parsed_data['measurement'][where_pm25_10[0]]
                                    obs_pm25 = parsed_data_pm25['measurement'][where_pm25[0]]
                                    if (obs_pm25_10 == '') or (obs_pm25 == ''):
                                        continue
                                    parsed_data['measurement'].append(float(obs_pm25_10) + float(obs_pm25))
                                    parsed_data['start_dt'].append(start_dt)
                                    parsed_data['end_dt'].append(end_dt)
                                    if 'lod' in parsed_data:
                                        parsed_data['lod'].append(old_parsed_data['lod'][where_pm25_10])
                            #break out of loop reading pm2.5 data if have parsed data
                            if len(parsed_data['start_dt']) > 0:
                                break

                #if data array is empty or all empty strings, after parse then continue to next sheet
                if (len(parsed_data['measurement']) == 0) or (np.all(parsed_data['measurement'] == '')):
                    self.run_details['log_file_str'] += 'NO DATA FOR {} NAPS PARAMETER NAME ON SHEET. SKIPPING\n'.format(NAPS_code)
                    continue 
                    
                #get unique station references (for continuous files from within file, for integrated from filename)
                #continuous data
                if data_file[-4:] == '.csv': 
                    unique_station_references_unformatted = np.unique(parsed_data['station_reference'])         
                #integrated data
                else:
                    unique_station_references_unformatted = [data_file.split('/')[-1].split('_')[0]]

                #do some corrections to station_references to make sure format is correct for mapping with external metadata
                unique_station_references = [ref[1:] if ref[0] == 'S' else ref for ref in unique_station_references_unformatted]
                unique_station_references = ['0{}'.format(ref) if len(ref) == 5 else ref for ref in unique_station_references]

                #iterate over all unique station references
                for unique_station_reference,unique_station_reference_unformatted in zip(unique_station_references,unique_station_references_unformatted):

                    self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_reference)

                    #if have variable method codes per line (only continuous pm2.5), then get all unique method codes
                    #iterate through method codes cutting data only associated with the station reference and method code (only continuous data)
                    if 'method code' in parsed_data:
                        unique_method_codes = np.unique(parsed_data['method code'])
                    else:
                        unique_method_codes = ['BLANK']
                    for unique_method_code in unique_method_codes:
                        #cut continuous data with method codes
                        if (data_file[-4:] == '.csv') & (unique_method_code != 'BLANK'): 
                            valid_inds = (parsed_data['station_reference'] == unique_station_reference_unformatted) & (parsed_data['method code'] == unique_method_code)
                            cut_parsed_data = {key:value[valid_inds] for key,value in parsed_data.items()}
                            if len(cut_parsed_data['measurement']) == 0:
                                continue 
                            parsed_metadata['method code'] = unique_method_code
                        #cut continuous data with no method codes
                        elif (data_file[-4:] == '.csv'):
                            valid_inds = parsed_data['station_reference'] == unique_station_reference_unformatted
                            cut_parsed_data = {key:value[valid_inds] for key,value in parsed_data.items()}
                        #do nothing for integrated data                        
                        else:
                            cut_parsed_data = copy.deepcopy(parsed_data)

                        #create site instances of metadata/data dictionaries
                        site_data =     get_standard_data(self.parameter_details)
                        site_metadata = get_standard_metadata(self.parameter_details)

                        #------------------------------------------------------------------------#
                        #PARSE METADATA
                        #------------------------------------------------------------------------#

                        #GHOST version
                        site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

                        #initialise string which will store all GHOST process warnings as empty            
                        site_metadata['process_warnings']['value'].append('')

                        #write known metadata  
                        site_metadata['network']['value'].append(self.run_details['network'])
                        site_metadata['data_level']['value'].append('none')   
                        site_metadata['contact_institution']['value'].append('ECCC (Environment and Climate Change Canada)')      
                        site_metadata['contact_email_address']['value'].append('open-ouvert@tbs-sct.gc.ca')    
                        site_metadata['data_licence']['value'].append('Public domain: http://open.canada.ca/en/open-government-licence-canada') 
                        #assume some datum information
                        site_metadata['ellipsoid']['value'].append('WGS 84') 
                        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
                        #known meta information
                        site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                        site_metadata['projection']['value'].append('geographic')   

                        #parse external metadata
                        #make join
                        #cut_external_metadata = self.external_metadata[np.where(self.external_metadata['NAPS_ID'] == unique_station_reference)[0]]
                        try:
                            cut_external_metadata = self.external_metadata[unique_station_reference]
                        except:
                            cut_external_metadata = 'no match found'

                        # if len(cut_external_metadata) == 1:
                        #     cut_external_metadata = cut_external_metadata[0]

                        # #non specific join - throw error
                        # elif len(cut_external_metadata) == 2:
                        #     self.run_details['log_file_str'] += 'META FILE NOT SPECIFIC FOR REFERENCE: {}'.format(unique_station_reference)
                        #     create_GHOST_stage_logfile(2,self.run_details) 

                        #cannot make join - try to make meta from other sources 
                        #for continuous data - have longitude/latitude from data file
                        #for integrated data - sometimes have station information 
                        #make synthetic dataset here
                        if cut_external_metadata == 'no match found':
                            #set metadata initially all as empty strings
                            cut_external_metadata = {'Station_Name':'', 'Longitude':'', 'Latitude':'', 'Elevation':'', 'Inlet_Height':'', 'Location_Address':'', 'Postal_Code':'', 'Scale':'', 'Land_Use':'', 'Site_Type':'', 'Timezone':''}
                            #continuous data - get unique parsed longitude and latitude and station name
                            #if more than 1 unique value, then leave longitude and latitude as empty strings --> station will be removed later
                            if data_file[-4:] == '.csv':
                                if 'PREC' in self.run_details['parameter']: # do match by station name
                                    station_name_by_data = parsed_metadata['sitename'][parsed_metadata['station_reference'].index(unique_station_reference)].lower()
                                    for station in self.external_metadata:
                                        if station_name_by_data in self.external_metadata[station]['Station_Name']['values'][-1].lower():
                                            cut_external_metadata = self.external_metadata[station]

                                else:
                                    unique_lons = np.unique(cut_parsed_data['longitude'])
                                    unique_lats = np.unique(cut_parsed_data['latitude'])
                                    unique_stn_name = np.unique(cut_parsed_data['station_name'])

                                    if (len(unique_lons) == 1) & (len(unique_lats) == 1) & (len(unique_stn_name) == 1):
                                        cut_external_metadata['Longitude'] = unique_lons[0]
                                        cut_external_metadata['Latitude'] = unique_lats[0]
                                        cut_external_metadata['Station_Name'] = unique_stn_name[0]
                            #integrated data, take relevant values from parsed metadata and put them into cut_external_meta 
                            else:
                                remapping_meta = {'station_name':'Station_Name', 'longitude':'Longitude', 'latitude':'Latitude', 'altitude':'Elevation', 'sampling_height':'Inlet_Height', 'address':'Location_Address', 'spatial_scale':'Scale', 'land_use':'Land_Use', 'site_type':'Site_Type'}
                                for meta_var, meta_val in remapping_meta.items():
                                    if meta_var in parsed_metadata:
                                        cut_external_metadata[meta_val] = parsed_metadata[meta_var]

                            #if after this process there is still no longitude or latitude then continue, as need them
                            if (cut_external_metadata['Longitude'] == '') or (cut_external_metadata['Latitude'] == ''):
                                continue

                        site_metadata['station_name']['value'].append(cut_external_metadata['Station_Name']['values'][-1])
                        site_metadata['longitude']['value'].append(cut_external_metadata['Longitude']['values'][-1])
                        site_metadata['latitude']['value'].append(cut_external_metadata['Latitude']['values'][-1])
                        site_metadata['altitude']['value'].append(cut_external_metadata['Elevation']['values'][-1])

                        #sampling height can differ per variable, do some work to get it
                        sampling_height = cut_external_metadata['Inlet_Height']['values'][-1]
                        if self.run_details['parameter'] == 'O3':
                            sampling_height_params = ['O3','OZONE']
                        elif self.run_details['parameter'] == 'NO':
                            sampling_height_params = ['NO','NOX']
                        elif self.run_details['parameter'] == 'NO2':
                            sampling_height_params = ['NO2','NOX']
                        elif self.run_details['parameter'] == 'PM10':
                            sampling_height_params = ['PM10','Dichot PM','dichot PM'] 
                        elif self.run_details['parameter'] == 'PM2.5':
                            sampling_height_params = ['PM2.5','PM 2.5','PM25','Dichot PM','dichot PM']   
                        elif self.run_details['parameter'] in ['C','EC','OC']:
                            sampling_height_params = ['Carb','particulate speciation','Partic Speciation']
                        elif self.run_details['parameter'] in ['PM10_C','PM10_EC','PM10_OC']:
                            sampling_height_params = ['Carb','particulate speciation','Partic Speciation','Dichot PM','dichot PM','PM10']
                        elif self.run_details['parameter'] in ['PM2.5_C','PM2.5_EC','PM2.5_OC']:
                            sampling_height_params = ['Carb','particulate speciation','Partic Speciation','Dichot PM','dichot PM','PM2.5','PM 2.5','PM25']
                        elif self.parameter_details['matrix'] == 'pm':
                            sampling_height_params = ['particulate speciation','Partic Speciation'] 
                        elif self.parameter_details['matrix'] == 'pm10':
                            sampling_height_params = ['particulate speciation','Partic Speciation','Dichot PM','dichot PM','PM10'] 
                        elif self.parameter_details['matrix'] == 'pm2.5':
                            sampling_height_params = ['particulate speciation','Partic Speciation','Dichot PM','dichot PM','PM2.5','PM 2.5','PM25']
                        elif self.run_details['parameter'] in ['SO2','CO','NH3']:
                            sampling_height_params = [copy.deepcopy(self.run_details['parameter'])]
                        elif self.run_details['parameter'] in ['CH4','ETHENE','ETHANE','PROPENE','PROPANE','ISOPRENE','BENZENE','TOLUENE']:
                            sampling_height_params = ['VOC']
                        else:
                            sampling_height = ''
                        #handle badly formatted sampling height meta, and get specific height for parameter
                        cut_height = True
                        if ';' in sampling_height:
                            cut_height = False
                            sampling_height_splits = [samp.strip() for samp in sampling_height.split(';')]
                            for sampling_height_split in sampling_height_splits:
                                if sampling_height_split != '':
                                    sampling_height_split_2 = sampling_height_split.split(' ')
                                    sampling_height_current = sampling_height_split_2[0]
                                    sampling_height_params_joined = ' '.join(sampling_height_split_2[1:])
                                    param_list = [param.strip() for param in sampling_height_params_joined.split(',')]
                                    if bool(set(param_list) & set(sampling_height_params)):
                                        sampling_height = copy.deepcopy(sampling_height_current)
                                        cut_height = True
                                        break    
                        elif '/' in sampling_height:
                            cut_height = False
                            sampling_height_splits = [samp.strip() for samp in sampling_height.split('/')]
                            for sampling_height_split in sampling_height_splits:
                                if sampling_height_split != '':
                                    sampling_height_split_2 = sampling_height_split.split(':')
                                    sampling_height_current = sampling_height_split_2[0]
                                    sampling_height_params_joined = sampling_height_split_2[1]
                                    param_list = [param.strip() for param in sampling_height_params_joined.split(',')]
                                    if bool(set(param_list) & set(sampling_height_params)):
                                        sampling_height = copy.deepcopy(sampling_height_current)
                                        cut_height = True
                                        break
                        elif ',' in sampling_height:
                            cut_height = False
                            sampling_height_splits = [samp.strip() for samp in sampling_height.split(',')]
                            for sampling_height_split in sampling_height_splits:
                                if sampling_height_split != '':
                                    sampling_height_split_2 = sampling_height_split.split(' ')
                                    sampling_height_current = sampling_height_split_2[0]
                                    sampling_height_params_joined = ' '.join(sampling_height_split_2[1:])
                                    param_list = [sampling_height_params_joined]
                                    if bool(set(param_list) & set(sampling_height_params)):
                                        sampling_height = copy.deepcopy(sampling_height_current)
                                        cut_height = True
                                        break

                        #if have not cut height then set it to be empty str
                        if not cut_height:
                            sampling_height = ''

                        #have valid sampling height?
                        if sampling_height != '':
                            #remove 'metre' str
                            if sampling_height[-1] == 'm':                 
                                sampling_height = sampling_height[:-1] 
                            #do check to ensure sampling height is numeric, otherwise throw error
                            try:
                                sampling_height_num = float(sampling_height)
                            except:
                                self.run_details['log_file_str'] += 'SAMPLING HEIGHT IS NOT NUMERIC: {}'.format(sampling_height)
                                create_GHOST_stage_logfile(2,self.run_details) 
                            site_metadata['sampling_height']['value'].append(sampling_height)

                        #non-standard comments
                        non_standard_fields_to_parse = {'Address':'Location_Address','Post Code':'Postal_Code'}
                        misc_str = ''
                        for field, meta_field in non_standard_fields_to_parse.items():
                            if cut_external_metadata[meta_field]['values'][-1] != '':
                                misc_str += '{}:{};'.format(field, cut_external_metadata[meta_field]['values'][-1])
                        if misc_str != '':
                            site_metadata['network_miscellaneous_details']['value'].append(misc_str[:-1])

                        #do sanity check for lod units (if have them), if fails break code
                        if 'lod' in cut_parsed_data:
                            if parsed_metadata['units'] != parsed_metadata['lod_units']:
                                self.run_details['log_file_str'] += 'LOD AND MEASUREMENT UNITS DO NOT MATCH: {} and {}'.format(parsed_metadata['units'],parsed_metadata['lod_units'])
                                create_GHOST_stage_logfile(2,self.run_details)

                        #add reported measurement unit
                        #convert raw reported units given to standard name
                        if self.run_details['parameter'] == 'PREC_ACCUM':
                            reported_units = 'mm'
                        elif '_PREC' in self.run_details['parameter']:
                            if self.metals:
                                reported_units = 'ng l-1'
                            else:
                                reported_units = 'mg l-1'
                        else:
                            reported_units = unit_dict[parsed_metadata['units']]

                        conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
                        site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

                        #process classification information
                        native_classifications = {'station_classification':cut_external_metadata['Site_Type']['values'][-1],'land_use':cut_external_metadata['Land_Use']['values'][-1],'measurement_scale':cut_external_metadata['Scale']['values'][-1]}
                        site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

                        #process measurement information
                        measurement_str = ''
                        #combine parsed measurement metadata
                        for meas_var in ['sampler','cartridge','medium','sample preparation','analytical instrument','method code']:
                            if meas_var in parsed_metadata:
                                if meas_var == 'cartridge':
                                    measurement_str = '{}_cart{}'.format(measurement_str,parsed_metadata[meas_var])
                                elif meas_var == 'method code':
                                    measurement_str = '{}_code{}'.format(measurement_str,parsed_metadata[meas_var])
                                elif meas_var == 'medium':
                                    measurement_str = '{}_med{}'.format(measurement_str,parsed_metadata[meas_var])
                                elif meas_var == 'sample preparation':
                                    measurement_str = '{}_prep{}'.format(measurement_str,parsed_metadata[meas_var])
                                else:
                                    measurement_str = '{}_{}'.format(measurement_str,parsed_metadata[meas_var])                    
                        #if have no measurement information for VOCS set as GCFID (stated in supplementary material)
                        if (measurement_str == '') & ('/VOC/' in data_file):
                            measurement_str = 'LVS_SummaCanister_GCFID'
                        #if have no measurement information for Carbonyls set as HPLC (stated in supplementary material)
                        if (measurement_str == '') & ('/Carbonyls/' in data_file):                        
                            measurement_str = 'LVS_cartDNPH_HPLC'
                        #if have continuous measurements, i.e.csv file, add a 'CON_' str to measurement str at start to make this clear
                        if data_file[-4:] == '.csv':
                            measurement_str = 'CON_{}'.format(measurement_str)
                        #limit length of measurement str    
                        measurement_str = measurement_str[:225]

                        site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(measurement_str, site_metadata, self.run_details, self.parameter_details)
                        #set measurement qa codes which are True as joined metadata str
                        site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
                        measurement_qa_flag_codes = []
                        for standard_flag_name in measurement_qa_flags.keys():
                            if measurement_qa_flags[standard_flag_name]:
                                measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
                        site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

                        #iterate through metadata variables, if any columns have not been appended to add a NaN
                        for column_header in list(site_metadata.keys()):
                            n_column = len(site_metadata[column_header]['value'])
                            #if column is empty, then append NaN
                            if n_column < 1:
                                site_metadata[column_header]['value'].append(np.NaN)
    
                        #set station_reference (provided network ID + standard measurement method abbreviation)
                        site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

                        #get data file download timestamp
                        site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

                        #add data revision timestamp
                        #if have it put it, or not set as 0001-01-01 00:00 for all files    
                        if 'file_revision_date' in parsed_metadata:
                            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(parsed_metadata['file_revision_date'])
                        else:
                            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
                        site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

                        #------------------------------------------------------------------------#
                        #PARSE TIME
                        #------------------------------------------------------------------------#

                        #time is local standard time (not correcting for daylights savings), use known station offset to UTC (from external metadata)
                        #if do not have any timezone information, get timezone for station from longitude and latitude, and then get base offset when daylights saving is not active
                        if cut_external_metadata['Timezone']['values'][-1] == '':
                            canada_tzs = {'America/Edmonton':25200, 'America/Halifax':14400, 'America/Thunder_Bay':18000, 'America/Vancouver':28800, 'Canada/Atlantic':14400, 'Canada/Central':21600, 'Canada/Eastern':18000, 'Canada/Mountain':25200, 'Canada/Newfoundland':12600, 'Canada/Pacific':28800, 'Canada/Saskatchewan':21600, 'Canada/Yukon':25200}
                            local_timezone = local_timezone_name(longitude=float(site_metadata['longitude']['value'][-1]), latitude=float(site_metadata['latitude']['value'][-1]))
                            tz_offset = canada_tzs[local_timezone]
                        #else, put into format for getting correct offset
                        else:
                            if '.5' in cut_external_metadata['Timezone']['values'][-1]:
                                tz_hour = cut_external_metadata['Timezone']['values'][-1].split('.')[0] 
                                tz_min = '30'
                            else:                               
                                tz_hour = copy.deepcopy(cut_external_metadata['Timezone']['values'][-1])
                                tz_min = '0'                        
                            if '-' in tz_hour:
                                tz_sign = '-'
                                tz_hour = tz_hour[1:]
                            else:
                                tz_sign = '+'
                            tz_offset = (60 * 60) * int(tz_hour)
                            if int(tz_min) != 0:
                                tz_offset += 60 * int(tz_min)
                            if tz_sign == '+':
                                tz_offset = -tz_offset

                        #convert datetimes to UTC
                        start_dt = [round_to_minute(dt) + datetime.timedelta(seconds=tz_offset) for dt in cut_parsed_data['start_dt']]
                        end_dt = [round_to_minute(dt) + datetime.timedelta(seconds=tz_offset) for dt in cut_parsed_data['end_dt']]

                        #convert each start measurement datetime to minutes from reference time
                        frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]
            
                        #get timedeltas between start and end of measurement window
                        temporal_resolutions = []
                        for start, end in zip(start_dt,end_dt):
                            dt_window = end - start
                            #convert hours to minutes
                            dt_window_minutes = int(dt_window.total_seconds()/60.)
                            temporal_resolutions.append(dt_window_minutes)  

                        #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
                        site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

                        #------------------------------------------------------------------------#
                        #PARSE QA/FLAGS
                        #------------------------------------------------------------------------#

                        #get dictionary necessary for mapping of CANANDA NAPS file data flags to standard data flag codes
                        data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['CANADA_NAPS']  

                        #get variables which contain data flags
                        flag_vars = [flag_var for flag_var in ['sample_type','data_flag'] if flag_var in cut_parsed_data]
                    
                        #set flags and qa lists
                        #if have no flags, set all points as un-flagged
                        if len(flag_vars) == 0:
                            flags = [[standard_data_flag_name_to_data_flag_code['Un-Flagged Data']] for i in range(len(frac_time))]
                            qa = [[] for i in range(len(frac_time))]
                        else:
                            flags = [[] for i in range(len(frac_time))]
                            qa = [[] for i in range(len(frac_time))]

                        for flag_var in flag_vars:
                            for data_flag_ii, data_flag in enumerate(cut_parsed_data[flag_var]):

                                #lists to append qa/data flags per measurement, set initially as empty to handle cases where
                                flags_per_measurement = []
                                qa_per_measurement = []

                                #set GHOST decreed validity/network decreed validity variables initially as 'V'
                                ghost_decreed_validity = 'V'
                                network_decreed_validity = 'V'

                                # map data flag/s and append
                                standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                                #get mapped standard data flags
                                standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                                #iterate through standard data flags 
                                for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                                    #check if standard data flag is empty (i.e. unmapped as non-useful)
                                    if standard_data_flag_name != '': 
                                        #get corresponding data flag code for standard data flag
                                        flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                        #append standard data flag
                                        if flag_code not in flags_per_measurement:
                                            flags_per_measurement.append(flag_code)
                                        #determine if GHOST/network decreed valid
                                        if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                            ghost_decreed_validity = 'I'
                                        if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                            network_decreed_validity = 'I'

                                #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                                if ghost_decreed_validity == 'I':
                                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                                if network_decreed_validity == 'I':
                                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])
        
                                # add timezone doubt qa flag for each measurement, if get timezone from longitude/latitude                      
                                if cut_external_metadata['Timezone'] == '':
                                    qa_per_measurement.append(standard_QA_name_to_QA_code['Timezone Doubt'])  

                                #append qa_per_measurement, flags_per_measurement to lists to write out
                                flags[data_flag_ii].extend(flags_per_measurement)
                                qa[data_flag_ii].extend(qa_per_measurement)

                        #------------------------------------------------------------------------#
                        #PARSE MEASUREMENTS
                        #------------------------------------------------------------------------#    
                        #there is no known CANADA NAPS standard for temperature and pressure
                        #if need to do conversion, assume it is 298.15 K temperature, and 1013.25 hPa following other North American networks
                        #there are sometimes temperature and pressure provided for integrated pm10/pm2.5 measurements, 
                        #but seeing as no information is provided for continuous measurements, do not use to keep things consistent 
                        #precip concentrations are reported in mg l-1, do not need conversion. Set network provided volume standard temperature/pressure to NaN
        
                        measurements = []
                        lod = []

                        standard_network_gas_volume = False
                        unknown_gas_volume = True
                        site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                        site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
            
                        #get standard reported units/quantity
                        standard_reported_units = conv_obj.output_standard_units
                        measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

                        #set flags for writing QA flags associated with assumptions in unit conversion
                        unit_conversion_standard_network = False
                        unit_conversion_educated_guess = False

                        #check if reported standard units, match required standard units
                        if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                            #mass density to mixing ratio?
                            if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                                #get conversion factor
                                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                                #do conversion with network provided sample gas temperature/pressure values
                                #if these are not set, then make educated guess (assume standards from particulates)
                                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                                    unit_conversion_standard_network = True
                                else:
                                    input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                                    unit_conversion_educated_guess = True
                                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
        
                            #mixing ratio to mass density?
                            elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                                #do conversion with network provided sample gas temperature/pressure values
                                #if these are not set, then make educated guess (assume standards from particulates)
                                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                                    unit_conversion_standard_network = True
                                else:
                                    input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                                    unit_conversion_educated_guess = True
                                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
       
                            #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                            else:
                                conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                            #process through each measurement/lod and convert value using derived conversion factor
                            for n, meas in enumerate(cut_parsed_data['measurement']):
                                #only do conversion if measurement value is above 0 and not infinite, and do not have data flag stating the measurement is missing 
                                try:
                                    float_meas = float(meas)
                                except:
                                    float_meas = np.NaN
                                if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                    measurements.append(float_meas*conv_obj.conversion_factor)
                                else:
                                    measurements.append(float_meas)

                                if 'lod' in cut_parsed_data:
                                    #only do conversion if lod value is >= 0 and not infinite
                                    lim = cut_parsed_data['lod'][n]
                                    try:
                                        float_lim = float(lim)
                                    except:
                                        float_lim = np.NaN
                                    if (float_lim >= 0.0) & (np.isfinite(float_lim)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                        lod.append(float_lim*conv_obj.conversion_factor)
                                    else:
                                        lod.append(float_lim)

                                #append sample gas volume/unit_conversion assumption QA flags if necessary
                                if standard_network_gas_volume:
                                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                                if unknown_gas_volume:
                                    qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                                if unit_conversion_standard_network:
                                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                                if unit_conversion_educated_guess:
                                    qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
            
                        #otherwise do nothing to measured/lod values
                        else:
                            for n, meas in enumerate(cut_parsed_data['measurement']):
                                try:
                                    float_meas = float(meas)
                                except:
                                    float_meas = np.NaN
                                measurements.append(float_meas)

                                if 'lod' in cut_parsed_data:
                                    lim = cut_parsed_data['lod'][n]
                                    try:
                                        float_lim = float(lim)
                                    except:
                                        float_lim = np.NaN
                                    lod.append(float_lim)

                        #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
                        valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
                        if len(valid_temporal_resolution_inds) != len(frac_time):
                            frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                            temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                            flags = [flags[x] for x in valid_temporal_resolution_inds]
                            qa = [qa[x] for x in valid_temporal_resolution_inds]
                            measurements = [measurements[x] for x in valid_temporal_resolution_inds]
                            if len(lod) > 0:
                                lod = [lod[x] for x in valid_temporal_resolution_inds]

                        #----------------------------------------------------------------------------#
                        #WRITE OUT DATA
                        #----------------------------------------------------------------------------# 
                        site_data['time']['value'] = frac_time
                        site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
                        site_data['temporal_resolution']['value'] = temporal_resolutions   
                        site_data['flag']['value'] = flags
                        site_data['qa']['value'] = qa
                        if len(lod) > 0:
                            site_data['reported_lower_limit_of_detection_per_measurement']['value'] = lod 

                        #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
                        for column_header in list(site_data.keys()):
                            n_column = len(site_data[column_header]['value'])
                            #if column is empty, then append NaN
                            if n_column == 0:
                                if site_data[column_header]['data_type'] != np.uint8:
                                    site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                                else:
                                    site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
                        #------------------------------------------------------

                        #standardise metadata/data formats and write out station data to numpy file
                        write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
                        if write_obj.written == False:
                            self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)