import codecs
import copy
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from standardisation_mapping_functions import EEA_AQ_eReporting_external_meta_map, EEA_AQ_eReporting_AIDE_D_external_meta_map, EEA_AIRBASE_external_meta_map, map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific station into a GHOST standard numpy station specific file'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files of station reference to parse
        data_files = self.gather_data_files()

        #parse station reference data files
        for data_file in data_files:
            self.parse_file(data_file)

        #if stage has completed successfully, with some valid data, write a valid process log file
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #else, if no data has been written for station write an error log file to inform station is not continuing in the pipeline
        elif not self.written_data:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get data files specific to station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)
    
        #sort files in date order
        data_file_years = [int(data_file.split('/')[-1].split('_')[3]) for data_file in data_files]
        data_files = np.array([x for _,x in sorted(zip(data_file_years,data_files))]) 
        data_file_years = sorted(data_file_years)    

        #load all external metadata into memory (3 files for EEA_AQ_eReporting) -- joins to specific station will be done later
        self.external_metadata = np.load(self.run_details['data_root']+'EEA_AQ_eReporting/metadata/processed/EEA_AQ_eReporting_META.npy', allow_pickle=True)
        self.external_metadata_AIDE_D = np.load(self.run_details['data_root']+'EEA_AQ_eReporting/metadata/processed/EEA_AQ_eReporting_AIDE_D_META.npy', allow_pickle=True)
        self.external_metadata_AIRBASE = np.load(self.run_details['data_root']+'EEA_AIRBASE/metadata/processed/EEA_AIRBASE_META.npy', allow_pickle=True)
    
        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        #set flag to know when have written data for station    
        self.written_data = False

        return data_files


    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
    
        #get size of file in bytes
        filesize = os.path.getsize(data_file)
        #if filesize is <= 32 bytes do not read as must be corrupt
        if filesize <= 32:
            self.run_details['log_file_str'] += 'FILE: {} IS NOT FORMATTED CORRECTLY. SKIPPING.\n'.format(data_file) 
            return

        #get first 32 bytes of file and determine encoding (UTF-8 or UTF-16)
        raw = open(data_file, 'rb').read(32)
        if raw.startswith(codecs.BOM_UTF16):
            encoding = 'utf-16'
        else:
            encoding = 'utf-8'

        with open(data_file, 'r', encoding=encoding) as f:  

            for index,line in enumerate(f): 
        
                #strip newline
                line = line.strip("\r\n")
        
                #get column names
                if index == 0:
                    column_names = line.split(',')
                    file_data = [[] for _ in range(len(column_names))]

                #get Data
                elif index >= 1:
                    line_split = line.split(',')
                    for column_name in column_names:
                        try:
                            file_data[column_names.index(column_name)].append(line_split[column_names.index(column_name)])   
                        except:
                            self.run_details['log_file_str'] += 'FILE: {} IS NOT FORMATTED CORRECTLY. SKIPPING.\n'.format(data_file)
                            return
        
        #check have some valid data
        if len(file_data[0]) == 0:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {}. SKIPPING.\n'.format(data_file)
            return
    
        #convert file_data to numpy record array
        try:
            file_data = np.rec.fromarrays(file_data, names=column_names)
        #have no data in file trying to read
        except:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {}. SKIPPING.\n'.format(data_file)
            return
                        
        #check species codes inside file are the correct ones for the species processing
        unique_file_species_codes, counts = np.unique(file_data['AirPollutantCode'], return_counts=True)
        non_empty = unique_file_species_codes != ''
        unique_file_species_codes, counts = unique_file_species_codes[non_empty], counts[non_empty]    
        modal_unique_species_code = unique_file_species_codes[np.argmax(counts)]
        try:
            if (int(modal_unique_species_code.split('/')[-1]) != int(self.parameter_details['airbase_code'])) or (len(unique_file_species_codes) > 1):
                self.run_details['log_file_str'] += 'INCORRECT SPECIES CODE INSIDE FILE FOR SPECIES PROCESSING. SKIPPING.\n'
                return
        except:
            self.run_details['log_file_str'] += 'FILE: {} IS NOT FORMATTED CORRECTLY. SKIPPING.\n'.format(data_file)
            return
           
        #Group data in file by "AirQualityStation" EEA Code
        #Do not group any lines which do not have station code
        all_grouped_data = []
        unique_eeacodes = np.unique(file_data['AirQualityStation'])
        non_empty = unique_eeacodes != ''
        unique_eeacodes = unique_eeacodes[non_empty]
        for unique_eeacode in unique_eeacodes:
            grouped_data_by_eeacode = file_data[np.where(file_data['AirQualityStation'] == unique_eeacode)[0]]

            #Group data in grouped data by SamplingPoint
            unique_sampling_points = np.unique(grouped_data_by_eeacode['SamplingPoint'])
            for unique_sampling_point in unique_sampling_points:
                grouped_data_by_eeacode_samplingpoint = grouped_data_by_eeacode[np.where(grouped_data_by_eeacode['SamplingPoint'] == unique_sampling_point)[0]]
                
                #Group data in file by SamplingProcess
                unique_sampling_processes = np.unique(grouped_data_by_eeacode_samplingpoint['SamplingProcess'])
                for unique_sampling_process in unique_sampling_processes:
                    all_grouped_data.append(grouped_data_by_eeacode_samplingpoint[np.where(grouped_data_by_eeacode_samplingpoint['SamplingProcess'] == unique_sampling_process)[0]])

        #iterate through grouped data
        for grouped_data in all_grouped_data:
            
            #create site instances of metadata/data dictionaries
            site_data =     get_standard_data(self.parameter_details)
            site_metadata = get_standard_metadata(self.parameter_details)

            #get eeacode/sampling point/sampling process/air pollutant code of group (will all be unique, so take first values in group)
            eeacode = grouped_data['AirQualityStation'][0]
            data_sampling_point = grouped_data['SamplingPoint'][0] 
            data_sampling_process = grouped_data['SamplingProcess'][0]
            data_species_code = grouped_data['AirPollutantCode'][0]
    
            #create a dictionary with unique units associated with indices in the file for conversion 
            unique_measurement_units = np.unique(grouped_data['UnitOfMeasurement'])
            unit_dict = {}
            for units in unique_measurement_units:
                if units == '':
                    continue
                #convert micro character (u'\xb5') to 'u'
                elif u'\xb5' in units:
                    changed_units = units.replace(u'\xb5','u')
                    unit_dict[changed_units] = np.where(grouped_data['UnitOfMeasurement'] == units)[0]
                else:
                    unit_dict[units] = np.where(grouped_data['UnitOfMeasurement'] == units)[0]
        
            #get external metadata specific for station & species (data is retrieved from 3 files)
    
            #ready other variables that will use for making join between data and metadata
            #get modal eoicode in group
            unique_eoicodes, counts = np.unique(grouped_data['AirQualityStationEoICode'], return_counts=True)   
            eoicode = unique_eoicodes[np.argmax(counts)]
            #check if eoicode is provided as an empty string
            if eoicode == '':
                #if empty string then get estimate eoicode by taking all characters after underscores/dashes in eeacode to be the reference code
                #if >= 7 characters then keep
                est_eoicode = eeacode.split('-')[-1].split('_')[-1] 
                if len(est_eoicode) >= 7:
                    eoicode = copy.deepcopy(est_eoicode) 
            #if not empty string then only take data after underscores/dashes to remove issues with incorrectly formatted codes
            else:
                eoicode = eoicode.split('-')[-1].split('_')[-1]
            pollutant_AIDE_D_format = self.parameter_details['airbase_name']
            data_species_code_AIRBASE_format = data_species_code.split('/')[-1]

            #STANDARD EXTERNAL METADATA FILE              
            #first try making data/metadata join with standard external meta file using the "SamplingPoint" (if not empty string)
            if data_sampling_point != '':
                cut_external_metadata_standard = self.external_metadata[np.where(self.external_metadata['SamplingPoint'] == data_sampling_point)[0]]
            else:
                cut_external_metadata_standard = []     

            #if cut_external_metadata_standard is empty or not specific - next try doing join with "SamplingProcess" (if not empty string)
            if data_sampling_process != '':
                if len(cut_external_metadata_standard) == 0:
                     cut_external_metadata_standard = self.external_metadata[np.where(self.external_metadata['SamplingProcess'] == data_sampling_process)[0]]
                elif len(cut_external_metadata_standard) > 1:
                    test_cut_external_metadata_standard = cut_external_metadata_standard[np.where(cut_external_metadata_standard['SamplingProcess'] == data_sampling_process)[0]]
                    if len(test_cut_external_metadata_standard) > 0:
                        cut_external_metadata_standard = copy.deepcopy(test_cut_external_metadata_standard)      

            #if cut_external_metadata_standard is empty or not specific - next try doing join with "AirQualityStation" and "AirPollutantCode" (if not empty strings)
            if (eeacode != '') & (data_species_code != ''):
                if len(cut_external_metadata_standard) == 0:
                    cut_external_metadata_standard = self.external_metadata[np.intersect1d(np.where(self.external_metadata['AirQualityStation'] == eeacode)[0], np.where(self.external_metadata['AirPollutantCode'] == data_species_code)[0])]
                elif len(cut_external_metadata_standard) > 1:
                    test_cut_external_metadata_standard = cut_external_metadata_standard[np.intersect1d(np.where(cut_external_metadata_standard['AirQualityStation'] == eeacode)[0], np.where(cut_external_metadata_standard['AirPollutantCode'] == data_species_code)[0])]
                    if len(test_cut_external_metadata_standard) > 0:
                        cut_external_metadata_standard = copy.deepcopy(test_cut_external_metadata_standard)              

            #if cut_external_metadata_standard is empty or not specific - next try doing join with "AirQualityStationEoICode" and "AirPollutantCode" (if not empty strings)
            if (eoicode != '') & (data_species_code != ''):
                if len(cut_external_metadata_standard) == 0:
                    cut_external_metadata_standard = self.external_metadata[np.intersect1d(np.where(self.external_metadata['AirQualityStationEoICode'] == eoicode)[0], np.where(self.external_metadata['AirPollutantCode'] == data_species_code)[0])]
                elif len(cut_external_metadata_standard) > 1:
                    test_cut_external_metadata_standard = cut_external_metadata_standard[np.intersect1d(np.where(cut_external_metadata_standard['AirQualityStationEoICode'] == eoicode)[0], np.where(cut_external_metadata_standard['AirPollutantCode'] == data_species_code)[0])]
                    if len(test_cut_external_metadata_standard) > 0:
                        cut_external_metadata_standard = copy.deepcopy(test_cut_external_metadata_standard)              

            #check if have specific standard metadata
            if len(cut_external_metadata_standard) != 0:
                #if not specific, preferentially take first index
                cut_external_metadata_standard = cut_external_metadata_standard[0]
                #if have successfully made join with standard external meta file, overwrite eoicode retrieved for station (if not empty string)
                if cut_external_metadata_standard['AirQualityStationEoICode'] != '':
                    #only take data after underscores/dashes to remove issues with incorrectly formatted codes
                    eoicode = cut_external_metadata_standard['AirQualityStationEoICode'].split('-')[-1].split('_')[-1]      

            #AIDE D METADATA FILE
            #first try making data/metadata join with AIDE D external metadata file using the "SamplingPointId" (if not empty string)
            if data_sampling_point != '':
                cut_external_metadata_AIDE_D = self.external_metadata_AIDE_D[np.where(self.external_metadata_AIDE_D['SamplingPointId'] == data_sampling_point)[0]]
            else:
                cut_external_metadata_AIDE_D = []        

            #if cut_external_metadata_AIDE_D is empty or not specific - next try doing join with "SamplingProcess" (if not empty string)
            if data_sampling_process != '':
                if len(cut_external_metadata_AIDE_D) == 0:
                    cut_external_metadata_AIDE_D = self.external_metadata_AIDE_D[np.where(self.external_metadata_AIDE_D['SamplingProcess'] == data_sampling_process)[0]]
                elif len(cut_external_metadata_AIDE_D) > 1:
                    test_cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[np.where(cut_external_metadata_AIDE_D['SamplingProcess'] == data_sampling_process)[0]]
                    if len(test_cut_external_metadata_AIDE_D) > 0:
                        cut_external_metadata_AIDE_D = copy.deepcopy(test_cut_external_metadata_AIDE_D)        

            #if cut_external_metadata_AIDE_D is empty or not specific - next try doing join with "EoICode" and "Pollutant" (if not empty strings)
            if (eoicode != '') & (pollutant_AIDE_D_format != ''):
                if len(cut_external_metadata_AIDE_D) == 0:
                    cut_external_metadata_AIDE_D = self.external_metadata_AIDE_D[np.intersect1d(np.where(self.external_metadata_AIDE_D['EoICode'] == eoicode)[0], np.where(self.external_metadata_AIDE_D['Pollutant'] == pollutant_AIDE_D_format)[0])]
                elif len(cut_external_metadata_AIDE_D) > 1:
                    test_cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[np.intersect1d(np.where(cut_external_metadata_AIDE_D['EoICode'] == eoicode)[0], np.where(cut_external_metadata_AIDE_D['Pollutant'] == pollutant_AIDE_D_format)[0])]
                    if len(test_cut_external_metadata_AIDE_D) > 0:
                        cut_external_metadata_AIDE_D = copy.deepcopy(test_cut_external_metadata_AIDE_D)           

            #check if have specific AIDE D metadata
            if (len(cut_external_metadata_AIDE_D) != 0):
                #if not specific, preferentially take first index
                cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[0]
                #if eoicode still empty string, and have made join with AIDE D meta file, then take eoicode retrieved for station (if not empty string)
                if eoicode == '':
                    if cut_external_metadata_AIDE_D['EoICode'] != '':
                        #only take data after underscores/dashes to remove issues with incorrectly formatted codes
                        eoicode = cut_external_metadata_AIDE_D['EoICode'].split('-')[-1].split('_')[-1] 

            #AIRBASE v8 METADATA FILE 
            #first try making data/metadata join with AIRBASE external metadata file using the "station_european_code" and "component_code" (if not empty strings)
            if (eoicode != '') & (data_species_code_AIRBASE_format != ''):
                cut_external_metadata_AIRBASE = self.external_metadata_AIRBASE[np.intersect1d(np.where(self.external_metadata_AIRBASE['station_european_code'] == eoicode)[0], np.where(self.external_metadata_AIRBASE['component_code'] == data_species_code_AIRBASE_format)[0])]
            else:
                cut_external_metadata_AIRBASE = []
            if (len(cut_external_metadata_AIRBASE) == 1):
                cut_external_metadata_AIRBASE = cut_external_metadata_AIRBASE[0]
            #if cut_external_metadata_AIRBASE is not specific - next try doing join taking into account measurement dates
            elif (len(cut_external_metadata_AIRBASE) > 1):
                #select the metadata row with the latest "measurement_end_date"
                #convert string metadata dates to datetime
                #*first convert empty strings to a date much later in future (i.e. empty string = no end date)
                measurement_end_dates = cut_external_metadata_AIRBASE['measurement_end_date']
                measurement_end_dates[np.where(measurement_end_dates == '')[0]] = '3000-01-01'
                measurement_end_dates = [datetime.datetime.strptime(date, '%Y-%m-%d').date() for date in measurement_end_dates]
                #finally get index of latest end date and cut external_metadata_AIRBASE
                cut_external_metadata_AIRBASE = cut_external_metadata_AIRBASE[np.argmax(measurement_end_dates)]

            #self.run_details['log_file_str'] += 'standard: {}\n'.format(cut_external_metadata_standard['Altitude'])
            #self.run_details['log_file_str'] += 'aide d: {}\n'.format(cut_external_metadata_AIDE_D['Altitude'])
            #self.run_details['log_file_str'] += 'airbase: {}\n'.format(cut_external_metadata_AIRBASE['station_altitude'])
        
            #UNIFY 3 METADATA files information into standardised dict
            cut_unified_metadata = {}
            #first load in standard metadata variables
            if len(cut_external_metadata_standard) > 0:
                for var in self.external_metadata.dtype.names:                    
                    if var in list(EEA_AQ_eReporting_external_meta_map.keys()):
                        #get standard_name of variable
                        standard_var = EEA_AQ_eReporting_external_meta_map[var]
                        #get variable value
                        var_value = cut_external_metadata_standard[var]
                        #do not write if empty string, or 'unknown' or '-999'
                        if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                            cut_unified_metadata[standard_var] = var_value

            #next load in AIDE D metadata variables
            if len(cut_external_metadata_AIDE_D) > 0:
                for var in self.external_metadata_AIDE_D.dtype.names:                    
                    if var in list(EEA_AQ_eReporting_AIDE_D_external_meta_map.keys()):
                        #get standard_name of variable
                        standard_var = EEA_AQ_eReporting_AIDE_D_external_meta_map[var]
                        #do not proceed if already have variable written in cut_unified_metadata dict
                        if standard_var not in list(cut_unified_metadata.keys()):
                            #get variable value
                            var_value = cut_external_metadata_AIDE_D[var]
                            #do not write if empty string, or 'unknown' or '-999'
                            if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                                cut_unified_metadata[standard_var] = var_value
                
            #next load in AIRBASE metadata variables
            if len(cut_external_metadata_AIRBASE) > 0:
                for var in self.external_metadata_AIRBASE.dtype.names:                    
                    if var in list(EEA_AIRBASE_external_meta_map.keys()):
                        #get standard_name of variable
                        standard_var = EEA_AIRBASE_external_meta_map[var]
                        #do not proceed if already have variable written in cut_unified_metadata dict
                        if standard_var not in list(cut_unified_metadata.keys()):
                            #get variable value
                            var_value = cut_external_metadata_AIRBASE[var]
                            #do not write if empty string, or 'unknown' or '-999'
                            if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                                cut_unified_metadata[standard_var] = var_value

            #Check have some valid metadata
            if len(cut_unified_metadata) == 0:
                self.run_details['log_file_str'] += 'NO VALID EXTERNAL METADATA FOUND FOR {}:{} PAIRING. SKIPPING\n'.format(eeacode, data_sampling_process)
                continue

            #make sure have some valid units from data file
            if unit_dict == {}:
                no_units = True       
                #if don't, check if have units in metadata 
                if 'measurement_unit' in list(cut_unified_metadata.keys()):
                    units = cut_unified_metadata['measurement_unit']
                    if units != '':
                        #convert micro character (u'\xb5') to 'u'
                        units = units.replace(u'\xb5','u')
                        unit_dict[units] = np.arange(len(grouped_data['UnitOfMeasurement']))
                        no_units = False
                #otherwise skip file
                if no_units:
                    self.run_details['log_file_str'] += 'NO UNITS FOR {}:{} PAIRING. SKIPPING\n'.format(eeacode, data_sampling_point)
                    return

            #------------------------------------------------------------------------#
            #PARSE TIME/DATA
            #------------------------------------------------------------------------#
      
            #parse start datetime - "DatetimeBegin"  (Defines the start time: yyyy-mm-dd hh:mm:ss Z, of the measurement -- includes timezone)
            #first parse into datetime (taking into account timezone information), convert to UTC datetime, then round to nearest minute
            start_dt = [round_to_minute(dateutil.parser.parse(date).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in grouped_data['DatetimeBegin']]

            #parse end datetime - "DatetimeEnd" (Defines the end time: yyyy-mm-dd hh:mm:ss Z, of the measurement -- includes timezone)
            #first parse into datetime (taking into account timezone information), convert to UTC datetime, then round to nearest minute
            end_dt = [round_to_minute(dateutil.parser.parse(date).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in grouped_data['DatetimeEnd']]

            #convert each start measurement datetime to minutes from reference time
            frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

            #if length of time array is 0, skip to next data group
            if len(frac_time) == 0:
                self.run_details['log_file_str'] += 'NO DATA IN FILE: {} FOR {}:{} PAIRING. SKIPPING.\n'.format(data_file,eeacode,data_sampling_process)
                continue

            #get timedeltas between start and end of measurement window
            temporal_resolutions = []
            for start, end in zip(start_dt,end_dt):
                dt_window = end - start
                #convert hours to minutes
                dt_window_minutes = int(dt_window.total_seconds()/60.)
                temporal_resolutions.append(dt_window_minutes)  
                
            #parse data flags
            flags = []
            qa = []

            #get dictionary necessary for mapping of EEA AQ eReporting file data flags to standard data flag codes
            data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['EEA_AQ_eReporting']                

            #get flags from data file - Validity
            data_flags = grouped_data['Validity']

            #get verification information - informs on data verification
            verification_data = grouped_data['Verification']

            #iterate through EEA AQ eReporting file data flags and map to standard flag codes
            for data_flag, verification_flag in zip(data_flags,verification_data):

                #lists to append qa/data flags per measurement
                flags_per_measurement = []
                qa_per_measurement = []

                #set GHOST decreed validity/network decreed validity variables initially as 'V'
                ghost_decreed_validity = 'V'
                network_decreed_validity = 'V'

                # map data flag/s and append
                standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                #get mapped standard data flags
                standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                #iterate through standard data flags 
                for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                    #check if standard data flag is empty (i.e. unmapped as non-useful)
                    if standard_data_flag_name != '': 
                        #get corresponding data flag code for standard data flag
                        flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                        #append standard data flag
                        if flag_code not in flags_per_measurement:
                            flags_per_measurement.append(flag_code)
                        #determine if GHOST/network decreed valid
                        if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                            ghost_decreed_validity = 'I'
                        if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                            network_decreed_validity = 'I'
            
                #write preliminary data flags if necessary (if "Verification" variable informs that data is preliminary)
                #1 = Verified, 2 = Preliminary verified, 3 = Not verified
                if (verification_flag == '2') or (verification_flag == '3'):
                    if standard_data_flag_name_to_data_flag_code['Preliminary Data'] not in flags_per_measurement:
                        flags_per_measurement.append(standard_data_flag_name_to_data_flag_code['Preliminary Data'])
    
                #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                if ghost_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                if network_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])
        
                #append qa_per_measurement, flags_per_measurement to lists to write out
                flags.append(flags_per_measurement)
                qa.append(qa_per_measurement)
    
            #parse measurements
            measurements = []
    
            #measurement values may need converting (i.e. mass density to mixing ratio)
    
            #measurement temperature and pressure values are never provided by EEA AQ eReporting so are always assumed to be the network standard pressure and temperatures
            #these are defined as for 293 K temperature, and 1013 hPa for pressure here for all gas-phase compounds: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32008L0050&from=en and here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32004L0107&from=EN 
            #for particulate matter and substances to be analysed in particulate matter (e.g. lead) the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
            #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.
            standard_network_gas_volume = False
            unknown_gas_volume = False
            if self.parameter_details['matrix'] == 'gas':
                site_metadata['network_provided_volume_standard_temperature']['value'].append('293.0')
                site_metadata['network_provided_volume_standard_pressure']['value'].append('1013.0')
                standard_network_gas_volume = True
            else:
                site_metadata['network_provided_volume_standard_temperature']['value'].append(np.NaN)
                site_metadata['network_provided_volume_standard_pressure']['value'].append(np.NaN)
                unknown_gas_volume = True
    
            #check if reported standard units, match required standard units
            #take care to deal with multiple units in file
            order_n = []
            for reported_units in list(unit_dict.keys()):

                #set flags for writing QA flags associated with assumptions in unit conversion
                unit_conversion_standard_network = False
                unit_conversion_educated_guess = False

                #convert raw reported units given to standard name
                try:
                    conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
                #if cannot convert units to standard units, then then a chemical formula is needed for conversion of units but do not have it: ug C/m3 for NMHC
                #therefore do not write concentrations and remove units form unit_dict
                except SystemExit:
                    del unit_dict[reported_units]
                    continue
                standard_reported_units = conv_obj.output_standard_units
                measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  
    
                #cut concentration data to be that just associated with units
                cut_concentrations = grouped_data['Concentration'][unit_dict[reported_units]]

                #check if reported standard units, match required standard units
                if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                    #mass density to mixing ratio?
                    if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                        #get conversion factor
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards for gases)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
            
                    #mixing ratio to mass density?
                    elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
           
                    #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                    else:
                        conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])
    
                    #process through each measurement and convert value using derived conversion factor
                    for n, meas in enumerate(cut_concentrations):
                        actual_n = unit_dict[reported_units][n]
                        order_n.append(actual_n)
                        #only do conversion if measurement value is above 0 and not infinite, and do not have data flag stating the measurement is missing 
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                        if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[actual_n]):
                            measurements.append(float_meas*conv_obj.conversion_factor)
                        else:
                            measurements.append(float_meas)

                        #append sample gas volume/unit_conversion assumption QA flags if necessary
                        if standard_network_gas_volume:
                            qa[actual_n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                        if unknown_gas_volume:
                            qa[actual_n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                        if unit_conversion_standard_network:
                            qa[actual_n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                        if unit_conversion_educated_guess:
                            qa[actual_n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
                
                #otherwise do nothing to measured values
                else:
                    for n, meas in enumerate(cut_concentrations):
                        order_n.append(unit_dict[reported_units][n])
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                        measurements.append(float_meas)
    
            #have 0 measurements after unit conversion? if so skip to next file
            if len(measurements) == 0:
                self.run_details['log_file_str'] += 'NO VALID UNITS IN FILE: {} FOR {}:{} PAIRING. SKIPPING.\n'.format(data_file,eeacode,data_sampling_process)
                continue

            #resort measurements by original order
            measurements = [x for _,x in sorted(zip(order_n,measurements))]
    
            #filter frac_time, temporal_resolutions, flags and qa by order_n (in case there are some measurements associated unknown units in file
            order_n = sorted(order_n)
            frac_time = [frac_time[x] for x in order_n]
            temporal_resolutions = [temporal_resolutions[x] for x in order_n]
            flags = [flags[x] for x in order_n]
            qa = [qa[x] for x in order_n]

            #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
            valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
            if len(valid_temporal_resolution_inds) != len(frac_time):
                frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                flags = [flags[x] for x in valid_temporal_resolution_inds]
                qa = [qa[x] for x in valid_temporal_resolution_inds]
                measurements = [measurements[x] for x in valid_temporal_resolution_inds]
            
            #------------------------------------------------------------------------#
            #PARSE METADATA
            #------------------------------------------------------------------------#

            #GHOST version
            site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

            #initialise string which will store all GHOST process warnings as empty            
            site_metadata['process_warnings']['value'].append('')
    
            #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
            site_metadata['meta_update_stamp']['value'].append(sorted(frac_time)[0])
    
            #write known metadata  
            site_metadata['network']['value'].append(self.run_details['network'])
            #get data level from file path 
            site_metadata['data_level']['value'].append(data_file.split('original_files/')[-1].split('/')[0])
            #data licence
            site_metadata['data_licence']['value'].append('Public domain where not explicitly stated otherwise: https://www.eea.europa.eu/legal/eea-data-policy') 

            #parse unique reported measurement unit
            #can be instances where have multiple units in a file, in this case write raw units as modal occurring unit (that is not blank or cannot be converted to standard units)
            #all raw units are taken into account when converting concentrations however
            unit_n = -1
            for unit_name in list(unit_dict.keys()):
                if len(unit_dict[unit_name]) > unit_n:
                    reported_units = unit_name   
                    unit_n = len(unit_dict[unit_name])        
            #convert raw reported units given to standard name
            conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
            site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)


            #define list of variable names that require no processing that can be simply added to site_metadata
            metadata_to_add = ['latitude','longitude','altitude','sampling_height','distance_to_kerb','distance_to_building',
                              'distance_to_junction','distance_to_source','street_type','street_width','daytime_traffic_speed',
                              'daily_passing_vehicles','station_name','representative_radius','principal_investigator_institution',
                              'principal_investigator_email_address','contact_institution','contact_email_address']

            #iterate through all metadata variable names and put values into standard dictionary
            for variable_name in metadata_to_add:
                #check variable is in unified metadata dict
                if variable_name in list(cut_unified_metadata.keys()):
                    site_metadata[variable_name]['value'].append(cut_unified_metadata[variable_name].replace('"',''))

            #parse associated networks 
            associated_network_string = ''
            #first add AIRBASE with EoICode (if not empty string)
            if eoicode != '':
                associated_network_string = 'EEA AIRBASE:{}'.format(eoicode)
                #second add EBAS, if have 'emep_station' variable and is 'yes'
                if 'emep_station' in list(cut_unified_metadata.keys()):
                    if cut_unified_metadata['emep_station'] == 'yes':
                        #add EBAS with EoICode
                        ebas_string = 'EBAS:{}'.format(eoicode)
                        associated_network_string = '{};{}'.format(associated_network_string, ebas_string)
            #finally add local network with local code
            if ('local_network_code' in list(cut_unified_metadata.keys())) & ('local_reference' in list(cut_unified_metadata.keys())):
                local_string = '{}:{}'.format(cut_unified_metadata['local_network_code'],cut_unified_metadata['local_reference'])
                if associated_network_string == '':   
                    associated_network_string = copy.deepcopy(local_string)
                else:
                    associated_network_string = '{};{}'.format(associated_network_string, local_string)
            #append associated_network_string if not empty
            if associated_network_string != '':
                site_metadata['associated_networks']['value'].append(associated_network_string)

            #parse horizontal geodetic datum - take from parsed spatial reference system information (certain spatial reference systems use specific datums - i.e. 'EPSG:4979' uses 'WSG84' datum)
            if 'spatial_reference_system' in list(cut_unified_metadata.keys()):
                if cut_unified_metadata['spatial_reference_system'] == 'EPSG:4979':
                   site_metadata['ellipsoid']['value'].append('WGS 84')  
                   site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984') 
                   site_metadata['vertical_datum']['value'].append('Ellipsoidal - WGS 84') 
                   site_metadata['projection']['value'].append('geographic')
                #if spatial reference system is unknown then write handled error logfile (2)
                else:
                    self.run_details['log_file_str'] += 'UNKNOWN SPATIAL REFERENCE SYSTEM:{}'.format(cut_unified_metadata['spatial_reference_system'])
                    create_GHOST_stage_logfile(2,self.run_details)
            #if no defined spatial reference system, assume datum
            else:
                site_metadata['ellipsoid']['value'].append('WGS 84')  
                site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984') 
                site_metadata['vertical_datum']['value'].append('Ellipsoidal - WGS 84') 
                site_metadata['projection']['value'].append('geographic')
            
            #parse population (need to check to remove non-numeric characters)
            if 'population' in list(cut_unified_metadata.keys()):
                population = cut_unified_metadata['population']
                population = re.sub("[^0-9]", "", population)
                site_metadata['population']['value'].append(population) 
    
            #parse 'principal_investigator_name' (concatenate title, first name and last name, if given)
            principal_investigator_name = ''
            if 'principal_investigator_title' in list(cut_unified_metadata.keys()):
                principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_title']
            if 'principal_investigator_first_name' in list(cut_unified_metadata.keys()):
                if principal_investigator_name == '':
                    principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_first_name']
                else:
                    principal_investigator_name = '{} {}'.format(principal_investigator_name, cut_unified_metadata['principal_investigator_first_name'])
            if 'principal_investigator_last_name' in list(cut_unified_metadata.keys()):
                if principal_investigator_name == '':
                    principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_last_name']
                else:
                    principal_investigator_name = '{} {}'.format(principal_investigator_name, cut_unified_metadata['principal_investigator_last_name'])
            if principal_investigator_name != '':
                site_metadata['principal_investigator_name']['value'].append(principal_investigator_name) 
     
            #parse 'contact_name' (concatenate title, first name and last name, if given)
            contact_name = ''
            if 'contact_title' in list(cut_unified_metadata.keys()):
                contact_name = contact_name + cut_unified_metadata['contact_title']
            if 'contact_first_name' in list(cut_unified_metadata.keys()):
                if contact_name == '':
                    contact_name = contact_name + cut_unified_metadata['contact_first_name']
                else:
                    contact_name = '{} {}'.format(contact_name, cut_unified_metadata['contact_first_name'])
            if 'contact_last_name' in list(cut_unified_metadata.keys()):
                if contact_name == '':
                    contact_name = contact_name + cut_unified_metadata['contact_last_name']
                else:
                    contact_name = '{} {}'.format(contact_name, cut_unified_metadata['contact_last_name'])
            if contact_name != '':
                site_metadata['contact_name']['value'].append(contact_name) 
    
            #parse miscellaneous details
            #deal with miscellaneous sampling details
            if 'sample_location_miscellaneous' in list(cut_unified_metadata.keys()):
                miscellaneous_sampling = 'Sampling location: {}'.format(cut_unified_metadata['sample_location_miscellaneous'])
                site_metadata['network_sampling_details']['value'].append(miscellaneous_sampling) 
    
            #deal with miscellaneous sampling inlet information
            if 'length_sampling_line' in list(cut_unified_metadata.keys()):
                inlet_info = 'Length: {}'.format(cut_unified_metadata['length_sampling_line'])
                site_metadata['measuring_instrument_inlet_information']['value'].append(inlet_info)
                    
            #deal with miscellaneous maintenance details
            miscellaneous_maintenance = ''
            if 'calibration_method' in list(cut_unified_metadata.keys()):
                string_to_add = 'Calibration methodology: {}'.format(cut_unified_metadata['calibration_method'])
                miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
            if 'calibration_frequency' in list(cut_unified_metadata.keys()):
                string_to_add = 'Calibration frequency: {}'.format(cut_unified_metadata['calibration_frequency'])
                if miscellaneous_maintenance == '':
                    miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
                else:
                    miscellaneous_maintenance = '{};{}'.format(miscellaneous_maintenance, string_to_add) 
            if 'calibration_description' in list(cut_unified_metadata.keys()):
                string_to_add = 'Calibration description: {}'.format(cut_unified_metadata['calibration_description'])
                if miscellaneous_maintenance == '':
                    miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
                else:
                    miscellaneous_maintenance = '{};{}'.format(miscellaneous_maintenance, string_to_add) 
            if miscellaneous_maintenance != '':
                site_metadata['network_maintenance_details']['value'].append(miscellaneous_maintenance)
    
            #deal with other miscellaneous details
            if 'station_description' in list(cut_unified_metadata.keys()):
                station_description = 'Station description: {}'.format(cut_unified_metadata['station_description'])
                site_metadata['network_miscellaneous_details']['value'].append(station_description)
    
            #parse country code
            #country_code = grouped_data['Countrycode'][0]
            #site_metadata['country']['value'].append(country_ISO3166_to_name[country_code]) 

            #parse measurement information (from external metadata) - use all possible variables of: 'sampling_equipment', 'measurement_equipment', 'analytical_technique', 'measurement_method', 'measurement_type', 'measurement_automatic'
            if 'sampling_equipment' in list(cut_unified_metadata.keys()):
                sampling_equipment = cut_unified_metadata['sampling_equipment']
            else:
                sampling_equipment = ''
            if 'measurement_equipment' in list(cut_unified_metadata.keys()):
                #remove 'unnessary long string from variable
                measurement_equipment = cut_unified_metadata['measurement_equipment'].split('measurementequipment/')[-1]
            else:
                measurement_equipment = ''
            if 'analytical_technique' in list(cut_unified_metadata.keys()):
                analytical_technique = cut_unified_metadata['analytical_technique']
            else:
                analytical_technique = ''
            if 'measurement_method' in list(cut_unified_metadata.keys()):
                #remove 'unnessary long string from variable
                measurement_method = cut_unified_metadata['measurement_method'].split('measurementmethod/')[-1]
            else:
                measurement_method = ''
            if 'measurement_type' in list(cut_unified_metadata.keys()):
                measurement_type = cut_unified_metadata['measurement_type']
            else:
                measurement_type = ''
            #only parse 'measurement_automatic' if don't have 'measurement_type'
            if ('measurement_automatic' in list(cut_unified_metadata.keys())) & (measurement_type == ''):
                measurement_automatic = cut_unified_metadata['measurement_automatic']
                if measurement_automatic == 'yes':
                    measurement_type = 'automatic'
                elif measurement_automatic == 'no':
                    measurement_type = 'not-automatic'
            else:
                measurement_automatic = ''
            #concatenate measurement information together
            measurement_information = '{} {} {} {} {}'.format(sampling_equipment, measurement_equipment, analytical_technique, measurement_method, measurement_type)
            #strip leading/trailing white spaces, and change multiple white spaces to be singular
            measurement_information = measurement_information.strip()
            measurement_information = ' '.join(measurement_information.split())
            site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(measurement_information, site_metadata, self.run_details, self.parameter_details)
            #set measurement qa codes which are True as joined metadata str
            site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
            measurement_qa_flag_codes = []
            for standard_flag_name in measurement_qa_flags.keys():
                if measurement_qa_flags[standard_flag_name]:
                    measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
            site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

            #parse station classifications (from external metadata) - use all possible variables of: 'area_classification', 'station_classification', 'main_emission_source', 'land_use', 'area_classification_rural_subdivision', 'representative_radius'
            if 'area_classification' in list(cut_unified_metadata.keys()):
                area_classification = cut_unified_metadata['area_classification']
            else:
                area_classification = ''
            if 'station_classification' in list(cut_unified_metadata.keys()):
                station_classification = cut_unified_metadata['station_classification']
            else:
                station_classification = ''
            if 'main_emission_source' in list(cut_unified_metadata.keys()):
                main_emission_source = cut_unified_metadata['main_emission_source']
            else:
                main_emission_source = ''
            if 'land_use' in list(cut_unified_metadata.keys()):
                land_use = cut_unified_metadata['land_use']
            else:
                land_use = ''
            if 'area_classification_rural_subdivision' in list(cut_unified_metadata.keys()):
                area_classification_rural_subdivision = cut_unified_metadata['area_classification_rural_subdivision']
            else:
                area_classification_rural_subdivision = ''   
            if 'representative_radius' in list(cut_unified_metadata.keys()):
                representative_radius = cut_unified_metadata['representative_radius']
            else:
                representative_radius = ''   
            native_classifications = {'area_classification':area_classification, 'station_classification':station_classification, 'main_emission_source':main_emission_source, 'land_use':land_use, 'area_classification_rural_subdivision':area_classification_rural_subdivision, 'representative_radius':representative_radius}
            site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)
    
            #iterate through metadata variables, if any columns have not been appended to add a NaN
            for column_header in list(site_metadata.keys()):
                n_column = len(site_metadata[column_header]['value'])
                #if column is empty, then append NaN
                if n_column < 1:
                    site_metadata[column_header]['value'].append(np.NaN)
            
            #set station_reference (provided network ID + standard measurement method abbreviation)
            site_metadata['station_reference']['value'][-1] = '{}_{}'.format(eeacode,standard_method_abbreviation) 

            #get data file download timestamp
            site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

            #add data revision timestamp, set as 0001-01-01 00:00 for all files
            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
            site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

            #----------------------------------------------------------------------------#
            #WRITE OUT DATA
            #----------------------------------------------------------------------------# 
            site_data['time']['value'] = frac_time
            site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
            site_data['temporal_resolution']['value'] = temporal_resolutions   
            site_data['flag']['value'] = flags
            site_data['qa']['value'] = qa
            #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
            for column_header in list(site_data.keys()):
                n_column = len(site_data[column_header]['value'])
                #if column is empty, then append NaN
                if n_column == 0:
                    if site_data[column_header]['data_type'] != np.uint8:
                        site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                    else:
                        site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
            #------------------------------------------------------

            #standardise metadata/data formats and write out station data to numpy file
            write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)

            #update flag to state that have written data for station (if have done so)
            if write_obj.written:
                self.written_data = True
            else:
                self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)