import copy
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseUKAIRFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, year, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.year = year
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current year
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per year'''

        #get specific data files for current year
        data_files = get_data_files(self.run_details, self.parameter_details, specific_year=self.year)
      
        #load all external metadata into memory 
        self.external_metadata = np.load(self.run_details['data_root']+'UK_AIR/metadata/processed/UK_AIR_META.npy', allow_pickle=True)

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_files

    def parse_file(self, data_file):

        '''Function to process a specific data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
        
        # iterate through all UK AIR valid parameter names for parameter
        # treat each one as separate file
        for uk_air_name in self.parameter_details['uk_air_name']:

            #create object for parsing UK AIR file, and read file
            parsing_obj = ParseUKAIRFile(data_file, self.run_details, self.parameter_details)
            parsing_obj.read_data()

            #get unique station names
            unique_station_names = np.unique(list(parsing_obj.parsed_data['start_dt'].keys()))

            #iterate over all unique station references
            for unique_station_name in unique_station_names:

                self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_name)
        
                #if no valid data then continue to next station reference
                if len(parsing_obj.parsed_data['start_dt']) == 0:
                    continue

                #get cut data variables for station
                start_dt = parsing_obj.parsed_data['start_dt'][unique_station_name]
                end_dt = parsing_obj.parsed_data['end_dt'][unique_station_name]
                measurements = parsing_obj.parsed_data['measurements'][unique_station_name]
                units = parsing_obj.parsed_data['units'][unique_station_name]
                flags = parsing_obj.parsed_data['flags'][unique_station_name]
                methods = parsing_obj.parsed_data['methods'][unique_station_name]
                
                #cut external metadata for station
                station_metadata = self.external_metadata[self.external_metadata['Site Name'] == unique_station_name]
                # if metadata is cannot be matched, continue to next station
                if len(station_metadata) == 0:
                    self.run_details['log_file_str'] += 'METADATA CANNOT BE MATCHED FOR STATION: {}. SKIPPING.\n'.format(unique_station_name)
                    continue

                # iterate through matched station metadata
                for station_ii, station_meta in enumerate(station_metadata):

                    # made specific match
                    if len(station_metadata) == 1:
                        valid_inds = np.arange(len(start_dt))

                    # if metadata is not specific for a station, then resolve match
                    # this is done by looking at station reporting start and end times
                    # then split data based upon reporting start/end date
                    elif len(station_metadata) > 1:

                        # get station reporting start time
                        if station_ii == 0:
                            station_start_date = datetime.datetime(1900,1,1,0,0)
                        else:
                            station_start_date = station_meta['Start Date']
                            if station_start_date.strip() == '':
                                station_start_date = datetime.datetime(1900,1,1,0,0)
                            else:
                                station_start_date_split = station_start_date.split('-')
                                station_start_date = datetime.datetime(int(station_start_date_split[0]),int(station_start_date_split[1]),int(station_start_date_split[2]),0,0)

                        # get station reporting end time    
                        if station_ii == (len(station_metadata)-1):
                            station_end_date = datetime.datetime(2100,1,1,0,0)
                        else:
                            station_end_date = station_meta['End Date']
                            if station_end_date.strip() == '':
                                station_end_date = datetime.datetime(2100,1,1,0,0)
                            else:
                                station_end_date_split = station_end_date.split('-')
                                station_end_date = datetime.datetime(int(station_end_date_split[0]),int(station_end_date_split[1]),int(station_end_date_split[2]),0,0)

                        # cut data based on station reporting start and end times
                        # make sure no gaps are imposed, or data duplicated, so start from where last valid inds stopped
                        valid_inds = np.where((start_dt >= station_start_date) & (end_dt < station_end_date))[0]                        
                        if station_ii == 0:
                            last_valid_ind = -1
                        else:
                            # if have a gap from last valid inds, add those in middle so have no gaps
                            if (valid_inds[0] - last_valid_ind) > 1:
                                valid_inds_to_insert = np.arange(last_valid_ind+1, valid_inds[0])
                                valid_inds = np.append(valid_inds_to_insert, np.array(valid_inds, dtype=np.uint32))
                            # elif valid inds overlap with previous valid inds, make sure it starts from last valid ind
                            # so have no duplicates 
                            elif valid_inds[0] <= last_valid_ind:
                                valid_inds = valid_inds[valid_inds > last_valid_ind]
                        
                        # if have no valid inds, then continue to next metadata match
                        if len(valid_inds) == 0:
                            continue
                        
                        # get last valid ind
                        last_valid_ind = valid_inds[-1]
                            
                    # cut data for relevant inds
                    start_dt_cut = start_dt[valid_inds]
                    end_dt_cut = end_dt[valid_inds]
                    measurements_cut = measurements[valid_inds]
                    units_cut = units[valid_inds]
                    flags_cut = flags[valid_inds]
                    methods_cut = methods[valid_inds]

                    # if have no data after the cut, then continue to next matched metadata
                    if len(start_dt_cut) == 0:
                        continue

                    #split station data by different measurement methods
                    unique_methods = np.unique(methods_cut)

                    # if number of unique methods is 2, and one is empty string, 
                    # then assume the methods are the same across all measurements
                    if (len(unique_methods) == 2) & ('' in unique_methods): 
                        unique_methods = np.delete(unique_methods, np.where(unique_methods == ''))

                    #iterate through unique measurement methods
                    for unique_method in unique_methods:
                        
                        # get relevant indices for method
                        valid_inds_method = np.where(methods_cut == unique_method)[0]
                        # if have no valid inds then continue to next method
                        if len(valid_inds_method) == 0:
                                continue

                        # cut data to that associated with method
                        start_dt_method = start_dt_cut[valid_inds_method]
                        end_dt_method = end_dt_cut[valid_inds_method]
                        measurements_method = measurements_cut[valid_inds_method]
                        units_method = units_cut[valid_inds_method]
                        flags_method = flags_cut[valid_inds_method]

                        # if all associated measurements are empty string, then continue
                        if np.all(measurements_method == ''):
                            continue

                        # split station data by different units
                        unique_units = np.unique(units_method)

                        # if number of unique units is 2, and one is empty string, 
                        # then assume the units are the same across all measurements
                        if (len(unique_units) == 2) & ('' in unique_units): 
                            unique_units = np.delete(unique_units, np.where(unique_units == ''))

                        #iterate through unique units
                        for unique_unit in unique_units:

                            # get relevant indices for units
                            valid_inds_unit = np.where(units_method == unique_unit)[0]
                            # if have no valid inds then continue to next set of units
                            if len(valid_inds_unit) == 0:
                                    continue

                            # cut data to that associated with unit
                            start_dt_method_unit = start_dt_method[valid_inds_unit]
                            end_dt_method_unit = end_dt_method[valid_inds_unit]
                            measurements_method_unit = measurements_method[valid_inds_unit]
                            flags_method_unit = flags_method[valid_inds_unit]

                            # if all associated measurements are empty string, then continue
                            if np.all(measurements_method_unit == ''):
                                continue

                            #create site instances of metadata/data dictionaries
                            site_data =     get_standard_data(self.parameter_details)
                            site_metadata = get_standard_metadata(self.parameter_details)

                            #------------------------------------------------------------------------#
                            #PARSE METADATA
                            #------------------------------------------------------------------------#

                            #GHOST version
                            site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

                            #initialise string which will store all GHOST process warnings as empty            
                            site_metadata['process_warnings']['value'].append('')

                            #write known metadata  
                            site_metadata['network']['value'].append(self.run_details['network'])
                            site_metadata['data_level']['value'].append('none')   
                            site_metadata['contact_institution']['value'].append('United Kingdom Department for Environment Food & Rural Affairs')      
                            site_metadata['contact_email_address']['value'].append('enquiries@environment-agency.gov.uk')    
                            site_metadata['data_licence']['value'].append('Public domain: https://uk-air.defra.gov.uk/about-these-pages')    

                            #assume datum information
                            site_metadata['ellipsoid']['value'].append('WGS 84') 
                            site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
                            site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                            site_metadata['projection']['value'].append('geographic')    

                            #site metadata
                            unique_station_reference = station_meta['UK-AIR ID']
                            site_metadata['longitude']['value'].append(station_meta['Longitude'])
                            site_metadata['latitude']['value'].append(station_meta['Latitude'])
                            site_metadata['altitude']['value'].append(station_meta['Altitude (m)'])
                            site_metadata['station_name']['value'].append(unique_station_name)

                            #associated networks
                            associated_network_string = ''
                            eea_reference = station_meta['EU Site ID']
                            if eea_reference.strip() != '':
                                associated_network_string = 'EEA AIRBASE:{}'.format(eea_reference)
                            ebas_reference = station_meta['EMEP Site ID']
                            if ebas_reference.strip() != '':
                                ebas_string = 'EBAS:{}'.format(ebas_reference)
                                if associated_network_string == '':
                                    associated_network_string = copy.deepcopy(ebas_string)
                                else:
                                    associated_network_string = '{};{}'.format(associated_network_string, ebas_string)
                            site_metadata['associated_networks']['value'].append(associated_network_string)

                            #station description
                            if station_meta['Site Description'].strip() != '':
                                station_description = 'Station description: {}'.format(station_meta['Site Description'])
                                site_metadata['network_miscellaneous_details']['value'].append(station_description)

                            #save reported units
                            #convert raw reported units given to standard name
                            conv_obj = unit_converter.convert_units(unique_unit, unique_unit, 1, measured_species = self.parameter_details['chemical_formula'])
                            site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

                            #parse station classifications
                            native_classifications = {'area_classification':station_meta['Environment Type']}
                            site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

                            #process measurement process information
                            site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(unique_method, site_metadata, self.run_details, self.parameter_details)
                            #set measurement qa codes which are True as joined metadata str
                            site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
                            measurement_qa_flag_codes = []
                            for standard_flag_name in measurement_qa_flags.keys():
                                if measurement_qa_flags[standard_flag_name]:
                                    measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
                            site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

                            #iterate through metadata variables, if any columns have not been appended to add a NaN
                            for column_header in list(site_metadata.keys()):
                                n_column = len(site_metadata[column_header]['value'])
                                #if column is empty, then append NaN
                                if n_column < 1:
                                    site_metadata[column_header]['value'].append(np.NaN)
                
                            #set station_reference (provided network ID + standard measurement method abbreviation)
                            site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

                            #get data file download timestamp
                            site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

                            #add data revision timestamp, set as 0001-01-01 00:00 for all files
                            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
                            site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

                            #------------------------------------------------------------------------#
                            #PARSE TIME - always given in GMT/UTC (no conversion neccessary)
                            #------------------------------------------------------------------------#   
                            #round datetimes to nearest minute
                            start_dt_method_unit = [round_to_minute(dt) for dt in start_dt_method_unit]
                            end_dt_method_unit = [round_to_minute(dt) for dt in end_dt_method_unit]

                            #convert each start measurement datetime to minutes from reference time
                            frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt_method_unit]
                        
                            #get timedeltas between start and end of measurement window
                            temporal_resolutions = []
                            for start, end in zip(start_dt_method_unit,end_dt_method_unit):
                                dt_window = end - start
                                #convert hours to minutes
                                dt_window_minutes = int(dt_window.total_seconds()/60.)
                                temporal_resolutions.append(dt_window_minutes)  

                            #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
                            site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

                            #------------------------------------------------------------------------#
                            #PARSE QA/FLAGS
                            #------------------------------------------------------------------------#

                            flags = []
                            qa = []

                            #get dictionary necessary for mapping of UK AIR file data flags to standard data flag codes
                            data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['UK_AIR']  

                            # iterate through data flags (can be multiple per measurement)
                            for data_flags in flags_method_unit:

                                #lists to append qa/data flags per measurement
                                flags_per_measurement = []
                                qa_per_measurement = []

                                #set GHOST decreed validity/network decreed validity variables initially as 'V'
                                ghost_decreed_validity = 'V'
                                network_decreed_validity = 'V'

                                # iterate through flags per measurement
                                data_flags_split = data_flags.split('|')
                                for data_flag in data_flags_split:

                                    # map data flag/s and append
                                    standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                                    #get mapped standard data flags
                                    standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                                    #iterate through standard data flags 
                                    for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                                        #check if standard data flag is empty (i.e. unmapped as non-useful)
                                        if standard_data_flag_name != '': 
                                            #get corresponding data flag code for standard data flag
                                            flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                            #append standard data flag
                                            if flag_code not in flags_per_measurement:
                                                flags_per_measurement.append(flag_code)
                                            #determine if GHOST/network decreed valid
                                            if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                                ghost_decreed_validity = 'I'
                                            if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                                network_decreed_validity = 'I'

                                #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                                if ghost_decreed_validity == 'I':
                                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                                if network_decreed_validity == 'I':
                                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])
                    
                                #append qa_per_measurement, flags_per_measurement to lists to write out
                                flags.append(flags_per_measurement)
                                qa.append(qa_per_measurement)

                            #------------------------------------------------------------------------#
                            #PARSE MEASUREMENTS
                            #------------------------------------------------------------------------#    
                            #measurement temperature and pressure values are never explicitly provided by UK AIR
                            #but there is implication that the measurements follow european standards, so assume this:
                            #https://uk-air.defra.gov.uk/networks/monitoring-methods?view=eu-standards
                            #these are defined as for 293 K temperature, and 1013 hPa for pressure here for all gas-phase compounds: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32008L0050&from=en and here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32004L0107&from=EN 
                            #for particulate matter and substances to be analysed in particulate matter (e.g. lead) the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
                            #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.

                            measurements = []

                            standard_network_gas_volume = False
                            unknown_gas_volume = False
                            if self.parameter_details['matrix'] == 'gas':
                                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '293.0'
                                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.0'
                                standard_network_gas_volume = True
                            else:
                                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
                                unknown_gas_volume = True
                                
                            #get standard reported units/quantity
                            standard_reported_units = conv_obj.output_standard_units
                            measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity 

                            #set flags for writing QA flags associated with assumptions in unit conversion
                            unit_conversion_standard_network = False
                            unit_conversion_educated_guess = False
                
                            #check if reported standard units, match required standard units
                            if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                                #mass density to mixing ratio?
                                if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                                    #get conversion factor
                                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                                    #do conversion with network provided sample gas temperature/pressure values
                                    #if these are not set, then make educated guess (assume standards from particulates)
                                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                                        unit_conversion_standard_network = True
                                    else:
                                        input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                                        unit_conversion_educated_guess = True
                                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
                    
                                #mixing ratio to mass density?
                                elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                                    #do conversion with network provided sample gas temperature/pressure values
                                    #if these are not set, then make educated guess (assume standards from particulates)
                                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                                        unit_conversion_standard_network = True
                                    else:
                                        input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                                        unit_conversion_educated_guess = True
                                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
                
                                #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                                else:
                                    conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                                #process through each measurement and convert value using derived conversion factor
                                for n, meas in enumerate(measurements_method_unit):
                                    #only do conversion if measurement value is above 0 and not infinite, and do not have data flag stating the measurement is missing 
                                    try:
                                        float_meas = float(meas)
                                    except:
                                        float_meas = np.NaN
                                    if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                        measurements.append(float_meas*conv_obj.conversion_factor)
                                    else:
                                        measurements.append(float_meas)

                                    #append sample gas volume/unit_conversion assumption QA flags if necessary
                                    if standard_network_gas_volume:
                                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                                    if unknown_gas_volume:
                                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                                    if unit_conversion_standard_network:
                                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                                    if unit_conversion_educated_guess:
                                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
                        
                            #otherwise do nothing to measured/lod values
                            else:
                                for n, meas in enumerate(measurements_method_unit):
                                    try:
                                        float_meas = float(meas)
                                    except:
                                        float_meas = np.NaN
                                    measurements.append(float_meas)

                            #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
                            valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
                            if len(valid_temporal_resolution_inds) != len(frac_time):
                                frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                                temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                                flags = [flags[x] for x in valid_temporal_resolution_inds]
                                qa = [qa[x] for x in valid_temporal_resolution_inds]
                                measurements = [measurements[x] for x in valid_temporal_resolution_inds]

                            #----------------------------------------------------------------------------#
                            #WRITE OUT DATA
                            #----------------------------------------------------------------------------# 
                            site_data['time']['value'] = frac_time
                            site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
                            site_data['temporal_resolution']['value'] = temporal_resolutions   
                            site_data['flag']['value'] = flags
                            site_data['qa']['value'] = qa

                            #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
                            for column_header in list(site_data.keys()):
                                n_column = len(site_data[column_header]['value'])
                                #if column is empty, then append NaN
                                if n_column == 0:
                                    if site_data[column_header]['data_type'] != np.uint8:
                                        site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                                    else:
                                        site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
                            #------------------------------------------------------

                            #standardise metadata/data formats and write out station data to numpy file
                            write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
                            if write_obj.written == False:
                                self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {} (METHOD:{}, UNITS:{}). SKIPPING.\n'.format(unique_station_name, unique_method, unique_unit)