import copy
import datetime
import itertools
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from geographic_functions import local_timezone_name, utm2dd
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseWMOWDCGGFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific station into a GHOST standard numpy station specific file'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files of station reference to parse
        data_files = self.gather_data_files()

        #parse station reference data files
        for data_file in data_files:
            self.parse_file(data_file)

        #if stage has completed successfully, with some valid data, write a valid process log file
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #else, if no data has been written for station write an error log file to inform station is not continuing in the pipeline
        elif not self.written_data:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)
    
    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get data files specific to station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        #set flag to know when have written data for station    
        self.written_data = False

        return data_files

    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
        
        #create object for parsing WMO WDCGG file, and read file
        parsing_obj = ParseWMOWDCGGFile(data_file)
        parsing_obj.read_metadata()
        #make manually exception to process 
        if (self.station_reference == 'cpt') & (parsing_obj.global_attributes['dataset_footnote'] == 'filtered'):
            self.run_details['log_file_str'] += 'NOT PROCESSING FILE: {}, AS IT IS A FILTERED VERSION OF OTHER DATASET.\n'.format(data_file)
            return
        parsing_obj.read_data()

        #check have some valid data
        if len(parsing_obj.parsed_data['value']) == 0:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {}. SKIPPING.\n'.format(data_file)
            return

        #split data by different instrument types
        unique_instruments = np.unique(parsing_obj.parsed_data['instrument'])
        #write files per unique instrument
        for unique_instrument in unique_instruments:

            #create site instances of metadata/data dictionaries
            site_data =     get_standard_data(self.parameter_details)
            site_metadata = get_standard_metadata(self.parameter_details)

            #get indices associated with unique_instrument
            instrument_inds = np.where(parsing_obj.parsed_data['instrument'] == unique_instrument)[0]

            #------------------------------------------------------------------------#
            #PARSE TIME
            #------------------------------------------------------------------------#
            data_timezone = parsing_obj.global_attributes['dataset_time_zone']
            #if dataset timezone is unknown, assume it is local standard time and try get offset from site_lst2utc field   
            if data_timezone.lower() == 'unknown':
                data_timezone = parsing_obj.global_attributes['site_lst2utc']
            #UTC
            if data_timezone == 'UTC':
                tz_offset = 0    
            #local standard time, calculate UTC offset in seconds
            else: 
                tz_sign = data_timezone[3]
                tz_hour = data_timezone[4:6]
                tz_min = data_timezone[7:9]
                tz_offset = (60 * 60) * int(tz_hour)
                if int(tz_min) != 0:
                    tz_offset += 60 * int(tz_min)
                if tz_sign == '+':
                    tz_offset = -tz_offset

            #convert datetimes to UTC
            start_dt = np.array([round_to_minute(dt) + datetime.timedelta(seconds=tz_offset) for dt in parsing_obj.parsed_data['start_dt'][instrument_inds]])
            end_dt = np.array([round_to_minute(dt) + datetime.timedelta(seconds=tz_offset) for dt in parsing_obj.parsed_data['end_dt'][instrument_inds]])

            #convert each start measurement datetime to minutes from reference time
            frac_time = np.array([datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt])

            #sort by frac_time
            frac_time_sort_inds = np.argsort(frac_time)
            frac_time = frac_time[frac_time_sort_inds]
            start_dt = start_dt[frac_time_sort_inds]
            end_dt = end_dt[frac_time_sort_inds]
            instrument_inds_sorted = instrument_inds[frac_time_sort_inds]

            #get timedeltas between start and end of measurement window
            #also save all duplicated windows (if flask measurement)
            temporal_resolutions = []
            if 'flask' in parsing_obj.global_attributes['dataset_project']:
                duplicated_window_inds = []
                current_duplicated_window_inds = []
                previous_start = datetime.datetime(1000,1,1,0,0,0)
                previous_end = datetime.datetime(1000,1,1,0,0,0)
                for time_ii, (start, end) in enumerate(zip(start_dt,end_dt)):
                    dt_window = end - start
                    #convert hours to minutes
                    dt_window_minutes = int(dt_window.total_seconds()/60.)
                    temporal_resolutions.append(dt_window_minutes) 
                    #time window duplicated?
                    if (previous_start == start) & (previous_end == end):
                        if len(current_duplicated_window_inds) == 0:
                            current_duplicated_window_inds.append(time_ii-1)
                        current_duplicated_window_inds.append(time_ii)
                    else:
                        if len(current_duplicated_window_inds) > 0:
                            duplicated_window_inds.append(np.array(current_duplicated_window_inds, dtype=np.int32))
                            current_duplicated_window_inds = []
                        previous_start = copy.deepcopy(start)
                        previous_end = copy.deepcopy(end)
            else:
                for start, end in zip(start_dt,end_dt):
                    dt_window = end - start
                    #convert hours to minutes
                    dt_window_minutes = int(dt_window.total_seconds()/60.)
                    temporal_resolutions.append(dt_window_minutes) 
            
            #------------------------------------------------------------------------#
            #FOR FLASK MEASUREMENTS, AVERAGE SIMULTANEOUS MEASUREMENTS/UNCERTAINTY
            #------------------------------------------------------------------------#
            
            raw_measurements = parsing_obj.parsed_data['value'][instrument_inds_sorted]
            raw_uncertainty = parsing_obj.parsed_data['value_unc'][instrument_inds_sorted]
            raw_flags = parsing_obj.parsed_data['QCflag'][instrument_inds_sorted]
            temporal_resolutions = np.array(temporal_resolutions, dtype=np.int32)

            #check if flask measurements
            if 'flask' in parsing_obj.global_attributes['dataset_project']:
                dropped_indices = []
                #iterate through grouped duplicate windows
                for duplicated_window in duplicated_window_inds:
                    duplicate_measurements = raw_measurements[duplicated_window]
                    duplicate_uncertainty = raw_uncertainty[duplicated_window]
                    duplicate_flags = raw_flags[duplicated_window]

                    #get inds of duplicate measurement windows which are not empty & not flagged as invalid
                    valid_duplicate_measurement_inds = np.arange(len(duplicate_measurements))[(duplicate_measurements != '') & (duplicate_flags != '3')]
                    #if have valid inds remaining take average, and set value back in measurements array
                    if len(valid_duplicate_measurement_inds) > 1: 
                        raw_measurements[duplicated_window[0]] = np.average(duplicate_measurements[valid_duplicate_measurement_inds].astype(np.float32))
                    #elif only 1 valid ind then set that value back in measurements array 
                    elif len(valid_duplicate_measurement_inds) == 1:
                        raw_measurements[duplicated_window[0]] = duplicate_measurements[valid_duplicate_measurement_inds[0]]
                    #else then set as empty in measurements array
                    else:
                        raw_measurements[duplicated_window[0]] = ''

                    #get inds of duplicate uncertainty windows which are not empty & not flagged as invalid
                    valid_duplicate_uncertainty_inds = np.arange(len(duplicate_uncertainty))[(duplicate_uncertainty != '') & (duplicate_flags != '3')]               
                    #if have valid inds remaining take average, and set value back in uncertainty array
                    if len(valid_duplicate_uncertainty_inds) > 1:
                        raw_uncertainty[duplicated_window[0]] = np.average(duplicate_uncertainty[valid_duplicate_uncertainty_inds].astype(np.float32))
                    #elif only 1 valid ind then set that value back in uncertainty array 
                    elif len(valid_duplicate_uncertainty_inds) == 1:
                        raw_uncertainty[duplicated_window[0]] = duplicate_uncertainty[valid_duplicate_uncertainty_inds[0]]
                    #else then set as empty in uncertainty array
                    else:
                        raw_uncertainty[duplicated_window[0]] = ''

                    #add indices to drop
                    dropped_indices.extend(duplicated_window[1:])

                #delete duplicated dropped indices from arrays
                raw_measurements = np.delete(raw_measurements, dropped_indices)
                raw_uncertainty = np.delete(raw_uncertainty, dropped_indices)
                raw_flags = np.delete(raw_flags, dropped_indices)
                temporal_resolutions = np.delete(temporal_resolutions, dropped_indices)
                frac_time = np.delete(frac_time, dropped_indices)
                instrument_inds_sorted = np.delete(instrument_inds_sorted, dropped_indices)

            #------------------------------------------------------------------------#
            #GET UNIQUE CHANGING METADATA
            #------------------------------------------------------------------------#
            #get all unique longitudes/latitudes/altitudes/sampling heights/calibration scales and get unique stamps
            line_meta = {'longitude':{'longitude':[]},'latitude':{'latitude':[]},'elevation':{'altitude':[]},'intake_height':{'sampling_height':[]}, 'scale':{'measuring_instrument_calibration_scale':[]}}
            meta_update_stamp_inds = []
            
            #create reduced dict of just calibration scale codes
            calibration_scale_code_dict = {key:value for key,value in parsing_obj.global_attributes.items() if ('scale_' in key) & ('_code' in key)}

            #iterate through line varying metadata
            for WMO_meta, GHOST_dict in line_meta.items():
                GHOST_meta = list(GHOST_dict.keys())[0]
                #get unique metadata across all lines
                uniques, unique_indices = np.unique(parsing_obj.parsed_data[WMO_meta][instrument_inds_sorted], return_index=True)
                #append unique indices to stamp array
                for unique, unique_index in zip(uniques, unique_indices):
                    #special processing to get calibration scale
                    if WMO_meta == 'scale':
                        if unique != '':
                            calibration_scale_index = list(calibration_scale_code_dict.keys())[list(calibration_scale_code_dict.values()).index(unique)] 
                            unique = parsing_obj.global_attributes['scale_{}_name'.format(calibration_scale_index.split('_')[1])]
                        else:
                            unique = np.NaN

                    if unique_index not in meta_update_stamp_inds:
                        meta_update_stamp_inds.append(unique_index)
                        line_meta[WMO_meta][GHOST_meta].append(unique)
                        #fill in values for other line varying metadata
                        for lmk, lmd in line_meta.items():
                            if lmk != WMO_meta:
                                lmv = list(lmd.keys())[0]
                                #special processing to get calibration scale
                                if lmk == 'scale':
                                    lmk_val = parsing_obj.parsed_data[lmk][instrument_inds_sorted[unique_index]]
                                    if lmk_val != '':
                                        calibration_scale_index = list(calibration_scale_code_dict.keys())[list(calibration_scale_code_dict.values()).index(lmk_val)] 
                                        lmk_val = parsing_obj.global_attributes['scale_{}_name'.format(calibration_scale_index.split('_')[1])]
                                    else:
                                        lmk_val = np.NaN
                                    line_meta[lmk][lmv].append(lmk_val)
                                else:
                                    line_meta[lmk][lmv].append(parsing_obj.parsed_data[lmk][instrument_inds_sorted[unique_index]])

                #have more than 1 unique metadata value?
                if len(uniques) > 1:
                    #iterate through unique values and ensure that unique value is not split apart across file
                    #if so, append metadata value and indices to stamp array where metadata is split
                    for unique in uniques:
                        where_unique = np.where(parsing_obj.parsed_data[WMO_meta][instrument_inds_sorted] == unique)[0]
                        if len(where_unique) == 1:
                            continue
                        where_unique_diffs = np.diff(where_unique)
                        if np.max(where_unique_diffs) == 1:
                            continue 
                        else:
                           changing_inds_to_append = where_unique[np.where(where_unique_diffs > 1)[0]+1]
                           for changing_ind_to_append in changing_inds_to_append:
                               if changing_ind_to_append not in meta_update_stamp_inds:
                                   meta_update_stamp_inds.append(changing_ind_to_append)
                                   line_meta[WMO_meta][GHOST_meta].append(unique)
                                   #fill in values for other line varying metadata
                                   for lmk, lmd in line_meta.items():
                                       if lmk != WMO_meta:
                                           lmv = list(lmd.keys())[0]
                                           #special processing to get calibration scale
                                           if lmk == 'scale':
                                               lmk_val = parsing_obj.parsed_data[lmk][instrument_inds_sorted[changing_ind_to_append]]
                                               if lmk_val != '':
                                                   calibration_scale_index = list(calibration_scale_code_dict.keys())[list(calibration_scale_code_dict.values()).index(lmk_val)] 
                                                   lmk_val = parsing_obj.global_attributes['scale_{}_name'.format(calibration_scale_index.split('_')[1])]
                                               else:
                                                   lmk_val = np.NaN
                                               line_meta[lmk][lmv].append(lmk_val)
                                           else:
                                               line_meta[lmk][lmv].append(parsing_obj.parsed_data[lmk][instrument_inds_sorted[changing_ind_to_append]])

            #sort stamp indices in ascending order (and also line varying metadata
            meta_update_stamp_inds = np.array(meta_update_stamp_inds, dtype=np.int32)
            meta_update_stamp_inds_xx = np.argsort(meta_update_stamp_inds)     
            meta_update_stamp_inds_sorted = meta_update_stamp_inds[meta_update_stamp_inds_xx]
            for meta_update_stamp_ind_sorted, meta_update_stamp_ind_xx in zip(meta_update_stamp_inds_sorted,meta_update_stamp_inds_xx):
                for WMO_meta, GHOST_dict in line_meta.items():
                    GHOST_meta = list(GHOST_dict.keys())[0]
                    site_metadata[GHOST_meta]['value'].append(line_meta[WMO_meta][GHOST_meta][meta_update_stamp_ind_xx])  
                #save metadata stamp
                site_metadata['meta_update_stamp']['value'].append(frac_time[meta_update_stamp_ind_sorted])
            metadata_len = len(site_metadata['meta_update_stamp']['value'])

            #------------------------------------------------------------------------#
            #PARSE REST OF METADATA
            #------------------------------------------------------------------------#

            #take metadata from 2 sources, in this priority -->
            #1. Data lines
            #2. File Header

            #GHOST version
            site_metadata['GHOST_version']['value'] = [self.run_details['ghost_version']]*metadata_len

            #initialise string which will store all GHOST process warnings as empty            
            site_metadata['process_warnings']['value'] = ['']*metadata_len

            #write known metadata  
            site_metadata['network']['value'] = [self.run_details['network']]*metadata_len
            site_metadata['data_level']['value'] = ['none']*metadata_len  
            site_metadata['data_licence']['value'] = ['Public domain: https://gaw.kishou.go.jp/policy/gaw']*metadata_len 

            #contact/principal investigator
            site_metadata['contact_name']['value'] = [parsing_obj.global_attributes['contact_1_name']]*metadata_len  
            site_metadata['principal_investigator_name']['value'] = [parsing_obj.global_attributes['contact_1_name']]*metadata_len
            if parsing_obj.global_attributes['contact_1_organization'] == '':
                site_metadata['contact_institution']['value'] = [parsing_obj.global_attributes['contributor_name']]*metadata_len 
                site_metadata['principal_investigator_institution']['value'] = [parsing_obj.global_attributes['contributor_name']]*metadata_len  
            else:    
                site_metadata['contact_institution']['value'] = [parsing_obj.global_attributes['contact_1_organization']]*metadata_len  
                site_metadata['principal_investigator_institution']['value'] = [parsing_obj.global_attributes['contact_1_organization']]*metadata_len
            site_metadata['contact_email_address']['value'] = [parsing_obj.global_attributes['contact_1_email']]*metadata_len  
            site_metadata['principal_investigator_email_address']['value'] = [parsing_obj.global_attributes['contact_1_email']]*metadata_len

            #station information
            site_metadata['station_name']['value'] = [parsing_obj.global_attributes['site_name']]*metadata_len

            #datum information (assume all but vertical datum)
            site_metadata['ellipsoid']['value'] = ['WGS 84']*metadata_len 
            site_metadata['horizontal_datum']['value'] = ['World Geodetic System 1984']*metadata_len
            site_metadata['vertical_datum']['value'] = ['tidal - mean sea level']*metadata_len
            site_metadata['projection']['value'] = ['geographic']*metadata_len   

            #add reported measurement unit
            #convert raw reported units given to standard name
            reported_units = parsing_obj.variable_attributes['value:units']
            #convert mu character (u'\u03bc') to 'u'
            reported_units = reported_units.replace(u'\u03bc','u')
            #convert micro character (u'\xb5') to 'u'
            reported_units = reported_units.replace(u'\xb5','u')
            if '-20C' in reported_units:
                reported_units = reported_units.split('-20C')[0]
                site_metadata['network_provided_volume_standard_temperature']['value'] = ['293.15']*metadata_len
                site_metadata['network_provided_volume_standard_pressure']['value'] = ['1013.25']*metadata_len
            elif '-25C' in reported_units:
                reported_units = reported_units.split('-25C')[0]
                site_metadata['network_provided_volume_standard_temperature']['value'] = ['298.15']*metadata_len
                site_metadata['network_provided_volume_standard_pressure']['value'] = ['1013.25']*metadata_len
            conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
            site_metadata['measuring_instrument_reported_units']['value'] = [conv_obj.output_standard_units]*metadata_len

            #process measurement process information
            if unique_instrument == '':
                instrument_merged = 'unknown'
            else:
                instrument_name = parsing_obj.global_attributes['instrument_{}'.format(unique_instrument)]
                instrument_method = parsing_obj.global_attributes['instrument_{}_measurement_method_name'.format(unique_instrument)]
                instrument_merged = '{}_{}'.format(instrument_name,instrument_method)
            #method uses flask?
            if 'flask' in parsing_obj.global_attributes['dataset_project']:
                instrument_merged = 'flask_{}'.format(instrument_merged)
            #limit instrument_text to 200 characters
            instrument_merged = instrument_merged[:200]
            standard_measurement_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(instrument_merged, site_metadata, self.run_details, self.parameter_details, write_to_metadata_array=False)
            for key, value in standard_measurement_metadata.items():
                site_metadata[key]['value'] = [value]*metadata_len
            #set measurement qa codes which are True as joined metadata str
            site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
            measurement_qa_flag_codes = []
            for standard_flag_name in measurement_qa_flags.keys():
                if measurement_qa_flags[standard_flag_name]:
                    measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
            site_metadata['measurement_qa_flags']['value'] = [','.join(measurement_qa_flag_codes)]*metadata_len

            #miscellaneous details 
            misc_str = ''
            if parsing_obj.global_attributes['dataset_description'] != '':
                misc_str += 'Dataset processing: {}'.format(parsing_obj.global_attributes['dataset_description'])
            if ('_hourly' in data_file) & (parsing_obj.global_attributes['dataset_hr_mean_processing'] != ''):
                if misc_str == '':
                    misc_str += 'Hourly mean processing: {}'.format(parsing_obj.global_attributes['dataset_hr_mean_processing']) 
                else:
                    misc_str += ';Hourly mean processing: {}'.format(parsing_obj.global_attributes['dataset_hr_mean_processing']) 
            elif ('_daily' in data_file) & (parsing_obj.global_attributes['dataset_da_mean_processing'] != ''):
                if misc_str == '':
                    misc_str += 'Daily mean processing: {}'.format(parsing_obj.global_attributes['dataset_da_mean_processing'])
                else:
                    misc_str += ';Daily mean processing: {}'.format(parsing_obj.global_attributes['dataset_da_mean_processing'])
            elif ('_monthly' in data_file) & (parsing_obj.global_attributes['dataset_mo_mean_processing'] != ''):
                if misc_str == '':
                    misc_str += 'Monthly mean processing: {}'.format(parsing_obj.global_attributes['dataset_mo_mean_processing'])
                else:
                    misc_str += ';Monthly mean processing: {}'.format(parsing_obj.global_attributes['dataset_mo_mean_processing'])
            if misc_str != '':
                site_metadata['network_miscellaneous_details']['value'] = [misc_str]*metadata_len

            if parsing_obj.global_attributes['sampling_frequency'] != '':
                site_metadata['network_sampling_details']['value'] = ['Native sampling frequency: {}'.format(parsing_obj.global_attributes['sampling_frequency'])]*metadata_len

            if parsing_obj.global_attributes['measurement_calibration'] != '':
                site_metadata['network_maintenance_details']['value'] = ['Calibration description: {}'.format(parsing_obj.global_attributes['measurement_calibration'])]*metadata_len

            #process station classifications
            native_classifications = {'area_classification':parsing_obj.global_attributes['dataset_aim_of_observation']}
            standard_classes = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details, write_to_metadata_array=False)
            for key, value in standard_classes.items():
                site_metadata[key]['value'] = [value]*metadata_len

            #iterate through metadata variables, if any columns have not been appended to add a NaN
            for column_header in list(site_metadata.keys()):
                n_column = len(site_metadata[column_header]['value'])
                #if column is empty, then append NaN
                if n_column < 1:
                    site_metadata[column_header]['value'] = [np.NaN] * metadata_len

            #set station_reference (provided network ID (capitalised) + standard measurement method abbreviation)
            site_metadata['station_reference']['value'] = ['{}_{}'.format(self.station_reference.upper(),standard_method_abbreviation)]*metadata_len 

            #get data file download timestamp
            site_metadata['data_download_stamp']['value'] = [get_datafile_download_timestamp(data_file)]*metadata_len

            #add data revision timestamp, from file metadata
            dataset_creation_date_split = parsing_obj.global_attributes['dataset_creation_date'].split('-')
            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(year=int(dataset_creation_date_split[0]),month=int(dataset_creation_date_split[1]),day=int(dataset_creation_date_split[2]),hour=0,minute=0))
            site_metadata['data_revision_stamp']['value'] = [data_revision_timestamp]*metadata_len

            #------------------------------------------------------------------------#
            #PARSE QA/FLAGS
            #------------------------------------------------------------------------#

            flags = []
            qa = []

            #get dictionary necessary for mapping of WMO WDCGG file data flags to standard data flag codes
            data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['WMO_WDCGG']  

            for data_flag in raw_flags:

                #lists to append qa/data flags per measurement
                flags_per_measurement = []
                qa_per_measurement = []

                #set GHOST decreed validity/network decreed validity variables initially as 'V'
                ghost_decreed_validity = 'V'
                network_decreed_validity = 'V'

                # map data flag/s and append
                standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                #get mapped standard data flags
                standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                #iterate through standard data flags 
                for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                    #check if standard data flag is empty (i.e. unmapped as non-useful)
                    if standard_data_flag_name != '': 
                        #get corresponding data flag code for standard data flag
                        flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                        #append standard data flag
                        if flag_code not in flags_per_measurement:
                            flags_per_measurement.append(flag_code)
                        #determine if GHOST/network decreed valid
                        if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                            ghost_decreed_validity = 'I'
                        if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                            network_decreed_validity = 'I'

                #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                if ghost_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                if network_decreed_validity == 'I':
                    qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])
        
                #append qa_per_measurement, flags_per_measurement to lists to write out
                flags.append(flags_per_measurement)
                qa.append(qa_per_measurement)

            #------------------------------------------------------------------------#
            #PARSE MEASUREMENTS
            #------------------------------------------------------------------------#
            #no network standard information about standard temperatures and pressures, assume they relate to ambient conditions unless stated 
            #all mass density units should provide gas volume standards
            #if need to convert units, and no standards are available make educated guess standards follow US: 298.15 K temperature, and 1013.25 hPa for pressure 
            #also provides reported uncertainty per measurement

            measurements = []
            uncertainty = []

            standard_network_gas_volume = False
            unknown_gas_volume = False
            
            if (pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):
                unknown_gas_volume = True

            #get standard reported units/quantity
            standard_reported_units = conv_obj.output_standard_units
            measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

            #set flags for writing QA flags associated with assumptions in unit conversion
            unit_conversion_standard_network = False
            unit_conversion_educated_guess = False

            #check for peace of mind that uncertainty units are same as measured units 
            if parsing_obj.variable_attributes['value:units'] != parsing_obj.variable_attributes['value_unc:units']:
                self.run_details['log_file_str'] += 'MEASUREMENT UNITS NOT THE SAME AS UNCERTAINTY UNITS'
                create_GHOST_stage_logfile(2,self.run_details)

            #check if reported standard units, match required standard units
            if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                #mass density to mixing ratio?
                if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                    #get conversion factor
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess 
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')

                #mixing ratio to mass density?
                elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')

                #otherwise, simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                else:
                    conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                #process through each measurement/uncertainty and convert values using derived conversion factor
                for n, (meas, unc) in enumerate(zip(raw_measurements, raw_uncertainty)):
                    #only do conversion if measurement value is above 0 and not infinite
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                        measurements.append(float_meas*conv_obj.conversion_factor)
                    else:
                        measurements.append(float_meas)

                    #only do conversion if uncertainty value is >= 0 and not infinite
                    try:
                        float_unc = float(unc)
                    except:
                        float_unc = np.NaN
                    if (float_unc >= 0.0) & (np.isfinite(float_unc)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                        uncertainty.append(float_unc*conv_obj.conversion_factor)
                    else:
                        uncertainty.append(float_unc)

                    #append sample gas volume/unit_conversion assumption QA flags if necessary
                    if standard_network_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                    if unknown_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                    if unit_conversion_standard_network:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                    if unit_conversion_educated_guess:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

            #otherwise do nothing to measured/uncertainty values
            else:
                for n, (meas, unc) in enumerate(zip(raw_measurements, raw_uncertainty)):
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    measurements.append(float_meas)

                    try:
                        float_unc = float(unc)
                    except:
                        float_unc = np.NaN
                    uncertainty.append(float_unc)

            #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
            valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
            if len(valid_temporal_resolution_inds) != len(frac_time):
                frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                flags = [flags[x] for x in valid_temporal_resolution_inds]
                qa = [qa[x] for x in valid_temporal_resolution_inds]
                measurements = [measurements[x] for x in valid_temporal_resolution_inds]
                uncertainty = [uncertainty[x] for x in valid_temporal_resolution_inds]

            #----------------------------------------------------------------------------#
            #WRITE OUT DATA
            #----------------------------------------------------------------------------# 
            site_data['time']['value'] = frac_time
            site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
            site_data['reported_uncertainty_per_measurement']['value'] = uncertainty
            site_data['temporal_resolution']['value'] = temporal_resolutions   
            site_data['flag']['value'] = flags
            site_data['qa']['value'] = qa
            #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
            for column_header in list(site_data.keys()):
                n_column = len(site_data[column_header]['value'])
                #if column is empty, then append NaN
                if n_column == 0:
                    if site_data[column_header]['data_type'] != np.uint8:
                        site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                    else:
                        site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
            #------------------------------------------------------

            #standardise metadata/data formats and write out station data to numpy file
            write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)

            #update flag to state that have written data for station (if have done so)
            if write_obj.written:
                self.written_data = True
            else:
                self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)