import copy
import datetime
import os
import time

import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ISDLineParser
from standardisation_mapping_functions import map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp


#define dictionary storing standard meteorological sampling heights for specific GHOST variables
standard_variable_sampling_heights = {
'WND_DIR_10M':10.0, 'WND_SPD_10M':10.0,
'TEMP_2M':2.0, 'RH_2M':2, 'SST':2.0, 'DEW_PT_2M':2.0, 'PRES_2M':2.0, 'SLP':2.0,
'PREC_ACCUM':2.0, 'SNOW_ACCUM':2.0, 'SNOW_DEPTH':2.0, 'CEILING_HEIGHT':2.0, 'VIS_DIST':2.0, 'CLOUD_CVG':2.0, 'CLOUD_CVG_FRAC':2.0
}

#define dictionary of data source codes to data source names
data_source_code_to_data_source_name = {
'1':'USAF SURFACE HOURLY observation, candidate for merge with NCEI SURFACE HOURLY (not yet merged, element cross-checks)',
'2':'NCEI SURFACE HOURLY observation, candidate for merge with USAF SURFACE HOURLY (not yet merged, failed element cross-checks)',
'3':'USAF SURFACE HOURLY/NCEI SURFACE HOURLY merged observation',
'4':'USAF SURFACE HOURLY observation',
'5':'NCEI SURFACE HOURLY observation',
'6':'ASOS/AWOS observation from NCEI',
'7':'ASOS/AWOS observation merged with USAF SURFACE HOURLY observation',
'8':'MAPSO observation (NCEI)',
'9':'Missing',
'A':'USAF SURFACE HOURLY/NCEI HOURLY PRECIPITATION merged observation, candidate for merge with NCEI SURFACE HOURLY (not yet merged, failed element cross-checks)',
'B':'NCEI SURFACE HOURLY/NCEI HOURLY PRECIPITATION merged observation, candidate for merge with USAF SURFACE HOURLY (not yet merged, failed element cross-checks)',
'C':'USAF SURFACE HOURLY/NCEI SURFACE HOURLY/NCEI HOURLY PRECIPITATION merged observation',
'D':'USAF SURFACE HOURLY/NCEI HOURLY PRECIPITATION merged observation',
'E':'NCEI SURFACE HOURLY/NCEI HOURLY PRECIPITATION merged observation',
'F':'Form OMR/1001 – Weather Bureau city office (keyed data)',
'G':'SAO surface airways observation, pre-1949 (keyed data)',
'H':'SAO surface airways observation, 1965-1981 format/period (keyed data)',
'I':'Climate Reference Network observation',
'J':'Cooperative Network observation',
'K':'Radiation Network observation',
'L':'Data from Climate Data Modernization Program (CDMP) data source',
'M':'Data from National Renewable Energy Laboratory (NREL) data source',
'N':'NCAR / NCEI cooperative effort (various national datasets)',
'O':'Summary observation created by NCEI using hourly observations that may not share the same data source flag',
}

data_source_hierarchy = {'fully-merged':3, 'partially-merged':2, 'non-merged':1}

class StandardiseNetworkData:

    '''Convert one data provider original file for a specific station into a GHOST standard numpy station specific file'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files of station reference to parse
        data_files = self.gather_data_files()

        #parse station reference data files
        for data_file in data_files:
            self.parse_file(data_file)

        #if have written data for station, return valid process code: 0
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time']) / 60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #otherwise no data has been written for station, therefore process code to state station will
        #not continue in further processing 
        else:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        # get data files specific to station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)

        # sort files in date order
        data_file_start_dates = [int(data_file.split('-')[-1]) for data_file in data_files]
        data_files = np.array([x for _, x in sorted(zip(data_file_start_dates, data_files))])

        # get dictionary necessary for mapping of NOAA file data flags to standard data flag codes
        if self.run_details['parameter'] == 'RH_2M':
            self.data_flag_map = network_data_flag_to_standard_data_flag_mapping['NOAA_ISD_CH']
        else:
            self.data_flag_map = network_data_flag_to_standard_data_flag_mapping['NOAA_ISD']

        # get metadata file
        metafile = np.load(self.run_details['data_root'] + self.run_details['network'] + '/metadata/processed/' + self.run_details[
            'network'] + '.npy')

        # find station's line in the metadata file
        name = data_files[0][-17:]
        self.metaline = metafile[(metafile['USAF'] == name[:6]) & (metafile['WBAN'] == name[7:12])]
        # successfully pulled metadata line for station
        # set station postion metadata variables   
        self.station_elv = np.NaN
        self.station_name = np.NaN
        self.icao_id = np.NaN
        if len(self.metaline) != 0:
            #station elevation    
            if self.metaline['ELEV(M)'][0] != '':
                self.station_elv = round(float(self.metaline['ELEV(M)'][0]))
            #station name
            if self.metaline['STATION NAME'][0] != '':
                self.station_name = self.metaline['STATION NAME'][0]
            #ICAO ID
            if self.metaline['ICAO'][0] != '':
                self.icao_id = self.metaline['ICAO'][0]
    
        # set flag to know when have written data for station
        self.written_data = False

        return data_files


    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file)

        # initialize list that will contain all accepted objects/lines of file
        lines = [] 

        #get assumed sampling height for parameter 
        sampling_height = standard_variable_sampling_heights[self.run_details['parameter']]
    
        #iterate through data file
        with open(data_file, encoding='ISO-8859-1') as f:
            for data_line in f.readlines():
                data_line = ISDLineParser(data_file, data_line)
                
                #first check data line has the parameter desired to be processed, otherwise don't keep line
                if not hasattr(data_line, self.parameter_details['noaa_isd_parameter_name']):
                    continue

                # if the data line has a useless report type, don't keep the line
                if data_line.code in ['FM-18', 'FM-14', 'FM-13']:
                    continue

                #add accepted line to lines list
                lines.append(data_line)

        # if length of lines == 0, then do not process file, as zero valid data
        if len(lines) == 0:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {}. SKIPPING.\n'.format(data_file)
            return

        #here handle the metadata associated with the measurement position 
        #in the data files, there are the longitude, latitude and measurement altitude
        #the data file measurement positons can vary within the data file, NOAA ISD note:
        #    "Latitude, longitude, elevation, and call letters for some locations with data from multiple sources (see data source flag above) will sometimes vary within a data file due to differences in the metadata from the originating source. 
        #    "This does not indicate that the station locations differ; only that the metadata have not yet been fully reflected in the data records.
        #handle this by getting the unique data source codes in the data file, and get the unique measurement positions associated with each unique data source code in the data file
        #implement an algorithm to get a singular measurement position, associated with a unique data source 
        #first, if there are more than 2 unique measurement positions per unique data source, first try reducing to 1 position by retaining positions with the minimum number of nan fields, and then keeping the modally occurring measurement position (first index is taken preferentially in case of a tie). 
        #next, if there are more than 2 unique data sources, take the associated measurement positions and try reducing to 1 position by retaining positions with the minimum number of nan fields, then by the prioritised data levels associated with the data source, and finally keeping the modally occurring measurement position across the codes (first index is taken preferentially in case of a tie). 

        #get unique data sources in the datafile, with associated unique measurement positions, and their relevant line indices
        unique_data_sources = {}
        #iterate through accepted datalines
        for line_ii, line in enumerate(lines):
            #is data source code a key in the dictionary?
            if line.data_source_flag not in list(unique_data_sources.keys()):
                unique_data_sources[line.data_source_flag] = {}
            mposition = '{},{},{}'.format(line.longitude,line.latitude,line.elevation)
            #is measurement position new data source code?
            if mposition not in list(unique_data_sources[line.data_source_flag].keys()):
                unique_data_sources[line.data_source_flag][mposition] = []
            #add line index to relevant data source code/measurement positon list
            unique_data_sources[line.data_source_flag][mposition].append(line_ii)

        #resolve multiple measurement positions per unique data source
        for unique_data_source in list(unique_data_sources.keys()):
            #more than 1 measurement position per data source code?
            if len(list(unique_data_sources[unique_data_source].keys())) > 1:
                #count how fields are nan in measurement position
                nan_counts = {mposition:mposition.split(',').count('nan') for mposition in list(unique_data_sources[unique_data_source].keys())} 
                #get the minimum number of nans in a measurement position
                min_nans = min(nan_counts.values())
                #keep measurement positions with the minimum number of nans
                valid_mpositions = [mposition for mposition in list(unique_data_sources[unique_data_source].keys()) if nan_counts[mposition] == min_nans]

                #now have just 1 measurement position?
                if len(valid_mpositions) == 1:
                    chosen_mposition = valid_mpositions[0]

                #else if still have more than 1 measurement position? take the modally occurring measurement position (first index is taken preferentially in case of a tie)
                elif len(valid_mpositions) > 1:
                    mposition_occurance = {mposition:len(unique_data_sources[unique_data_source][mposition]) for mposition in valid_mpositions}
                    chosen_mposition = max(mposition_occurance, key=mposition_occurance.get)
                
                #delete non-valid measurement positions, adding their relevant line numbers to relevant measurement position list
                for mposition in list(unique_data_sources[unique_data_source].keys()):
                    if mposition != chosen_mposition:
                        unique_data_sources[unique_data_source][chosen_mposition].extend(unique_data_sources[unique_data_source][mposition])
                        del unique_data_sources[unique_data_source][mposition]
                #make sure line numbers are in ascending order
                unique_data_sources[unique_data_source][chosen_mposition] = sorted(unique_data_sources[unique_data_source][chosen_mposition])

        #get data levels associated with each unique data source
        unique_data_source_data_levels = {}
        for unique_data_source in list(unique_data_sources.keys()):
            if unique_data_source in ['1','2','4','5','6','8','9','F','G','H','I','J','K','L','M','N','O']:
                unique_data_source_data_levels[unique_data_source] = 'non-merged'
            elif unique_data_source in ['A','B']:
                unique_data_source_data_levels[unique_data_source] = 'partially-merged'
            elif unique_data_source in ['3','7','C','D','E']:
                unique_data_source_data_levels[unique_data_source] = 'fully-merged'  

        #have just 1 unique data source?
        if len(list(unique_data_sources.keys())) == 1: 
            chosen_data_source = list(unique_data_sources.keys())[0]

        #else if have more than 1 unique data source? choose data source to get measurement positon from, by retaining measurement position with least nans, then by data level hierarchy, and then by modally occurring position
        elif len(list(unique_data_sources.keys())) > 1:
            #count how fields are nan in measurement position associated with data source
            nan_counts = {unique_data_source:list(unique_data_sources[unique_data_source].keys())[0].split(',').count('nan') for unique_data_source in list(unique_data_sources.keys())} 
            #get the minimum number of nans in a measurement position
            min_nans = min(nan_counts.values())
            #keep unique data sources associated with the measurement positions with the minimum number of nans
            valid_unique_data_sources = [unique_data_source for unique_data_source in list(unique_data_sources.keys()) if nan_counts[unique_data_source] == min_nans]

            #now have just 1 unique data source?
            if len(valid_unique_data_sources) == 1:
                chosen_data_source = valid_unique_data_sources[0]
            #else if have still more than 1 unique data source? retain just data sources associated with the highest hierarchal data level 
            elif len(valid_unique_data_sources) > 1:
                #get hierarchal code associated with each unique data source code data level
                unique_data_source_hierarchal_codes = {unique_data_source:data_source_hierarchy[unique_data_source_data_levels[unique_data_source]] for unique_data_source in valid_unique_data_sources}
                #get the maximum hierarchal code
                max_hierarchal_code = max(unique_data_source_hierarchal_codes.values())
                #keep unique data sources with the maximum hierarchal code
                valid_unique_data_sources = [unique_data_source for unique_data_source in valid_unique_data_sources if unique_data_source_hierarchal_codes[unique_data_source] == max_hierarchal_code]
        
                #now have just 1 unique data source?
                if len(valid_unique_data_sources) == 1:
                    chosen_data_source = valid_unique_data_sources[0]
                #else if have still more than 1 unique data source? take the data source code associated with the modally occurring measurement position (first index is taken preferentially in case of a tie)
                elif len(valid_unique_data_sources) > 1:
                    mposition_occurance = {unique_data_source:len(list(unique_data_sources[unique_data_source].values())[0]) for unique_data_source in valid_unique_data_sources}
                    chosen_data_source = max(mposition_occurance, key=mposition_occurance.get)

        #get measurement position fields associated with chosen data source
        measurement_position_split = list(unique_data_sources[chosen_data_source].keys())[0].split(',')    
        longitude = np.float64(measurement_position_split[0])
        latitude = np.float64(measurement_position_split[1])
        measurement_altitude = np.float32(measurement_position_split[2])

        #convert 'lines' to np.array
        #lines = np.array(lines)

        #self.run_details['log_file_str'] += 'N TOTAL LINES:{}\n'.format(len(lines))

        #iterate through unique data source codes, writing out specific files with the associated data attributed with these unique codes
        #for unique_data_source in list(unique_data_sources.keys()):
        #get relevant lines associated with unique data source   
        #relevant_lines = lines[np.array(unique_data_sources[unique_data_source][list(unique_data_sources[unique_data_source].keys())[0]])]
        #self.run_details['log_file_str'] += 'N RELEVANT LINES:{}\n'.format(len(relevant_lines))

        # create site instances of metadata/data dictionaries
        site_data = get_standard_data(self.parameter_details)
        site_metadata = get_standard_metadata(self.parameter_details)

        #get measurement metadata variables/qa codes
        site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information('', site_metadata, self.run_details, self.parameter_details)
        #set measurement qa codes which are True as joined metadata str
        site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
        measurement_qa_flag_codes = []
        for standard_flag_name in measurement_qa_flags.keys():
            if measurement_qa_flags[standard_flag_name]:
                measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
        site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

        # ----------------------------------------------------------------------------#
        # PROCESS METADATA
        # ----------------------------------------------------------------------------#

        # GHOST version
        site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

        #initialise string which will store all GHOST process warnings as empty            
        site_metadata['process_warnings']['value'].append('')

        #add measurement position metadata
        site_metadata['longitude']['value'].append(longitude)
        site_metadata['latitude']['value'].append(latitude)
        site_metadata['altitude']['value'].append(np.NaN)
        site_metadata['sampling_height']['value'].append(sampling_height)
        site_metadata['measurement_altitude']['value'].append(measurement_altitude)

        #assume datum information
        site_metadata['ellipsoid']['value'].append('WGS 84')
        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
        site_metadata['vertical_datum']['value'].append('tidal - mean sea level') 
        site_metadata['projection']['value'].append('geographic')

        # append station_reference now that we have the standard method abbreviation
        site_metadata['station_reference']['value'].append('{}_{}'.format(self.station_reference, standard_method_abbreviation))

        #add data level (variable)
        site_metadata['data_level']['value'].append('variable')

        # station_name (taken from metadata file)
        site_metadata['station_name']['value'].append(self.station_name)
        
        #Do not parse Country as format of code is variable!
        
        # append network's name
        site_metadata['network']['value'].append('NOAA_ISD')

        # process and add associated networks.
        # Format of field is -> USAF: 00724; WBA: 9182; ICAO:ENRS
        anet = []
        if lines[0].station_idu != '999999':
            usaf = 'USAF:' + lines[0].station_idu
            anet.append(usaf)
        if lines[0].station_idn != '99999':
            wban = 'WBAN:' + lines[0].station_idn
            anet.append(wban)
        if not pd.isnull(self.icao_id):
            icao = 'ICAO:' + self.icao_id
            anet.append(icao)
        if len(anet) != 0:
            res = ''
            for net in anet:
                res = res + net + ';'
            res = res[:-1]
            site_metadata['associated_networks']['value'].append(res)

        #get data file download timestamp
        site_metadata['data_download_stamp']['value'].append(get_datafile_download_timestamp(data_file))

        # contact institution
        site_metadata['contact_institution']['value'].append("NOAA ISD")
        # contact email
        site_metadata['contact_email_address']['value'].append("ncei.orders@noaa.gov")
        # data licence
        site_metadata['data_licence']['value'].append('Public domain if non-profit: WMO Resolution 40 NOAA Policy')  

        #add original data source to miscellaneous details
        site_metadata['network_miscellaneous_details']['value'].append('Original data source: {}'.format(data_source_code_to_data_source_name[unique_data_source]))

        # iterate through metadata variables, if any columns have not been appended to add a NaN
        for column_header in list(site_metadata.keys()):
            n_column = len(site_metadata[column_header]['value'])
            # if column is empty, then append NaN
            if n_column == 0:
                site_metadata[column_header]['value'].append(np.NaN)

        # ----------------------------------------------------------------------------#
        # PROCESS DATA
        # ----------------------------------------------------------------------------#

        #iterate through lines
        for line in lines:

            # initialize lists for flags and qa
            flags = []
            qa = []

            #set GHOST decreed validity/network decreed validity variables initially as 'V'
            ghost_decreed_validity = 'V'
            network_decreed_validity = 'V'

            #process time of measurement, according to ghost standard
            # time is always given in UTC
            year = int(line.datetime_ascii[0:4])
            month = int(line.datetime_ascii[4:6])
            day = int(line.datetime_ascii[6:8])
            hour = int(line.datetime_ascii[8:10])
            minute = int(line.datetime_ascii[10:12])
            #very few instances where minute is given as 60, so handle this (also for hour)
            if hour == 24:
                hour = 0
            if minute == 60:
                minute = 0
            date = datetime_to_fractional_minutes_from_reference(datetime.datetime(year, month, day, hour, minute))
            site_data['time']['value'].append(date)

            # get variable's measurement and append it
            meas = getattr(line, self.parameter_details['noaa_isd_parameter_name'])
            site_data[self.parameter_details['bsc_parameter_name']]['value'].append(meas)
            # all the variables are instantaneous, except rain/snow/RH, which are treated separately
            # assign 0 to the rest
            if self.run_details['parameter'] == "PREC_ACCUM":
                site_data['temporal_resolution']['value'].append(line.prec_period)
            elif self.run_details['parameter'] == "SNOW_ACCUM":
                site_data['temporal_resolution']['value'].append(line.snow_acc_period)
            elif self.run_details['parameter'] == "RH_2M":
                site_data['temporal_resolution']['value'].append(line.avg_rh_period)
            else:
                site_data['temporal_resolution']['value'].append(0) 

            # map measurement data flag/s to standard data flags
            meas_qc = getattr(line, self.parameter_details['noaa_isd_parameter_name'] + '_qc')
            #get mapped standard data flags
            standard_data_flag_names = self.data_flag_map[meas_qc]['standard_data_flag_name']
            #iterate through standard data flags 
            for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                #check if standard data flag is empty (i.e. unmapped as non-useful)
                if standard_data_flag_name != '': 
                    #get corresponding data flag code for standard data flag
                    flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                    #append standard data flag
                    if flag_code not in flags:
                        flags.append(flag_code)
                    #determine if GHOST/network decreed valid
                    if self.data_flag_map[meas_qc]['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                        ghost_decreed_validity = 'I'
                    if self.data_flag_map[meas_qc]['network_decreed_validity'][standard_data_flag_ii] == 'I':
                        network_decreed_validity = 'I'

            # if data source is not fully-merged then add QA flag: 'Not Maximum Data Quality Level'
            if unique_data_source_data_levels[line.data_source_flag] != 'fully-merged':
                qa.append(standard_QA_name_to_QA_code['Not Maximum Data Quality Level'])

            # weather observation flags
            all_wf_flags = [f for f in line.__dict__.keys() if f[-2:] == 'wf']
            # if we have weather flags
            have_au = True
            if all_wf_flags:
                # keep flags according to this hierarchy: 1.AU, 2.AW, 3.MW
                # keep AU flags only
                wf_flags = [f for f in all_wf_flags if f[:2] == 'au']
                # if there's no AU, keep the AW
                if not wf_flags:
                    wf_flags = [f for f in all_wf_flags if f[:2] == 'aw']
                    have_au = False
                # if there's no AW, keep the MW
                if not wf_flags:
                    wf_flags = [f for f in all_wf_flags if f[:2] == 'mw']

                # get dictionary that contains subdicts of weather flags
                wf_map = network_data_flag_to_standard_data_flag_mapping['NOAA_ISD_WF']
                # get each flag's standard name
                au_dict = {}
                for f in wf_flags:
                    weather_flag_type = f[:-4]

                    #process 'AU' flags different to the rest
                    #there are multiple nested flag types so need to assimilate the information together
                    if have_au:
                        #have all au flags per measurement?
                        if len(list(au_dict.keys())) == 6:
                            
                            #create 3 potential flag strings that may need mapping to GHOST standard data flags
                            precip_flag = ''
                            thunderstorm_flag = ''
                            general_weather_flag = ''
                            obscuration_flag = ''

                            #if proximity code is 'in vicinity' then do not process any weather flag strings for measurements  
                            if au_dict['au_proximity'] == '4':
                                #reset au flag dictionary
                                au_dict = {}
                                continue

                            #process precip flag (if not missing )
                            if au_dict['au_precip_code'] not in ['99','']:
                                precip_flag += 'P{}'.format(au_dict['au_precip_code'])
                            #if precipitation code is drizzle, rain, snow, snow grains, diamond dust, ice pellets, hail, snow pellets or unknown precipitation, get associated proximity code
                            if au_dict['au_precip_code'] in ['01','02','03','04','05','06','07','08','09']:
                                #if proximity code is slight, moderate or heavy, then add proximity code to precip flag 
                                if au_dict['au_proximity'] in ['1','2','3']:
                                    precip_flag += 'I{}'.format(au_dict['au_proximity'])

                            #if precipitation code is drizzle, get descriptor flag
                            if au_dict['au_precip_code'] == '01':
                                #if descriptor flag is freezing then add it to precip flag
                                if au_dict['au_descriptor'] == '8':
                                    precip_flag += 'D{}'.format(au_dict['au_descriptor'])

                            #if precipitation code is rain, get descriptor flag
                            if au_dict['au_precip_code'] == '02':
                                #if descriptor flag is shower or freezing then add it to precip flag
                                if au_dict['au_descriptor'] in ['6','8']:  
                                    precip_flag += 'D{}'.format(au_dict['au_descriptor']) 

                            #if precipitation code is snow, get descriptor flag
                            if au_dict['au_precip_code'] == '03':
                                #if descriptor flag is blowing, drifting or shower then add it to precip flag  
                                if au_dict['au_descriptor'] in ['4','5','6']:      
                                    precip_flag += 'D{}'.format(au_dict['au_descriptor'])

                            #if precipitation code is ice pellets, hail or snow pellets, get descriptor flag
                            if au_dict['au_precip_code'] in ['06','07','08']:
                                #if descriptor flag is shower then add it to precip flag
                                if au_dict['au_descriptor'] == '6':  
                                    precip_flag += 'D{}'.format(au_dict['au_descriptor'])   

                            #process thunderstorm flag
                            #is descriptor code thunderstorm, if so add it to thunderstorm flag
                            if au_dict['au_descriptor'] == '7':
                                thunderstorm_flag += 'D{}'.format(au_dict['au_descriptor'])

                            #process general weather flag
                            #is other weather code dust/sand whirls, squalls, funnel clouds, sandstorm, duststorm?
                            if au_dict['au_other_weather'] in ['1','2','3','4','5']:
                                 general_weather_flag += 'W{}'.format(au_dict['au_other_weather'])
                            #if no precipitation and no other weather then set precip flag to be no significant weather
                            if (precip_flag == 'P00') & (au_dict['au_other_weather'] == '0'):
                                precip_flag += 'W{}'.format(au_dict['au_other_weather'])
                            #otherwise have no precipitation flag
                            else:
                                precip_flag = ''
                                
                            #process obscuration flag (if not missing)
                            if au_dict['au_obscuration'] not in ['9','']:
                                obscuration_flag += 'O{}'.format(au_dict['au_obscuration'])
                            #is obscuration code fog get descriptor flag
                            if au_dict['au_obscuration'] == '2':
                                #is descriptor code shallow or freezing, if so then add it to obscuration flag
                                if au_dict['au_descriptor'] in ['1','8']:
                                    obscuration_flag += 'D{}'.format(au_dict['au_descriptor'])
                            #is obscuration code dust or sand get descriptor flag
                            if au_dict['au_obscuration'] in ['5','6']:
                                #is descriptor code blowing or drifting, if so then add it to obscuration flag
                                if au_dict['au_descriptor'] in ['4','5']:
                                    obscuration_flag += 'D{}'.format(au_dict['au_descriptor'])   

                            #map precip_flag, thunderstorm_flag, general_weather_flag and obscuration_flag to standard data flags (if not empty strings)
                            for current_au_flag in [precip_flag, thunderstorm_flag, general_weather_flag, obscuration_flag]:
                                #flag is non-empty?
                                if current_au_flag != '':
                                    standard_data_flag_dict = wf_map['au'][current_au_flag]
                                    #get mapped standard data flags
                                    standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                                    #iterate through standard data flags 
                                    for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                                        #check if standard data flag is empty (i.e. unmapped as non-useful)
                                        if standard_data_flag_name != '': 
                                            #get corresponding data flag code for standard data flag
                                            flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                            #append standard data flag
                                            if flag_code not in flags:
                                                flags.append(flag_code)
                                            #determine if GHOST/network decreed valid
                                            if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                                ghost_decreed_validity = 'I'
                                            if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                                network_decreed_validity = 'I'

                            #reset au flag dictionary
                            au_dict = {}

                        if weather_flag_type == 'au_proximity': 
                            au_dict['au_proximity'] = getattr(line, f)
                                
                        elif weather_flag_type == 'au_descriptor':
                            au_dict['au_descriptor'] = getattr(line, f)
 
                        elif weather_flag_type == 'au_precip_code':
                            au_dict['au_precip_code'] = getattr(line, f)

                        elif weather_flag_type == 'au_obscuration':
                            au_dict['au_obscuration'] = getattr(line, f)

                        elif weather_flag_type == 'au_other_weather':
                            au_dict['au_other_weather'] = getattr(line, f)

                        elif weather_flag_type == 'au_combination':
                            au_dict['au_combination'] = getattr(line, f)

                    #other types of weather flags than AU?
                    else:
                        # map weather data flag/s and append
                        standard_data_flag_dict = wf_map[weather_flag_type][getattr(line, f)]
                            
                        #get mapped standard data flags
                        standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                        #iterate through standard data flags 
                        for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                            #check if standard data flag is empty (i.e. unmapped as non-useful)
                            if standard_data_flag_name != '': 
                                #get corresponding data flag code for standard data flag
                                flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                #append standard data flag
                                if flag_code not in flags:
                                    flags.append(flag_code)
                                #determine if GHOST/network decreed valid
                                if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                    ghost_decreed_validity = 'I'
                                if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                    network_decreed_validity = 'I'

             # finally, append all flags
            site_data['flag']['value'].append(flags)

            #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
            if ghost_decreed_validity == 'I':
                qa.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
            if network_decreed_validity == 'I':
                qa.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])

            #finally, append the qa code list for this measure to the dict field
            site_data['qa']['value'].append(qa)

            #iterate through data variables, if any columns have not been appended to on this pass, add NaN or 255 depending on datatype
            for column_header in list(site_data.keys()):
                n_column = len(site_data[column_header]['value'])
                #if column length is < length time, then append NaN or 255
                if n_column < len(site_data['time']['value']):
                    if site_data[column_header]['data_type'] != np.uint8:
                        site_data[column_header]['value'].append(np.NaN)
                    else:
                        site_data[column_header]['value'].append(255)

        # add metadata update stamp (taken from first sorted time)
        sorted_time = sorted(site_data['time']['value'])
        site_metadata['meta_update_stamp']['value'][-1] = sorted_time[0]

        #add data revision timestamp, set as 0001-01-01 00:00 for all files
        data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
        site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

        #standardise metadata/data formats and write out station data to numpy file
        write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)

        # update flag to state that have written data for station (if have done so)
        if write_obj.written:
            self.written_data = True
        else:
            self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)