import copy
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd
import json

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseUSEPACASTNETFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, year, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.year = year
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current year
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per year'''

        #get specific data files for current year
        data_files = get_data_files(self.run_details, self.parameter_details, specific_year=self.year)
   
        #load all external metadata into memory 
        #self.external_metadata = np.load(self.run_details['data_root']+'US_EPA_CASTNET/metadata/processed/US_EPA_CASTNET_META.npy', allow_pickle=True)
        #load external metadata into memory 
        with open(self.run_details['data_root']+'US_EPA_CASTNET/metadata/{}/processed/US_EPA_CASTNET_META.json'.format(self.run_details['ghost_version']), 'r', encoding='utf-8') as f:
            self.external_metadata = json.loads(f.read())

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_files


    def parse_file(self, data_file):

        '''Function to a specific annual data file to GHOST standard numpy station specific files'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
        
        #create object for parsing US EPA CASTNET annual file, and read file
        parsing_obj = ParseUSEPACASTNETFile(data_file)
        parsing_obj.read_data()

        #get unique station references
        unique_station_references = np.unique(parsing_obj.parsed_data['SITE_ID'])

        #get file type
        file_type = data_file.split('/')[-2]

        #get castnet parameter name/s for current species
        castnet_parameter_names = self.parameter_details['castnet_parameter_name']
        if type(castnet_parameter_names) == dict:
            castnet_parameter_names = [castnet_parameter_names[file_type]]
        elif type(castnet_parameter_names) == str:
            castnet_parameter_names = [castnet_parameter_names]

        #iterate through castnet parameters names
        for castnet_parameter_name in castnet_parameter_names:

            #iterate over all unique station references
            for unique_station_reference in unique_station_references:

                #cut read data to just that associated with unique station reference
                valid_inds = parsing_obj.parsed_data['SITE_ID'] == unique_station_reference

                #get specific data variables by file type
                if file_type == 'O3':
                    start_dt = parsing_obj.parsed_data['DATE_TIME'][valid_inds]
                    end_dt = [] 
                    temporal_resolutions = np.array([60]*len(start_dt))
                    data_levels = parsing_obj.parsed_data['QA_CODE'][valid_inds]
                    comment_codes = []
                    joined_codes = []
                    data_flags = parsing_obj.parsed_data['{}_F'.format(castnet_parameter_name)][valid_inds]
                    cut_measurements = parsing_obj.parsed_data[castnet_parameter_name][valid_inds]
                elif file_type == 'hourly_gas':
                    valid_inds_hourly = parsing_obj.parsed_data['PARAMETER'] == castnet_parameter_name
                    valid_inds = valid_inds & valid_inds_hourly
                    start_dt = parsing_obj.parsed_data['DATE_TIME'][valid_inds]
                    end_dt = []
                    temporal_resolutions = np.array([60]*len(start_dt))
                    data_levels = parsing_obj.parsed_data['QA_CODE'][valid_inds]
                    comment_codes = []
                    joined_codes = []
                    data_flags = parsing_obj.parsed_data['VALUE_F'][valid_inds]
                    cut_measurements = parsing_obj.parsed_data['VALUE'][valid_inds]
                elif file_type == 'drychem':
                    start_dt = parsing_obj.parsed_data['DATEON'][valid_inds]
                    end_dt = parsing_obj.parsed_data['DATEOFF'][valid_inds]
                    temporal_resolutions = []
                    data_levels = parsing_obj.parsed_data['QA_CODE'][valid_inds]
                    comment_codes = parsing_obj.parsed_data['COMMENT_CODES'][valid_inds]
                    joined_codes = []
                    data_flags = parsing_obj.parsed_data['{}_F'.format(castnet_parameter_name)][valid_inds]
                    cut_measurements = parsing_obj.parsed_data[castnet_parameter_name][valid_inds]
                elif file_type == 'vischem':
                    start_dt = parsing_obj.parsed_data['DATEON'][valid_inds]
                    end_dt = parsing_obj.parsed_data['DATEOFF'][valid_inds]
                    temporal_resolutions = []
                    data_levels = np.array([3]*len(start_dt))
                    comment_codes = parsing_obj.parsed_data['COMMENT_CODES'][valid_inds]
                    joined_codes = []
                    data_flags = parsing_obj.parsed_data['{}_F'.format(castnet_parameter_name)][valid_inds]
                    cut_measurements = parsing_obj.parsed_data[castnet_parameter_name][valid_inds]
                elif file_type == 'wetdep':
                    start_dt = parsing_obj.parsed_data['DATEON'][valid_inds]
                    end_dt = parsing_obj.parsed_data['DATEOFF'][valid_inds]
                    temporal_resolutions = []
                    data_levels = np.array([3]*len(start_dt))
                    val_codes = list(parsing_obj.parsed_data['VALCODE'][valid_inds])
                    inval_codes = list(parsing_obj.parsed_data['INVALCODE'][valid_inds])
                    comment_codes = []
                    cut_measurements = parsing_obj.parsed_data[castnet_parameter_name][valid_inds]
                    if self.run_details['parameter'] != 'PREC_ACCUM':
                        data_flags = parsing_obj.parsed_data['{}_F'.format(castnet_parameter_name)][valid_inds]
                    else: # no data flags *_F for precipitation amount
                        data_flags = ['' for x in cut_measurements]

                    joined_codes = [] # list to join both code types, they never appear together for one sample
                    for ii in range(len(val_codes)):
                        # remove white spaces
                        val_codes[ii] = val_codes[ii].replace(' ', '')
                        inval_codes[ii] = inval_codes[ii].replace(' ', '')
                        
                        if len(inval_codes[ii]) > 1: # split invalcodes in seperate flags as every letter is its own flag
                            invalid_single_codes = [str(x) for x in inval_codes[ii]]

                            inval_codes[ii] = invalid_single_codes
                        else:
                            inval_codes[ii] = [inval_codes[ii]]

                        # join both lists in one
                        if val_codes[ii] == '': 
                            joined_codes.append(inval_codes[ii])
                        else:
                            joined_codes.append([val_codes[ii]])
                    
                #if no valid time then continue to next station reference
                if len(start_dt) == 0:
                    continue

                self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_reference)

                #create site instances of metadata/data dictionaries
                site_data =     get_standard_data(self.parameter_details)
                site_metadata = get_standard_metadata(self.parameter_details)
                
                #------------------------------------------------------------------------#
                #PARSE METADATA
                #------------------------------------------------------------------------#

                #GHOST version
                site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

                #initialise string which will store all GHOST process warnings as empty            
                site_metadata['process_warnings']['value'].append('')

                #write known metadata  
                site_metadata['network']['value'].append(self.run_details['network'])
                site_metadata['data_level']['value'].append('variable')  
                site_metadata['ellipsoid']['value'].append('WGS 84') 
                site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
                site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                site_metadata['projection']['value'].append('geographic')     
                site_metadata['contact_name']['value'].append('Melissa Puchalski')
                site_metadata['contact_institution']['value'].append('US EPA CASTNET')      
                site_metadata['contact_email_address']['value'].append('puchalski.melissa@epa.gov')    
                site_metadata['principal_investigator_name']['value'].append('Melissa Puchalski')     
                site_metadata['principal_investigator_email_address']['value'].append('puchalski.melissa@epa.gov')  
                site_metadata['data_licence']['value'].append('Public domain: https://edg.epa.gov/EPA_Data_License.html')                   

                #some parameters are measured in varying ways, with varying accuracy on the methods, to ensure these are afterwards consistently merged, 
                #set the "Corrected Parameter" qa flag for priority parameters
                corrected_parameter = False 
                if self.run_details['parameter'] == 'SO2':
                    if castnet_parameter_name == 'SO2_GA':
                        corrected_parameter = True
                elif self.run_details['parameter'] == 'HNO3':
                    if castnet_parameter_name == 'HNO3':
                        corrected_parameter = True
                elif self.run_details['parameter'] == 'NO3-':
                    if castnet_parameter_name == 'TNO3':
                        corrected_parameter = True
                elif self.run_details['parameter'] == 'SO4--':
                    if castnet_parameter_name == 'TSO4':
                        corrected_parameter = True  
                elif self.run_details['parameter'] == 'PM2.5_EC':
                    if castnet_parameter_name == 'EHTC':
                        corrected_parameter = True
                elif self.run_details['parameter'] == 'PM2.5_OC':
                    if castnet_parameter_name == 'OHTC':
                        corrected_parameter = True
                   
                #add reported measurement unit
                #convert raw reported units given to standard name
                if file_type == 'O3':
                    reported_units = 'nmol/mol'
                elif file_type == 'hourly_gas':
                    if self.run_details['parameter'] == 'TEMP_2M':
                        reported_units = 'degC'
                    else:
                        reported_units = 'nmol/mol'
                elif file_type == 'drychem':
                    reported_units = 'ug/m3'
                elif file_type == 'vischem':
                    reported_units = 'ug/m3' 
                elif file_type == 'wetdep':
                    if self.run_details['parameter'] == 'PREC_ACCUM':
                        reported_units = 'inches'
                    elif self.run_details['parameter'] == 'H+_PREC': # after manual conversion in network parser from pH to H+
                        reported_units = 'mol l-1'
                    else:
                        reported_units = 'mg l-1' #manually changed units!!! 'ug m-3'

                print(reported_units)
                conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
                site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

                #add external metadata
                #cut_external_metadata = self.external_metadata[np.where(self.external_metadata['SITE_ID'] == unique_station_reference)[0]]
                cut_external_metadata = self.external_metadata[unique_station_reference]
                site_metadata['station_name']['value'].append(cut_external_metadata['SITE_NAME']['values'][-1])
                site_metadata['longitude']['value'].append(cut_external_metadata['LONGITUDE']['values'][-1])
                site_metadata['latitude']['value'].append(cut_external_metadata['LATITUDE']['values'][-1])
                site_metadata['altitude']['value'].append(cut_external_metadata['ELEVATION']['values'][-1])
                site_metadata['associated_networks']['value'].append('NADP:{}'.format(cut_external_metadata['NADP_ID']['values'][-1]))
                agency = cut_external_metadata['AGENCY']['values'][-1]
                if agency == 'BLM':
                    agency = 'Bureau of Land Management'
                elif agency == 'EPA':
                    agency = 'US Environmental Protection Agency'
                elif agency == 'NPS':
                    agency = 'National Park Service'
                site_metadata['principal_investigator_institution']['value'].append(agency)

                #get local timezone (used for processing time)
                local_timezone = cut_external_metadata['TIME_ZONE']['values'][-1].upper()

                #set sampling height for some parameters (from document report: CASTNET_QAPP_v9-3)
                if file_type == 'O3':
                    site_metadata['sampling_height']['value'].append('10.0')   
                elif file_type == 'hourly_gas':
                    if self.run_details['parameter'] == 'TEMP_2M':
                        site_metadata['sampling_height']['value'].append('2.0') 
                    else:
                        site_metadata['sampling_height']['value'].append('10.0') 
                elif file_type == 'drychem':
                    site_metadata['sampling_height']['value'].append('10.0')

                #parse station classifications (from external metadata)
                native_classifications = {'terrain':cut_external_metadata['TERRAIN']['values'][-1], 'land_use':cut_external_metadata['LAND_USE']['values'][-1]}
                site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

                #process measurement process information
                site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(file_type, site_metadata, self.run_details, self.parameter_details)
                #set measurement qa codes which are True as joined metadata str
                site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
                measurement_qa_flag_codes = []
                for standard_flag_name in measurement_qa_flags.keys():
                    if measurement_qa_flags[standard_flag_name]:
                        measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
                site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

                #iterate through metadata variables, if any columns have not been appended to add a NaN
                for column_header in list(site_metadata.keys()):
                    n_column = len(site_metadata[column_header]['value'])
                    #if column is empty, then append NaN
                    if n_column < 1:
                        site_metadata[column_header]['value'].append(np.NaN)
        
                #set station_reference (provided network ID + standard measurement method abbreviation)
                site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

                #get data file download timestamp
                site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

                #add data revision timestamp, set as 0001-01-01 00:00 for all files
                data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
                site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

                #------------------------------------------------------------------------#
                #PARSE TIME
                #------------------------------------------------------------------------#
                tzinfos = {'AL':dateutil.tz.gettz('US/Alaska'), 'AT':dateutil.tz.gettz('Canada/Atlantic'), 'CE':dateutil.tz.gettz('US/Central'), 'EA':dateutil.tz.gettz('US/Eastern'), 'EANO':-18000, 'HA':dateutil.tz.gettz('US/Aleutian'), 'HANO':-36000, 'MO':dateutil.tz.gettz('US/Mountain'), 'MONO':-25200, 'PA':dateutil.tz.gettz('US/Pacific')}
                
                #parse start date/time into datetime (taking into account timezone information), convert to UTC datetime, then round to nearest minute
                start_dt = [round_to_minute(dateutil.parser.parse('{} {}'.format(date,local_timezone), tzinfos=tzinfos).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in start_dt]
                #parse end date/time into datetime (taking into account timezone information), convert to UTC datetime, then round to nearest minute
                #also calculate temporal resolutions
                if len(end_dt) > 0:
                    end_dt = [round_to_minute(dateutil.parser.parse('{} {}'.format(date,local_timezone), tzinfos=tzinfos).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in end_dt]
                    #get temporal resolutions
                    for start, end in zip(start_dt,end_dt):
                        dt_window = end - start
                        #convert hours to minutes
                        dt_window_minutes = int(dt_window.total_seconds()/60.)
                        temporal_resolutions.append(dt_window_minutes)  

                #convert each start measurement datetime to minutes from reference time
                frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

                #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
                site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

                #------------------------------------------------------------------------#
                #PARSE QA/FLAGS
                #------------------------------------------------------------------------#
                
                #flags/qa lists to write out
                flags = []
                qa = []

                #get dictionary necessary for mapping of US EPA CASTNET file data flags to standard data flag codes
                data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['US_EPA_CASTNET'][file_type]

                #iterate through US EPA CASTNET file data flags/comment_codes and map to standard flag codes
                for data_flag_ii, data_flag in enumerate(data_flags):

                    #lists to append qa/data flags per measurement
                    flags_per_measurement = []
                    qa_per_measurement = []
                    #set GHOST decreed validity/network decreed validity variables initially as 'V'
                    ghost_decreed_validity = 'V'
                    network_decreed_validity = 'V'

                    # map data flag/s and append
                    standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                    #get mapped standard data flags
                    standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                    #iterate through standard data flags 
                    for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                        #check if standard data flag is empty (i.e. unmapped as non-useful)
                        if standard_data_flag_name != '': 
                            #get corresponding data flag code for standard data flag
                            flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                            #append standard data flag
                            if flag_code not in flags_per_measurement:
                                flags_per_measurement.append(flag_code)
                            #determine if GHOST/network decreed valid
                            if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                ghost_decreed_validity = 'I'
                            if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                network_decreed_validity = 'I'

                    # map comment code/s and append
                    #iterate through comment codes per measurement
                    if (len(comment_codes) > 0):
                        comment_codes_per_measurement = comment_codes[data_flag_ii].split(' ')
                        for comment_code in comment_codes_per_measurement:
                            #check if have prefix/s before number in code
                            comment_code_prefixes = re.findall("[a-zA-Z]+", comment_code)
                            comment_code_numbers = re.findall(r'\d+', comment_code)
                            #if have more than 1 non-joined prefixes or numbers, then continue to next code as current code is formatted badly
                            if (len(comment_code_prefixes) > 1) or (len(comment_code_numbers) > 1):
                                continue
                            #if have no code numbers then continue as have no code to map to
                            if len(comment_code_numbers) == 0:
                                continue
                            #now isolate prefix/numer
                            if len(comment_code_prefixes) == 0:
                                comment_code_prefix = ''
                            else:
                                comment_code_prefix = comment_code_prefixes[0]
                            comment_code_number = comment_code_numbers[0]
                            #convert comment code to int and then back to str to ensure formatting is correct
                            comment_code_number = str(int(comment_code_number))
                            #if prefix has a longer length then 1, or code number longer than 2 then continue as they shouldn't be so long
                            if (len(comment_code_prefix) > 1) or (len(comment_code_number) > 2):
                                continue

                            #if have a prefix, then check if it applies to parameter currently processing (only for drychem file type, have prefixes for vischem but impossible to link to parameter)
                            #if not, continue to next code 
                            if len(comment_code_prefix) > 0:
                                if file_type == 'drychem':
                                    #G - applies to all parameters
                                    if comment_code_prefix == 'G':
                                        pass
                                    #N - Nylon filter
                                    elif comment_code_prefix == 'N':
                                        if castnet_parameter_name not in ['NHNO3', 'NSO4']:
                                            continue
                                    #T - Teflon filter
                                    elif comment_code_prefix == 'T':
                                        if castnet_parameter_name not in ['CA','CL','K','MG','NA','TNH4','TNO3','TSO4']:
                                            continue
                                    #W - Whatman filter
                                    elif comment_code_prefix == 'W':
                                        if castnet_parameter_name not in ['WSO2','WNO3']:
                                            continue
                                    #unknown prefix - assume applies to all parameters
                                    else:
                                        pass

                            # map comment code number
                            standard_data_flag_dict = data_flag_mapping_dictionary[comment_code_number]
                            #get mapped standard data flags
                            standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                            #iterate through standard data flags 
                            for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                                #check if standard data flag is empty (i.e. unmapped as non-useful)
                                if standard_data_flag_name != '': 
                                    #get corresponding data flag code for standard data flag
                                    flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                    #append standard data flag (if not already appended)
                                    if flag_code not in flags_per_measurement:
                                        flags_per_measurement.append(flag_code)
                                    #determine if GHOST/network decreed valid
                                    if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                        ghost_decreed_validity = 'I'
                                    if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                        network_decreed_validity = 'I'

                    # invalid/valid codes from wetdep data
                    if len(joined_codes) > 0:
                        if len(joined_codes[data_flag_ii]) > 0:

                            for qr_flag in joined_codes[data_flag_ii]: # every letter is its own flag
                                # map qr flag and append
                                standard_data_flag_dict = data_flag_mapping_dictionary[qr_flag]
                                #get mapped standard data flags
                                standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                                #iterate through standard data flags 
                                for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                                    #check if standard data flag is empty (i.e. unmapped as non-useful)
                                    if standard_data_flag_name != '': 
                                        #get corresponding data flag code for standard data flag
                                        flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                        #append standard data flag (if not already appended)
                                        if flag_code not in flags_per_measurement:
                                            flags_per_measurement.append(flag_code)
                                        #determine if GHOST/network decreed valid
                                        if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                            ghost_decreed_validity = 'I'
                                        if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                            network_decreed_validity = 'I'


                    #certain data levels give information about potential invalidity of data, map this to a standard data flag
                    #also if data level is not maximum (3), then write 'Not Maximum Data Quality Level' QA flag
                    data_level = data_levels[data_flag_ii]
                    if data_level in ['GS','X']:
                        if standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags_per_measurement:
                            flags_per_measurement.append(standard_data_flag_name_to_data_flag_code['Missing Data'])
                        ghost_decreed_validity = 'I'
                        network_decreed_validity = 'I'
                    elif data_level == '1X':
                        if standard_data_flag_name_to_data_flag_code['Irregular Data Manually Screened and Rejected'] not in flags_per_measurement:
                            flags_per_measurement.append(standard_data_flag_name_to_data_flag_code['Irregular Data Manually Screened and Rejected'])
                        ghost_decreed_validity =  'I'
                        network_decreed_validity = 'I'
                    if data_level != '3':
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Not Maximum Data Quality Level'])

                    #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                    if ghost_decreed_validity == 'I':
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                    if network_decreed_validity == 'I':
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])

                    #write corrected parameter qa flag if necessary
                    if corrected_parameter:
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Corrected Parameter'])

                    #append qa_per_measurement, flags_per_measurement to lists to write out
                    flags.append(flags_per_measurement)
                    qa.append(qa_per_measurement)

                #------------------------------------------------------------------------#
                #PARSE MEASUREMENTS
                #------------------------------------------------------------------------#

                #measurement values may need converting (i.e. mass density to mixing ratio)

                #measurement temperature and pressure values are never provided so are always assumed to be the US EPA standard pressure and temperatures
                #these are defined as for 298.15 K temperature, and 1013.25 hPa for pressure here for all compounds, except PM2.5 and lead: https://www.govinfo.gov/content/pkg/CFR-2011-title40-vol2/pdf/CFR-2011-title40-vol2-chapI.pdf
                #for PM2.5 and lead the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
                #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.
                # wet concentration need simple conversion from ug m-3 to mg l-1, prec_accum from inches to mm conversion
        
                measurements = []

                standard_network_gas_volume = False
                unknown_gas_volume = False

                if (self.parameter_details['matrix'] == 'pm2.5') or ('Pb' in self.run_details['parameter']) or ('PREC' in self.run_details['parameter']):
                    site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                    site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
                    unknown_gas_volume = True
                else:
                    site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '298.15'
                    site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.25'
                    standard_network_gas_volume = True
        
                #get standard reported units/quantity
                standard_reported_units = conv_obj.output_standard_units
                measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

                #set flags for writing QA flags associated with assumptions in unit conversion
                unit_conversion_standard_network = False
                unit_conversion_educated_guess = False

                #check if reported standard units, match required standard units
                if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                    #mass density to mixing ratio?
                    if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                        #get conversion factor
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
    
                    #mixing ratio to mass density?
                    elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
    
                    # molarity to mass density 
                    elif (measuring_instrument_reported_unit_quantity == 'molarity') & (self.measured_unit_quantity == 'mass_density'):
                        input_units = {'molar_mass':'kg mol-1', 'molarity':standard_reported_units}     
                        #do conversion 
                        input_vals = {'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'molarity':1.0}
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species=self.parameter_details['chemical_formula'],conversion_input_quantity = 'molarity')
                
                    #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                    else:
                        conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                    #process through each measurement and convert value using derived conversion factor
                    for n, meas in enumerate(cut_measurements):
                        #only do conversion if measurement value is above 0 and not infinite
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                        
                        #special exception conversion for temperature (celcius to kelvin)
                        if self.run_details['parameter'] == 'TEMP_2M':
                            if (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                measurements.append(float_meas+273.15)
                            else:
                                measurements.append(float_meas)
                        #all other parameters
                        else:
                            if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                measurements.append(float_meas*conv_obj.conversion_factor)
                            else:
                                measurements.append(float_meas)

                        #append sample gas volume/unit_conversion assumption QA flags if necessary
                        if standard_network_gas_volume:
                            qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                        if unknown_gas_volume:
                            qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                        if unit_conversion_standard_network:
                            qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                        if unit_conversion_educated_guess:
                            qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

                #otherwise do nothing to measured values
                else:
                    for n, meas in enumerate(cut_measurements):
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                        measurements.append(float_meas)

                #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
                valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
                if len(valid_temporal_resolution_inds) != len(frac_time):
                    frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                    temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                    flags = [flags[x] for x in valid_temporal_resolution_inds]
                    qa = [qa[x] for x in valid_temporal_resolution_inds]
                    measurements = [measurements[x] for x in valid_temporal_resolution_inds]

                #----------------------------------------------------------------------------#
                #WRITE OUT DATA
                #----------------------------------------------------------------------------# 
                site_data['time']['value'] = frac_time
                site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
                site_data['temporal_resolution']['value'] = temporal_resolutions   
                site_data['flag']['value'] = flags
                site_data['qa']['value'] = qa
                #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
                for column_header in list(site_data.keys()):
                    n_column = len(site_data[column_header]['value'])
                    #if column is empty, then append NaN
                    if n_column == 0:
                        if site_data[column_header]['data_type'] != np.uint8:
                            site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                        else:
                            site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
                #------------------------------------------------------

                #standardise metadata/data formats and write out station data to numpy file
                write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
                if write_obj.written == False:
                    self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)