import copy
import datetime
from dateutil.relativedelta import relativedelta
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from geographic_functions import local_timezone_name
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseUSEPAAQSFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


#define dictionary mapping units to cleaned units
unit_dict = {'Milligrams/cubic meter':'mg m-3', 'Micrograms/cubic meter':'ug m-3', 'Nanograms/cubic meter':'ng m-3', 'Parts per million':'umol mol-1', 'Parts per billion':'nmol mol-1', 'Parts per trillion':'pmol mol-1', 'Parts per billion Carbon':'nmolC mol-1', 'Parts per million Carbon':'umolC mol-1'}

class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, year, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.year = year
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current year
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per year'''

        #get specific data files for current year
        data_files = get_data_files(self.run_details, self.parameter_details, specific_year=self.year)

        #load all external metadata into memory (3 files for EEA_AQ_eReporting) -- joins to specific station will be done later
        self.external_metadata_aqs_monitors = np.load(self.run_details['data_root']+'US_EPA_AQS/metadata/processed/US_EPA_AQS_MONITORS.npy', allow_pickle=True)
        self.external_metadata_aqs_sites = np.load(self.run_details['data_root']+'US_EPA_AQS/metadata/processed/US_EPA_AQS_SITES.npy', allow_pickle=True)

        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_files

    def parse_file(self, data_file):

        '''Function to a specific annual data file to GHOST standard numpy station specific files'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
        
        #create object for parsing US EPA AQS annual file, and read file
        parsing_obj = ParseUSEPAAQSFile(data_file, self.parameter_details['aqs_code'])
        parsing_obj.read_data()

        #get file resolution
        file_resolution = data_file.split('/')[-1].split('_')[0]

        #get unique station references
        unique_station_references = np.unique(parsing_obj.parsed_data['station_reference'])

        #iterate over all unique station references
        for unique_station_reference in unique_station_references:

            self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_reference)

            #break station reference down to state code, county code and site number
            state_code = unique_station_reference[:2]
            county_code = unique_station_reference[2:5]
            site_number = unique_station_reference[5:]
            
            #map station specific external metadata 
            cut_external_metadata = {}
            meta_index_site = np.where((self.external_metadata_aqs_sites['State Code'] == state_code) & (self.external_metadata_aqs_sites['County Code'] == county_code) & (self.external_metadata_aqs_sites['Site Number'] == site_number))[0]
            if len(meta_index_site) == 1:
                for var in self.external_metadata_aqs_sites.dtype.names:
                    cut_external_metadata[var] = self.external_metadata_aqs_sites[var][meta_index_site[0]]
            #no match found, fill in non-existing fields as empty strings
            elif len(meta_index_site) == 0:
                for var in self.external_metadata_aqs_sites.dtype.names:
                    cut_external_metadata[var] = ''            
            #more than 1 match, throw error
            elif len(meta_index_site) > 1:
                self.run_details['log_file_str'] += 'NON-SPECIFIC MONITOR METADATA FOR STATION:{}'.format(unique_station_reference)
                create_GHOST_stage_logfile(2,self.run_details)

            #cut read data to just that associated with unique station reference & parameter being processed
            station_parsed_data = parsing_obj.parsed_data[np.where(parsing_obj.parsed_data['station_reference'] == unique_station_reference)[0]]

            #get all unique parameter codes
            unique_parameter_codes = np.unique(station_parsed_data['Parameter Code'])
            
            #iterate through all unique parameter codes
            for unique_parameter_code in unique_parameter_codes:

                #determine if variable is a corrected variable or not (for black carbon) 
                #also treat some elemental carbon and organic carbon variable codes as 'corrected' to set priority in case of revised methodologies
                if unique_parameter_code in ['82305','82307','88317','88320','88321','88381','88382']:
                    corrected_variable = True
                else:
                    corrected_variable = False

                #get data associated with unique parameter code
                parameter_parsed_data = station_parsed_data[np.where(station_parsed_data['Parameter Code'] == unique_parameter_code)[0]]

                #get all unique POCs
                unique_POCs = np.unique(parameter_parsed_data['POC'])

                #iterate through all unique POCs
                for unique_POC in unique_POCs:

                    #map station+parameter+POC specific external metadata (overwrite previous site specific meatdata if not empty str)
                    meta_index_monitor = np.where((self.external_metadata_aqs_monitors['State Code'] == state_code) & (self.external_metadata_aqs_monitors['County Code'] == county_code) & (self.external_metadata_aqs_monitors['Site Number'] == site_number) & (self.external_metadata_aqs_monitors['Parameter Code'] == unique_parameter_code) & (self.external_metadata_aqs_monitors['POC'] == unique_POC))[0]
                    if len(meta_index_monitor) == 1:
                        for var in self.external_metadata_aqs_monitors.dtype.names:
                            var_val = self.external_metadata_aqs_monitors[var][meta_index_monitor[0]]
                            if var not in list(cut_external_metadata.keys()):
                                cut_external_metadata[var] = var_val
                            else:
                                if var_val != '':
                                    cut_external_metadata[var] = var_val
                    #no match found, fill in non-existing fields as empty strings
                    elif len(meta_index_monitor) == 0:
                        for var in self.external_metadata_aqs_monitors.dtype.names:
                            if var not in list(cut_external_metadata.keys()):
                                cut_external_metadata[var] = ''
                    #more than 1 match, throw error
                    elif len(meta_index_monitor) > 1:
                        self.run_details['log_file_str'] += 'NON-SPECIFIC MONITOR METADATA FOR STATION/PARAMETER CODE/POC:{}/{}/{}'.format(unique_station_reference, unique_parameter_code, unique_POC)
                        create_GHOST_stage_logfile(2,self.run_details)

                    #get data associated with unique parameter code
                    poc_parsed_data = parameter_parsed_data[np.where(parameter_parsed_data['POC'] == unique_POC)[0]]

                    #fill any empty lon/lat data values with external metadata
                    if 'Longitude' in list(cut_external_metadata.keys()): 
                        if cut_external_metadata['Longitude'] != '':
                            poc_parsed_data['Longitude'][np.where(poc_parsed_data['Longitude'] == '')[0]] = cut_external_metadata['Longitude']
                    if 'Latitude' in list(cut_external_metadata.keys()): 
                        if cut_external_metadata['Latitude'] != '':
                            poc_parsed_data['Latitude'][np.where(poc_parsed_data['Latitude'] == '')[0]] = cut_external_metadata['Latitude']

                    #get all unique lon/lat pairs
                    all_measurement_positions = np.array([[lon, lat] for lon,lat in zip(poc_parsed_data['Longitude'], poc_parsed_data['Latitude'])])
                    unique_measurement_positions, unique_measurement_positions_indices = np.unique(all_measurement_positions, axis=0, return_index=True)
                    unique_measurement_positions = [x for _,x in sorted(zip(unique_measurement_positions_indices,unique_measurement_positions))]

                    #iterate through unique measurement configurations
                    for unique_measurement_position in unique_measurement_positions:
            
                        unique_longitude = unique_measurement_position[0]
                        unique_latitude = unique_measurement_position[1]
                
                        #get data associated with unique measurement position
                        mposition_parsed_data = poc_parsed_data[np.where((poc_parsed_data['Longitude'] == unique_longitude) & (poc_parsed_data['Latitude'] == unique_latitude))[0]]

                        #fill any empty method code data values with external metadata
                        if 'Last Method Code' in list(cut_external_metadata.keys()): 
                            if cut_external_metadata['Last Method Code'] != '':
                                mposition_parsed_data['Method Code'][np.where(mposition_parsed_data['Method Code'] == '')[0]] = cut_external_metadata['Last Method Code']
             
                        #get unique method codes
                        unique_method_codes = np.unique(mposition_parsed_data['Method Code'])
                
                        #iterate through unique method codes
                        for unique_method_code in unique_method_codes:

                            #get data associated with unique method code
                            method_parsed_data = mposition_parsed_data[np.where(mposition_parsed_data['Method Code'] == unique_method_code)[0]]

                            #get all unique measurement units
                            unique_units = np.unique(method_parsed_data['Units of Measure'])

                            #iterate through unique units
                            for unique_unit in unique_units:

                                #if unique unit is per Carbon, but number of carbon atoms in species is unknown then cannot convert to standard units
                                #therefore pass over unit
                                if ('Carbon' in unique_unit) & ('C' not in self.parameter_details['chemical_formula']):
                                    continue

                                #get data associated with unique unit
                                unit_parsed_data = method_parsed_data[np.where(method_parsed_data['Units of Measure'] == unique_unit)[0]]

                                #fill any empty datum data values with external metadata
                                if 'Datum' in list(cut_external_metadata.keys()): 
                                    if cut_external_metadata['Datum'] != '':
                                        unit_parsed_data['Datum'][np.where(unit_parsed_data['Datum'] == '')[0]] = cut_external_metadata['Datum']

                                #get all unique datums
                                unique_datums = np.unique(unit_parsed_data['Datum'])

                                #iterate through unique datums
                                for unique_datum in unique_datums:

                                    #get data associated with unique unit
                                    final_parsed_data = unit_parsed_data[np.where(unit_parsed_data['Datum'] == unique_datum)[0]]
                                
                                    #create site instances of metadata/data dictionaries
                                    site_data =     get_standard_data(self.parameter_details)
                                    site_metadata = get_standard_metadata(self.parameter_details)

                                    #------------------------------------------------------------------------#
                                    #PARSE METADATA
                                    #------------------------------------------------------------------------#

                                    #GHOST version
                                    site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

                                    #initialise string which will store all GHOST process warnings as empty            
                                    site_metadata['process_warnings']['value'].append('')

                                    #write known metadata  
                                    site_metadata['network']['value'].append(self.run_details['network'])
                                    site_metadata['data_level']['value'].append('none')                         
                                    site_metadata['contact_institution']['value'].append('US EPA AQS')      
                                    site_metadata['contact_email_address']['value'].append('https://www.epa.gov/aqs/forms/aqs-contact-us')    
                                    site_metadata['data_licence']['value'].append('Public domain: https://edg.epa.gov/EPA_Data_License.html') 

                                    #non-standard comments
                                    non_standard_fields_to_parse = ['Owning Agency','Reporting Agency','Collecting Agency','Monitoring Objective','Address']
                                    misc_str = ''
                                    for field in non_standard_fields_to_parse:
                                        if cut_external_metadata[field] != '':
                                            misc_str += '{}:{};'.format(field, cut_external_metadata[field])
                                    if misc_str != '':
                                        site_metadata['network_miscellaneous_details']['value'].append(misc_str[:-1])

                                    #datum information (if unknown assume WGS84)
                                    if unique_datum in ['WGS84','UNKNOWN']:
                                        site_metadata['ellipsoid']['value'].append('WGS 84') 
                                        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
                                        site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                                        site_metadata['projection']['value'].append('geographic')    
                                    elif unique_datum == 'NAD27':
                                        site_metadata['ellipsoid']['value'].append('Clarke 1866') 
                                        site_metadata['horizontal_datum']['value'].append('North American Datum 1927')
                                        site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                                        site_metadata['projection']['value'].append('geographic')   
                                    elif unique_datum == 'NAD83':
                                        site_metadata['ellipsoid']['value'].append('GRS 80') 
                                        site_metadata['horizontal_datum']['value'].append('North American Datum 1983')
                                        site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                                        site_metadata['projection']['value'].append('geographic')   
                                    else:
                                    #throw error if datum is unknown
                                        self.run_details['log_file_str'] += 'UNKNOWN DATUM:{}'.format(unique_datum)
                                        create_GHOST_stage_logfile(2,self.run_details)

                                    #site metadata
                                    site_metadata['longitude']['value'].append(unique_longitude)
                                    site_metadata['latitude']['value'].append(unique_latitude)
                                    site_metadata['altitude']['value'].append(cut_external_metadata['Elevation'])
                                    site_metadata['station_name']['value'].append(cut_external_metadata['Local Site Name'])

                                    #convert raw reported units given to standard name
                                    #if temperature included in units, add it to metadata
                                    if '(0 C)' in unique_unit:
                                        site_metadata['network_provided_volume_standard_temperature']['value'].append('273.15')
                                        site_metadata['network_provided_volume_standard_pressure']['value'].append('1013.25')
                                        unique_unit = unique_unit.split('(0 C)')[0].strip()
                                    elif '(20 C)' in unique_unit:
                                        site_metadata['network_provided_volume_standard_temperature']['value'].append('293.15')
                                        site_metadata['network_provided_volume_standard_pressure']['value'].append('1013.25')
                                        unique_unit = unique_unit.split('(20 C)')[0].strip()
                                    elif '(25 C)' in unique_unit:
                                        site_metadata['network_provided_volume_standard_temperature']['value'].append('298.15')
                                        site_metadata['network_provided_volume_standard_pressure']['value'].append('1013.25')
                                        unique_unit = unique_unit.split('(25 C)')[0].strip()
                                    elif '(LC)' in unique_unit:
                                        site_metadata['network_provided_volume_standard_temperature']['value'].append(np.NaN)
                                        site_metadata['network_provided_volume_standard_pressure']['value'].append(np.NaN)
                                        unique_unit = unique_unit.split('(LC)')[0].strip()
                                        unknown_gas_volume = True
                                    #break if have unknown units with a temperature
                                    elif ('(') in unique_unit:  
                                        self.run_details['log_file_str'] += 'UNKNOWN UNITS:{}'.format(unique_unit)
                                        create_GHOST_stage_logfile(2,self.run_details)                          
                                    cleaned_units = unit_dict[unique_unit]
                                    conv_obj = unit_converter.convert_units(cleaned_units, cleaned_units, 1, measured_species = self.parameter_details['chemical_formula'])
                                    site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

                                    #parse station classifications
                                    native_classifications = {'area_classification':cut_external_metadata['Location Setting'], 'land_use':cut_external_metadata['Land Use'], 'measurement_scale':cut_external_metadata['Measurement Scale']}
                                    site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

                                    #process measurement information
                                    if unique_method_code != '':
                                        unique_method_code = str(int(unique_method_code))
                                    #handle some small inconsistencies for PM2.5 methods across parameter codes
                                    if unique_parameter_code == '88502':
                                        if unique_method_code in ['183','702','771']:
                                            unique_method_code = '{}_pm2.5_alt'.format(unique_method_code)
                                    #handle inconsistencies for PM10/PM2.5 BC and EC
                                    elif self.run_details['parameter'] in ['PM10_EC','PM10_OC','PM2.5_EC','PM2.5_OC']:
                                        if unique_method_code in ['815','816','838','843','883','888','892']:
                                            #tor methods
                                            if unique_parameter_code in ['85370','85380','88320','88321','88370','88380']:
                                                unique_method_code = '{}_tor'.format(unique_method_code)
                                            #tot methods
                                            elif unique_parameter_code in ['84307','84305','85355','85357','88305','88307','88355','88357','88381','88382']:
                                                if (unique_method_code == '843') & (unique_parameter_code in ['88305','88307']):
                                                    unique_method_code = '{}_tot_alt'.format(unique_method_code)
                                                else:
                                                    unique_method_code = '{}_tot'.format(unique_method_code)
                                    #handle inconsistencies for other speciated pm
                                    elif self.parameter_details['matrix'] in ['pm10','pm2.5']:
                                        #cl diffs
                                        if ('_Cl-' in self.run_details['parameter']) & (unique_method_code in ['92']):
                                            unique_method_code = '{}_cl_alt'.format(unique_method_code)
                                        #no3 diffs
                                        elif ('_NO3-' in self.run_details['parameter']) & (unique_method_code in ['92','875']):
                                            unique_method_code = '{}_no3_alt'.format(unique_method_code)
                                        #pb diffs
                                        elif ('_Pb' in self.run_details['parameter']) & (unique_method_code in ['805','806','807','811']):
                                            unique_method_code = '{}_pb_alt'.format(unique_method_code)
                                        #pm2.5 diffs 
                                        elif (self.parameter_details['matrix'] == 'pm2.5') & (unique_method_code == '820'):
                                            unique_method_code = '{}_pm2.5_alt'.format(unique_method_code)
                                        #other diffs
                                        elif (unique_method_code == '102') & (self.run_details['parameter'] in ['PM10_Cl-','PM10_K+','PM10_NH4+','PM10_NO3-','PM10_SO4--']):
                                            unique_method_code = '{}_alt'.format(unique_method_code)
                                        elif (unique_method_code == '103') & (self.run_details['parameter'] in ['PM10_NO3-','PM10_SO4--']):
                                            unique_method_code = '{}_alt'.format(unique_method_code)

                                    site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(unique_method_code, site_metadata, self.run_details, self.parameter_details)
                                    #set measurement qa codes which are True as joined metadata str
                                    site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
                                    measurement_qa_flag_codes = []
                                    for standard_flag_name in measurement_qa_flags.keys():
                                        if measurement_qa_flags[standard_flag_name]:
                                            measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
                                    site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

                                    #iterate through metadata variables, if any columns have not been appended to add a NaN
                                    for column_header in list(site_metadata.keys()):
                                        n_column = len(site_metadata[column_header]['value'])
                                        #if column is empty, then append NaN
                                        if n_column < 1:
                                            site_metadata[column_header]['value'].append(np.NaN)

                                    #set station_reference (provided network ID + standard measurement method abbreviation)
                                    site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

                                    #get data file download timestamp
                                    site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

                                    #add data revision timestamp, set as 0001-01-01 00:00 for all files
                                    data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
                                    site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

                                    #------------------------------------------------------------------------#
                                    #PARSE TIME
                                    #------------------------------------------------------------------------#  

                                    #for hourly resolution, timezone is GMT
                                    if file_resolution == 'hourly': 
                                        tzinfo = dateutil.tz.gettz("GMT")
                                    #for daily resolution, get local timezone offset from UTC
                                    elif file_resolution == 'daily': 
                                        local_timezone = local_timezone_name(longitude=float(site_metadata['longitude']['value'][-1]), latitude=float(site_metadata['latitude']['value'][-1]))
                                        tzinfo = dateutil.tz.gettz(local_timezone)        

                                    #convert timezone to UTC
                                    start_dt = [round_to_minute(dt.replace(tzinfo=tzinfo).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for dt in final_parsed_data['start_datetime']]
  
                                    #convert each start measurement datetime to minutes from reference time
                                    frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]
        
                                    #get timedeltas between start and end of measurement window
                                    temporal_resolutions = []
                                    end_dt = []
                                    for start in start_dt:
                                        if file_resolution == 'hourly': 
                                            end_dt.append(start + relativedelta(hours=1))
                                        elif file_resolution == 'daily': 
                                            end_dt.append(start + relativedelta(days=1))
                                        dt_window = end_dt[-1] - start
                                        #convert hours to minutes
                                        dt_window_minutes = int(dt_window.total_seconds()/60.)
                                        temporal_resolutions.append(dt_window_minutes)  

                                    #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
                                    site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

                                    #------------------------------------------------------------------------#
                                    #PARSE QA/FLAGS
                                    #------------------------------------------------------------------------#

                                    flags = []
                                    qa = []

                                    #get dictionary necessary for mapping of US EPA AQS file data flags to standard data flag codes
                                    data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['US_EPA_AQS']  

                                    #set data flags
                                    if file_resolution == 'hourly':
                                        data_flags = final_parsed_data['Qualifier']
                                    elif file_resolution == 'daily':
                                        data_flags = final_parsed_data['Observation Percent']

                                    for data_flag in data_flags:

                                        #lists to append qa/data flags per measurement
                                        flags_per_measurement = []
                                        qa_per_measurement = []

                                        #set GHOST decreed validity/network decreed validity variables initially as 'V'
                                        ghost_decreed_validity = 'V'
                                        network_decreed_validity = 'V'

                                        if file_resolution == 'daily':
                                            if data_flag == '':
                                                pass     
                                            elif float(data_flag) < 90:
                                                data_flag = 'DAILY_<90'
                                            elif float(data_flag) < 75:
                                                data_flag = 'DAILY_<75'
                                            elif float(data_flag) < 66:
                                                data_flag = 'DAILY_<66'
                                            elif float(data_flag) < 50:
                                                data_flag = 'DAILY_<50'
                                            elif float(data_flag) < 25:
                                                data_flag = 'DAILY_<25'   
                                            else:
                                                data_flag = '' 
        
                                        if data_flag != '':
                                            # map data flag/s and append
                                            standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                                            #get mapped standard data flags
                                            standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                                            #iterate through standard data flags 
                                            for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                                                #check if standard data flag is empty (i.e. unmapped as non-useful)
                                                if standard_data_flag_name != '': 
                                                    #get corresponding data flag code for standard data flag
                                                    flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                                                    #append standard data flag
                                                    if flag_code not in flags_per_measurement:
                                                        flags_per_measurement.append(flag_code)
                                                    #determine if GHOST/network decreed valid
                                                    if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                                        ghost_decreed_validity = 'I'
                                                    if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                                        network_decreed_validity = 'I'

                                        #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                                        if ghost_decreed_validity == 'I':
                                            qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                                        if network_decreed_validity == 'I':
                                            qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])

                                        #write corrected parameter qa flag if necessary
                                        if corrected_variable:
                                            qa_per_measurement.append(standard_QA_name_to_QA_code['Corrected Parameter'])
    
                                        # add timezone doubt qa flag if file_resolution is daily for each measurement, as get timezone from longitude/latitude
                                        if file_resolution == 'daily': 
                                            qa_per_measurement.append(standard_QA_name_to_QA_code['Timezone Doubt'])

                                        #append qa_per_measurement, flags_per_measurement to lists to write out
                                        flags.append(flags_per_measurement)
                                        qa.append(qa_per_measurement)

                                    #------------------------------------------------------------------------#
                                    #PARSE MEASUREMENTS
                                    #------------------------------------------------------------------------#    
                                    #EPA AQS measurements all should come with units which indicate the measurement standards.

                                    #however if these are not set, the US EPA standard pressure and temperatures are used instead
                                    #these are defined as for 298.15 K temperature, and 1013.25 hPa for pressure here for all compounds, except PM2.5 and lead: https://www.govinfo.gov/content/pkg/CFR-2011-title40-vol2/pdf/CFR-2011-title40-vol2-chapI.pdf
                                    #for PM2.5 and lead the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
                                    #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.
                            
                                    #also provides reported uncertainty per measurement, and limit of detection per measurement         

                                    measurements = []
                                    uncertainty = []
                                    lod = []

                                    standard_network_gas_volume = False
                                    unknown_gas_volume = False
                                    if (pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                                        if (self.parameter_details['matrix'] == 'pm2.5') or ('Pb' in self.run_details['parameter']):
                                            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                                            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
                                            unknown_gas_volume = True
                                        else:
                                            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '298.15'
                                            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.25'
                                            standard_network_gas_volume = True
                            
                                    #get standard reported units/quantity
                                    standard_reported_units = conv_obj.output_standard_units
                                    measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity 

                                    #set flags for writing QA flags associated with assumptions in unit conversion
                                    unit_conversion_standard_network = False
                                    unit_conversion_educated_guess = False

                                    #check if reported standard units, match required standard units
                                    if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                                        #mass density to mixing ratio?
                                        if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                                            #get conversion factor
                                            input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                                            #do conversion with network provided sample gas temperature/pressure values
                                            #if these are not set, then make educated guess (assume standards from particulates)
                                            if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                                                input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                                                unit_conversion_standard_network = True
                                            else:
                                                input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                                                unit_conversion_educated_guess = True
                                            conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
    
                                        #mixing ratio to mass density?
                                        elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                                            input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                                            #do conversion with network provided sample gas temperature/pressure values
                                            #if these are not set, then make educated guess (assume standards from particulates)
                                            if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                                                input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                                                unit_conversion_standard_network = True
                                            else:
                                                input_vals = {'temperature':'298.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                                                unit_conversion_educated_guess = True
                                            conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
   
                                        #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                                        else:
                                            conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                                        #process through each measurement/uncertainty/lod and convert value using derived conversion factor
                                        for n, meas in enumerate(final_parsed_data['measurement']):
                                            #only do conversion if measurement value is above 0 and not infinite, and do not have data flag stating the measurement is missing 
                                            try:
                                                float_meas = float(meas)
                                            except:
                                                float_meas = np.NaN
                                            if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                                measurements.append(float_meas*conv_obj.conversion_factor)
                                            else:
                                                measurements.append(float_meas)

                                            if file_resolution == 'hourly':
                                                #only do conversion if uncertainty value is above 0 and not infinite
                                                unc = final_parsed_data['Uncertainty'][n]
                                                try:
                                                    float_unc = float(unc)
                                                except:
                                                    float_unc = np.NaN
                                                if (float_unc >= 0.0) & (np.isfinite(float_unc)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                                    uncertainty.append(float_unc*conv_obj.conversion_factor)
                                                else:
                                                    uncertainty.append(float_unc)

                                                #only do conversion if lod value is >= 0 and not infinite
                                                lim = final_parsed_data['MDL'][n]
                                                try:
                                                    float_lim = float(lim)
                                                except:
                                                    float_lim = np.NaN
                                                if (float_lim >= 0.0) & (np.isfinite(float_lim)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                                    lod.append(float_lim*conv_obj.conversion_factor)
                                                else:
                                                    lod.append(float_lim)

                                            #append sample gas volume/unit_conversion assumption QA flags if necessary
                                            if standard_network_gas_volume:
                                                qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                                            if unknown_gas_volume:
                                                qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                                            if unit_conversion_standard_network:
                                                qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                                            if unit_conversion_educated_guess:
                                                qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
        
                                    #otherwise do nothing to measured/uncertainty/lod values
                                    else:
                                        for n, meas in enumerate(final_parsed_data['measurement']):
                                            try:
                                                float_meas = float(meas)
                                            except:
                                                float_meas = np.NaN
                                            measurements.append(float_meas)

                                            if file_resolution == 'hourly':
                                                unc = final_parsed_data['Uncertainty'][n]
                                                try:
                                                    float_unc = float(unc)
                                                except:
                                                    float_unc = np.NaN
                                                uncertainty.append(float_unc)
                                        
                                                lim = final_parsed_data['MDL'][n]
                                                try:
                                                    float_lim = float(lim)
                                                except:
                                                    float_lim = np.NaN
                                                lod.append(float_lim)

                                    #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
                                    valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
                                    if len(valid_temporal_resolution_inds) != len(frac_time):
                                        frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                                        temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                                        flags = [flags[x] for x in valid_temporal_resolution_inds]
                                        qa = [qa[x] for x in valid_temporal_resolution_inds]
                                        measurements = [measurements[x] for x in valid_temporal_resolution_inds]
                                        if file_resolution == 'hourly':
                                            lod = [lod[x] for x in valid_temporal_resolution_inds]
                                            uncertainty = [uncertainty[x] for x in valid_temporal_resolution_inds]

                                    #----------------------------------------------------------------------------#
                                    #WRITE OUT DATA
                                    #----------------------------------------------------------------------------# 
                                    site_data['time']['value'] = frac_time
                                    site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
                                    site_data['temporal_resolution']['value'] = temporal_resolutions   
                                    site_data['flag']['value'] = flags
                                    site_data['qa']['value'] = qa
                                    if file_resolution == 'hourly':
                                        site_data['reported_lower_limit_of_detection_per_measurement']['value'] = lod 
                                        site_data['reported_uncertainty_per_measurement']['value'] = uncertainty

                                    #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
                                    for column_header in list(site_data.keys()):
                                        n_column = len(site_data[column_header]['value'])
                                        #if column is empty, then append NaN
                                        if n_column == 0:
                                            if site_data[column_header]['data_type'] != np.uint8:
                                                site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                                            else:
                                                site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
                                    #------------------------------------------------------

                                    #standardise metadata/data formats and write out station data to numpy file
                                    write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
                                    if write_obj.written == False:
                                        self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)