import copy
import datetime
import os
import re
import sys
import time

import dateutil
import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseCAPMoNFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, year, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.year = year
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current year
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)


    def gather_data_files(self):

        '''Function to gather data files to parse per year'''

        #get specific data files for current year
        data_files = get_data_files(self.run_details, self.parameter_details, specific_year=self.year)
   
        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_files

    def parse_file(self, data_file):

        '''Function to a specific annual data file to GHOST standard numpy station specific files'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file)
        
        #create object for parsing CAPMoN annual file, and read file
        parsing_obj = ParseCAPMoNFile(data_file, self.run_details['parameter'], self.parameter_details)
        parsing_obj.read_data()
        
        #get unique station references
        unique_station_references = np.unique(parsing_obj.parsed_data['siteid'])

        #iterate over all unique station references
        for unique_station_reference in unique_station_references:

            self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_reference)

            #cut read data to just that associated with unique station reference & parameter being processed
            valid_inds = np.where(parsing_obj.parsed_data['siteid'] == unique_station_reference)[0]
    
            #if no valid inds then continue to next station reference
            if len(valid_inds) == 0:
                continue

            #get cut data variables for station
            start_dt = parsing_obj.parsed_data['start_dt'][valid_inds]
            end_dt = parsing_obj.parsed_data['end_dt'][valid_inds]
            cut_measurements = parsing_obj.parsed_data['measurements'][valid_inds]
            cut_flags = parsing_obj.parsed_data['flags'][valid_inds]
            data_file_year = int(data_file.split('-')[-1][:4])
            if (self.run_details['parameter'] == 'O3') or ('PREC' in self.run_details['parameter']):
                cut_lods = parsing_obj.parsed_data['lods'][valid_inds]
            else:
                start_dt_years = np.array([dt.year for dt in start_dt], dtype=np.uint32)

            #get index for station specific metadata
            site_meta_index = np.where(parsing_obj.parsed_metadata['siteid'] == unique_station_reference)[0][0] 

            #split file by different measurement configurations
            #for O3 by unique detection limits / for particulates by year (detection limits change per year)
            if (self.run_details['parameter'] == 'O3') or ('PREC' in self.run_details['parameter']):
                unique_configs  = np.unique(cut_lods)
            else:
                unique_configs  = np.unique(start_dt_years)

            #iterate through unique measurement configurations
            for unique_config in unique_configs:
                
                #get data associated with unique config
                #if no valid data per config, continue to next config
                if (self.run_details['parameter'] == 'O3') or ('PREC' in self.run_details['parameter']):
                    valid_inds_config = np.where(cut_lods == unique_config)[0]
                    if len(valid_inds_config) == 0:
                        continue
                    cut_lods_config = cut_lods[valid_inds_config]
                else:
                    valid_inds_config = np.where(start_dt_years == unique_config)[0]
                    if len(valid_inds_config) == 0:
                        continue
                    #get index for detection limit for year/station reference      
                    site_lod_index = np.where((parsing_obj.parsed_detection_limits['siteid'] == unique_station_reference) & (parsing_obj.parsed_detection_limits['year'] == str(unique_config)))[0][0]

                start_dt_config = start_dt[valid_inds_config]
                end_dt_config = end_dt[valid_inds_config]
                cut_measurements_config = cut_measurements[valid_inds_config]
                cut_flags_config = cut_flags[valid_inds_config]

                #create site instances of metadata/data dictionaries
                site_data =     get_standard_data(self.parameter_details)
                site_metadata = get_standard_metadata(self.parameter_details)

                #------------------------------------------------------------------------#
                #PARSE METADATA
                #------------------------------------------------------------------------#

                #GHOST version
                site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

                #initialise string which will store all GHOST process warnings as empty            
                site_metadata['process_warnings']['value'].append('')

                #write known metadata  
                site_metadata['network']['value'].append(self.run_details['network'])
                if self.run_details['parameter'] == 'O3':
                    site_metadata['data_level']['value'].append('2.0')   
                else:
                    site_metadata['data_level']['value'].append('3.0')
                site_metadata['contact_institution']['value'].append('Environment and Climate Change Canada')      
                site_metadata['contact_email_address']['value'].append('enviroinfo@canada.ca')    
                if self.run_details['parameter'] != 'O3':
                    site_metadata['principal_investigator_name']['value'].append('Hayley Hung')
                    site_metadata['principal_investigator_institution']['value'].append('Environment and Climate Change Canada')
                    site_metadata['principal_investigator_email_address']['value'].append('hayley.hung@canada.ca')
                site_metadata['data_licence']['value'].append('Public domain: http://open.canada.ca/en/open-government-licence-canada') 

                #non-standard comments
                if self.run_details['parameter'] == 'O3':
                    site_metadata['network_sampling_details']['value'].append('Inlet Type: Filter in front of sampling line')
                    site_metadata['network_qa_details']['value'].append('Concentrations are adjusted to the National Institute of Standards and Technology Standard Reference Photometer (NIST SRP) #12 for 2001-2002 and to NIST SRP #16 for post-2013. Prior to 2013, no zero or negative hourly-average mixing ratios were reported. From 2013 forward, zero and negative hourly-average mixing ratios were reported and flagged as below the measurement detection limit. These zero or negative values resulted from the application of NIST-referenced calibration curves to very low mixing ratio values. Values of -999.0 represent missing or invalid data.')
                    site_metadata['network_maintenance_details']['value'].append('Blank correction: Not blank corrected')
                    site_metadata['network_uncertainty_details']['value'].append('Uncertainty of hourly average concentrations is 0.6 ppb (0.8 ppb in warm season, 0.5 ppb in cold season). The uncertainty is based on simultaneous field measurements of two collocated instruments both adjusted to National Institute of Standards and Technology Standard Reference Photometer (NIST/SRP) #16. The uncertainty was calculated as the standard deviation of differences from the true value assumed equal to the mean of the two instruments.')  
                elif 'PREC' in self.run_details['parameter']:
                    site_metadata['sampling_height']['value'].append('1.5') # same for all stations
                    # site_metadata['network_sampling_details']['value'].append('')
                    # site_metadata['network_qa_details']['value'].append('Concentrations are adjusted to the National Institute of Standards and Technology Standard Reference Photometer (NIST SRP) #12 for 2001-2002 and to NIST SRP #16 for post-2013. Prior to 2013, no zero or negative hourly-average mixing ratios were reported. From 2013 forward, zero and negative hourly-average mixing ratios were reported and flagged as below the measurement detection limit. These zero or negative values resulted from the application of NIST-referenced calibration curves to very low mixing ratio values. Values of -999.0 represent missing or invalid data.')
                    # site_metadata['network_maintenance_details']['value'].append('Blank correction: Not blank corrected')
                    # site_metadata['network_uncertainty_details']['value'].append('Uncertainty of hourly average concentrations is 0.6 ppb (0.8 ppb in warm season, 0.5 ppb in cold season). The uncertainty is based on simultaneous field measurements of two collocated instruments both adjusted to National Institute of Standards and Technology Standard Reference Photometer (NIST/SRP) #16. The uncertainty was calculated as the standard deviation of differences from the true value assumed equal to the mean of the two instruments.')  
                else:
                    site_metadata['network_sampling_details']['value'].append('Inlet Type: Impactor--direct; Filter Medium: Whatman41 Cellulose')
                    site_metadata['network_qa_details']['value'].append('A comparison between the ICP-ES and ICP-MS analysis was performed for the 1995 samples showing a good correlation, but some bias. Corrections were performed on the pre-1995 data to correct for this bias and ensure a reliable long-term trend.')
                    site_metadata['network_maintenance_details']['value'].append('Blank correction: Not blank corrected')
                    site_metadata['network_uncertainty_details']['value'].append('Instrument Detection Limits are provided by the contract laboratory. They are divided by the average sample volume for that site and year to give a value in the same units as the samples. Values that are reported by the laboratory as below the detection limit are reported as two-thirds of the detection limit before 2016, whereafter the laboratory reports values that are below the detection limit. Compounds that are not detected are replaced with two-thirds the instrument detection limit of that compound.')
                    site_metadata['network_miscellaneous_details']['value'].append('The filters were cut into quarters and one quarter was sent to the laboratory for analysis. The sample was extracted in a nitric acid solution before subsequent analysis.')

                #datum information
                if parsing_obj.parsed_metadata['latlonrefdatum'][site_meta_index] == 'WGS84 (World Geodetic System 1984)':
                    site_metadata['ellipsoid']['value'].append('WGS 84') 
                    site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
                else:
                    self.run_details['log_file_str'] += 'UNKNOWN DATUM:{}'.format(parsing_obj.parsed_metadata['latlonrefdatum'][site_meta_index])
                    create_GHOST_stage_logfile(2,self.run_details)
                site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
                site_metadata['projection']['value'].append('geographic')    

                #site metadata
                site_metadata['longitude']['value'].append(parsing_obj.parsed_metadata['longitude_deg'][site_meta_index])
                site_metadata['latitude']['value'].append(parsing_obj.parsed_metadata['latitude_deg'][site_meta_index])
                site_metadata['altitude']['value'].append(parsing_obj.parsed_metadata['groundelevamsl_m'][site_meta_index])
                site_metadata['station_name']['value'].append(parsing_obj.parsed_metadata['sitename'][site_meta_index])
                #sampling height is variable over time, get correct height for data file year
                if ('samplingheightag' in list(parsing_obj.parsed_metadata.keys())) and (parsing_obj.parsed_metadata['samplingheightag'] != []):
                    sampling_height = parsing_obj.parsed_metadata['samplingheightag'][site_meta_index].replace(" ", "")
                    if ';' in sampling_height:
                        sampling_height_split = sampling_height.split(';')
                        for sampling_height_pair in sampling_height_split:
                            sampling_height_pair_split = sampling_height_pair.split(':')
                            if 'onwards' in sampling_height_pair_split[0]:
                                sampling_height_pair_split[0] = sampling_height_pair_split[0].replace("onwards", self.run_details['end_date'][:4])
                                sampling_height_years = sampling_height_pair_split[0].split('-')
                                sampling_height_start_year = int(sampling_height_years[0][-4:])
                                sampling_height_end_year = int(sampling_height_years[1][-4:])
                            else:
                                sampling_height_years = sampling_height_pair_split[0].split('-')
                                sampling_height_start_year = int(sampling_height_years[0][-4:])
                                sampling_height_end_year = int(sampling_height_years[1][-4:])+1
                            if (data_file_year >= sampling_height_start_year) & (data_file_year < sampling_height_end_year):
                                sampling_height = sampling_height_pair_split[1][:-1]
                                break
                    else:
                        sampling_height = sampling_height[:-1]
                    site_metadata['sampling_height']['value'].append(sampling_height)

                #save reported units
                if self.run_details['parameter'] == 'O3':
                    reported_units = 'nmol mol-1'
                elif self.run_details['parameter'] == 'PREC_ACCUM':
                    reported_units = 'mm'
                elif '_PREC' in self.run_details['parameter']:
                    reported_units = 'mg l-1'
                else:
                    reported_units = 'ng m-3'
                #convert raw reported units given to standard name
                conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
                site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

                #parse station classifications
                native_classifications = {'area_classification':parsing_obj.parsed_metadata['sitelanduse'][site_meta_index]}
                site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)

                #process measurement process information
                if self.run_details['parameter'] == 'O3':
                    if unique_station_reference in ['ALT','RTR']:
                        if data_file_year < 2013:
                            instrument_str = 'Thermo 49C'
                        elif data_file_year >= 2013:
                            instrument_str = 'Thermo 49i' 
                    else:
                        if (data_file_year >= 1988) & (data_file_year < 1992):
                            instrument_str = 'Dasibi'
                        elif(data_file_year >= 1992) & (data_file_year < 2003): 
                            instrument_str = 'Thermo 49'
                        elif(data_file_year >= 2003) & (data_file_year < 2013):  
                            instrument_str = 'Thermo 49C'
                        elif(data_file_year >= 2013):
                            if unique_config == '2.0':
                                instrument_str = 'Thermo 49C'
                            elif unique_config == '1.0':
                                instrument_str = 'Thermo 49i'
                elif 'PREC' in self.run_details['parameter']:
                    instrument_str = str(parsing_obj.parsed_metadata['analytical_method'])
                    instrument_str = re.sub("\(.*?\)","()",instrument_str).replace('(', '').replace(')', '')[:-1]
                else:
                    if unique_config < 1995:
                        instrument_str = 'high volume ICP-ES'
                    elif unique_config < 2006:
                        instrument_str = 'high volume ICP-MS'
                    elif unique_config >= 2006:
                        instrument_str = 'Tisch Environmental 6070V ICP-MS'

                site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(instrument_str, site_metadata, self.run_details, self.parameter_details)
                #set measurement qa codes which are True as joined metadata str
                site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
                measurement_qa_flag_codes = []
                for standard_flag_name in measurement_qa_flags.keys():
                    if measurement_qa_flags[standard_flag_name]:
                        measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
                site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

                #add reported detection limits 
                if self.run_details['parameter'] == 'O3':
                    if unique_config not in ['4.0','2.0','1.0']:
                        self.run_details['log_file_str'] += 'UNEXPECTED DETECTION LIMIT:{}'.format(unique_config)
                        create_GHOST_stage_logfile(2,self.run_details)
                    reported_lower_detection_limit = copy.deepcopy(unique_config)
                elif 'PREC' in self.run_details['parameter']:
                    reported_lower_detection_limit = copy.deepcopy(unique_config)
                else:
                    reported_lower_detection_limit = parsing_obj.parsed_detection_limits['instrument_detection_limit'][site_lod_index]
                site_metadata['measuring_instrument_reported_lower_limit_of_detection']['value'].append(reported_lower_detection_limit)

                #iterate through metadata variables, if any columns have not been appended to add a NaN
                for column_header in list(site_metadata.keys()):
                    n_column = len(site_metadata[column_header]['value'])
                    #if column is empty, then append NaN
                    if n_column < 1:
                        site_metadata[column_header]['value'].append(np.NaN)
    
                #set station_reference (provided network ID + standard measurement method abbreviation)
                site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

                #get data file download timestamp
                site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

                #add data revision timestamp, from file metadata
                file_revision_date_split = parsing_obj.parsed_metadata['file_revision_date'][0].split('-')
                data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(year=int(file_revision_date_split[0]),month=int(file_revision_date_split[1]),day=int(file_revision_date_split[2]),hour=0,minute=0))
                site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

                #------------------------------------------------------------------------#
                #PARSE TIME
                #------------------------------------------------------------------------#   
                #round datetimes to nearest minute
                start_dt_config = [round_to_minute(dt) for dt in start_dt_config]
                end_dt_config = [round_to_minute(dt) for dt in end_dt_config]

                #convert each start measurement datetime to minutes from reference time
                frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt_config]
            
                #get timedeltas between start and end of measurement window
                temporal_resolutions = []
                for start, end in zip(start_dt_config,end_dt_config):
                    dt_window = end - start
                    #convert hours to minutes
                    dt_window_minutes = int(dt_window.total_seconds()/60.)
                    temporal_resolutions.append(dt_window_minutes)  

                #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
                site_metadata['meta_update_stamp']['value'][-1] = sorted(frac_time)[0]

                #------------------------------------------------------------------------#
                #PARSE QA/FLAGS
                #------------------------------------------------------------------------#

                flags = []
                qa = []

                #get dictionary necessary for mapping CAPMoN file data flags to standard data flag codes
                data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['CAPMoN']  

                for data_flag in cut_flags_config:

                    #lists to append qa/data flags per measurement
                    flags_per_measurement = []
                    qa_per_measurement = []

                    #set GHOST decreed validity/network decreed validity variables initially as 'V'
                    ghost_decreed_validity = 'V'
                    network_decreed_validity = 'V'

                    # map data flag/s and append
                    standard_data_flag_dict = data_flag_mapping_dictionary[data_flag]
                    #get mapped standard data flags
                    standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                    #iterate through standard data flags 
                    for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                        #check if standard data flag is empty (i.e. unmapped as non-useful)
                        if standard_data_flag_name != '': 
                            #get corresponding data flag code for standard data flag
                            flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                            #append standard data flag
                            if flag_code not in flags_per_measurement:
                                flags_per_measurement.append(flag_code)
                            #determine if GHOST/network decreed valid
                            if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                                ghost_decreed_validity = 'I'
                            if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                                network_decreed_validity = 'I'

                    #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
                    if ghost_decreed_validity == 'I':
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
                    if network_decreed_validity == 'I':
                        qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])
        
                    #append qa_per_measurement, flags_per_measurement to lists to write out
                    flags.append(flags_per_measurement)
                    qa.append(qa_per_measurement)

                #------------------------------------------------------------------------#
                #PARSE MEASUREMENTS
                #------------------------------------------------------------------------#    
                #for O3, measurement temperature and pressure values are never explicitly provided by CAPMoN, but units are always ppbV
                #for particulate matter the measurements are standardised to 273.15K, and 1013.25hPa (stated in measurement file)

                measurements = []
                lod = []

                standard_network_gas_volume = False
                unknown_gas_volume = False
                if self.run_details['parameter'] == 'O3':
                    site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                    site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
                    unknown_gas_volume = True
                else:
                    site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '273.15'
                    site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.25'
                    standard_network_gas_volume = True

                #get standard reported units/quantity
                standard_reported_units = conv_obj.output_standard_units
                measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity 

                #set flags for writing QA flags associated with assumptions in unit conversion
                unit_conversion_standard_network = False
                unit_conversion_educated_guess = False
    
                #check if reported standard units, match required standard units
                if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                    #mass density to mixing ratio?
                    if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                        #get conversion factor
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume standards from particulates)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
        
                    #mixing ratio to mass density?
                    elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume standards from particulates)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_standard_network = True
                        else:
                            input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                            unit_conversion_educated_guess = True
                        conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
       
                    #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                    else:
                        conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                    #process through each measurement/lod and convert value using derived conversion factor
                    for n, meas in enumerate(cut_measurements_config):
                        #only do conversion if measurement value is above 0 and not infinite, and do not have data flag stating the measurement is missing 
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                        if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                            measurements.append(float_meas*conv_obj.conversion_factor)
                        else:
                            measurements.append(float_meas)

                        if (self.run_details['parameter'] == 'O3') or ('PREC' in self.run_details['parameter']):
                            #only do conversion if lod value is >= 0 and not infinite
                            lim = cut_lods_config[n]
                            try:
                                float_lim = float(lim)
                            except:
                                float_lim = np.NaN
                            if (float_lim >= 0.0) & (np.isfinite(float_lim)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                                lod.append(float_lim*conv_obj.conversion_factor)
                            else:
                                lod.append(float_lim)

                        #append sample gas volume/unit_conversion assumption QA flags if necessary
                        if standard_network_gas_volume:
                            qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                        if unknown_gas_volume:
                            qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                        if unit_conversion_standard_network:
                            qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                        if unit_conversion_educated_guess:
                            qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
            
                #otherwise do nothing to measured/lod values
                else:
                    for n, meas in enumerate(cut_measurements_config):
                        try:
                            float_meas = float(meas)
                        except:
                            float_meas = np.NaN
                        measurements.append(float_meas)

                        if (self.run_details['parameter'] == 'O3') or ('PREC' in self.run_details['parameter']):
                            lim = cut_lods_config[n]
                            try:
                                float_lim = float(lim)
                            except:
                                float_lim = np.NaN
                            lod.append(float_lim)

                #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
                valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
                if len(valid_temporal_resolution_inds) != len(frac_time):
                    frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                    temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                    flags = [flags[x] for x in valid_temporal_resolution_inds]
                    qa = [qa[x] for x in valid_temporal_resolution_inds]
                    measurements = [measurements[x] for x in valid_temporal_resolution_inds]
                    if (self.run_details['parameter'] == 'O3') or ('PREC' in self.run_details['parameter']):
                        lod = [lod[x] for x in valid_temporal_resolution_inds]

                #----------------------------------------------------------------------------#
                #WRITE OUT DATA
                #----------------------------------------------------------------------------# 
                site_data['time']['value'] = frac_time
                site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
                site_data['temporal_resolution']['value'] = temporal_resolutions   
                site_data['flag']['value'] = flags
                site_data['qa']['value'] = qa
                if (self.run_details['parameter'] == 'O3') or ('PREC' in self.run_details['parameter']):
                    site_data['reported_lower_limit_of_detection_per_measurement']['value'] = lod 

                #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
                for column_header in list(site_data.keys()):
                    n_column = len(site_data[column_header]['value'])
                    #if column is empty, then append NaN
                    if n_column == 0:
                        if site_data[column_header]['data_type'] != np.uint8:
                            site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                        else:
                            site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
                #------------------------------------------------------

                #standardise metadata/data formats and write out station data to numpy file
                write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
                if write_obj.written == False:
                    self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)