import copy
import datetime
import os
import re
import sys
import time

import numpy as np
import pandas as pd
import json

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseMITECOFile
from standardisation_mapping_functions import EEA_external_meta_map, EEA_AQ_eReporting_AIDE_D_external_meta_map, EEA_AIRBASE_external_meta_map, MITECO_external_meta_map, map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


#define dictionary storing measurement units per species
standard_reporting_units = {
'O3':'ug/m3', 'NO':'ug/m3', 'NO2':'ug/m3', 'SO2':'ug/m3', 'CO':'mg/m3', 'BENZENE':'ug/m3', 'TOLUENE':'ug/m3', 'XYLENE':'ug/m3', 
'PM10':'ug/m3', 'PM2.5':'ug/m3', 
'PM10_As':'ng/m3', 'PM10_Cd':'ng/m3', 'PM10_Ni':'ng/m3', 'PM10_Pb':'ug/m3', 'PM10_BaP':'ng/m3', 
}


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific year into GHOST standard numpy station specific files'''

    def __init__(self, year, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.year = year
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details
        

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files for current year
        data_files = self.gather_data_files()

        #parse data files (writing standard files out per station)
        for data_file in data_files:
            self.parse_file(data_file)

        #write a valid process log file
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details)

    
    def gather_data_files(self):

        '''Function to gather data files to parse per year'''

        #get specific data files for current month
        data_files = get_data_files(self.run_details, self.parameter_details, specific_year=self.year)

        # load external meta data
        with open(self.run_details['data_root']+'{}/metadata/{}/processed/{}_META.json'.format(self.run_details['network'], self.run_details['ghost_version'], self.run_details['network']), 'r', encoding='utf-8') as f:
            self.external_metadata_MITECO = json.loads(f.read())

        # load external meta data from EEA
        with open(self.run_details['data_root']+'EEA/metadata/{}/processed/EEA_ES_META.json'.format(self.run_details['ghost_version']), 'r', encoding='utf-8') as f:
            self.external_metadata_EEA = json.loads(f.read())
        
        # get code_local of all station for selection of station later
        self.all_station_refs = []
        for station in self.external_metadata_MITECO:
            if 'COD_LOCAL' in list(self.external_metadata_MITECO[station].keys()):
                self.all_station_refs.append(self.external_metadata_MITECO[station]['COD_LOCAL']['values'][-1])
            else:
                self.all_station_refs.append('')
    
        #get standard measured unit quantity
        conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
        self.measured_unit_quantity = conv_obj.output_represented_quantity

        return data_files


    def parse_file(self, data_file):

        '''Function to a specific annual data file to GHOST standard numpy station specific files'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
        
        #create object for parsing MITECO annual file, and read file
        parsing_obj = ParseMITECOFile(data_file,self.parameter_details)
        parsing_obj.read_data()

        #get unique sampling points (PUNTO_MUESTREO) 
        unique_sampling_points = np.unique(parsing_obj.punto_muestreo)
    
        #iterate through unique sampling points
        for sampling_point in unique_sampling_points:

            #create site instances of metadata/data dictionaries
            site_data =     get_standard_data(self.parameter_details)
            site_metadata = get_standard_metadata(self.parameter_details)

            #strip data sampling point string to get local station code --> take to be the unique station reference
            split_sampling_point = sampling_point.split('_')[0]
            if split_sampling_point[0] == '0':
                unique_station_reference = split_sampling_point[1:]
            else:
                unique_station_reference = copy.deepcopy(split_sampling_point)

            self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(unique_station_reference)

            #cut read data to just that associated with sampling point
            valid_inds = parsing_obj.punto_muestreo == sampling_point
            start_dt = parsing_obj.start_dt[valid_inds]
            temporal_resolutions = parsing_obj.temporal_resolutions[valid_inds]
            cut_measurements = parsing_obj.measurements[valid_inds]

            #map to external MITECO metadata with unique_station_reference
            if len(np.where(unique_station_reference == np.array(self.all_station_refs))[0]) > 0:
                station_reference = list(self.external_metadata_MITECO.keys())[np.where(unique_station_reference == np.array(self.all_station_refs))[0][0]]
            else:
                self.run_details['log_file_str'] += 'NO MATCHING METADATA FOUND FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)
                continue
            
            # select current station
            cut_external_metadata_MITECO = self.external_metadata_MITECO[station_reference]

            #get eeacode / eea sampling point / air pollutant code etc. to map to EEA external metadata
            eeacode = cut_external_metadata_MITECO['COD_ESTACION_DEM']['values'][-1]
            eea_sampling_point = 'SP_{}'.format(sampling_point)
            data_species_code = 'http://dd.eionet.europa.eu/vocabulary/aq/pollutant/{}'.format(int(self.parameter_details['airbase_code']))
            pollutant_AIDE_D_format = self.parameter_details['airbase_name']
            data_species_code_AIRBASE_format = data_species_code.split('/')[-1]
            #estimate eoicode by taking all characters after underscores/dashes in eeacode to be the reference code
            #if >= 7 characters then keep
            est_eoicode = eeacode.split('-')[-1].split('_')[-1] 
            if len(est_eoicode) >= 7:
                eoicode = copy.deepcopy(est_eoicode) 
            else:
                eoicode = ''

            #EEA EXTERNAL METADATA FILE

            # get all eea codes from external metadata file
            # all_eea_codes_in_metadata = []
            # for station in self.external_metadata_EEA:
            #     if 'Air Quality Station EoI Code' in list(self.external_EEA[station].keys()):
            #         all_eea_codes_in_metadata.append(self.external_metadata_EEA[station]['Air Quality Station EoI Code']['values'][-1])
            #     else:
            #         all_eea_codes_in_metadata.append('')    
                   
            #first try making data/metadata join with EEA external meta file using the "SamplingPoint" (if not empty string), then "EEA code" 
            if (eea_sampling_point != '') and (eea_sampling_point in list(self.external_metadata_EEA.keys())):
                cut_external_metadata_eea = self.external_metadata_EEA[eea_sampling_point]
            # elif (eeacode != '') and (eeacode in all_eea_codes_in_metadata):
            #     if len(np.where(eeacode == np.array(all_eea_codes_in_metadata))[0]) > 0:
            #         eea_sampling_point_meta = list(self.external_metadata_EEA.keys())[np.where(eeacode == np.array(all_eea_codes_in_metadata))[0][0]]
            #         cut_external_metadata_eea = self.external_metadata_EEA[eea_sampling_point_meta]
            #     else:
            #         cut_external_metadata_eea = [] 
            else:
                cut_external_metadata_eea = []           

            #if cut_external_metadata_eea is empty or not specific - next try doing join with "AirQualityStation" and "AirPollutantCode" (if not empty strings)
            # if (eeacode != '') & (data_species_code != ''):
            #     if len(cut_external_metadata_eea) == 0:
            #         cut_external_metadata_eea = self.external_metadata_EEA[np.intersect1d(np.where(self.external_metadata_EEA['AirQualityStation'] == eeacode)[0], np.where(self.external_metadata_EEA['AirPollutantCode'] == data_species_code)[0])]
            #     elif len(cut_external_metadata_eea) > 1:
            #         test_cut_external_metadata_eea = cut_external_metadata_eea[np.intersect1d(np.where(cut_external_metadata_eea['AirQualityStation'] == eeacode)[0], np.where(cut_external_metadata_eea['AirPollutantCode'] == data_species_code)[0])]
            #         if len(test_cut_external_metadata_eea) > 0:
            #             cut_external_metadata_eea = copy.deepcopy(test_cut_external_metadata_eea)              

            # #if cut_external_metadata_eea is empty or not specific - next try doing join with "AirQualityStationEoICode" and "AirPollutantCode" (if not empty strings)
            # if (eoicode != '') & (data_species_code != ''):
            #     if len(cut_external_metadata_eea) == 0:
            #         cut_external_metadata_eea = self.external_metadata_EEA[np.intersect1d(np.where(self.external_metadata_EEA['AirQualityStationEoICode'] == eoicode)[0], np.where(self.external_metadata_EEA['AirPollutantCode'] == data_species_code)[0])]
            #     elif len(cut_external_metadata_eea) > 1:
            #         test_cut_external_metadata_eea = cut_external_metadata_eea[np.intersect1d(np.where(cut_external_metadata_eea['AirQualityStationEoICode'] == eoicode)[0], np.where(cut_external_metadata_eea['AirPollutantCode'] == data_species_code)[0])]
            #         if len(test_cut_external_metadata_eea) > 0:
            #             cut_external_metadata_eea = copy.deepcopy(test_cut_external_metadata_eea)              

            # #check if have specific eea metadata
            # if len(cut_external_metadata_eea) != 0:
            #     #if not specific, preferentially take first index
            #     cut_external_metadata_eea = cut_external_metadata_eea[0]
            #     #if have successfully made join with eea external meta file, overwrite eoicode retrieved for station (if not empty string)
            #     if cut_external_metadata_eea['AirQualityStationEoICode'] != '':
            #         #only take data after underscores/dashes to remove issues with incorrectly formatted codes
            #         eoicode = cut_external_metadata_eea['AirQualityStationEoICode'].split('-')[-1].split('_')[-1] 


            #AIDE D METADATA FILE
            #first try making data/metadata join with AIDE D external metadata file using the "SamplingPointId" (if not empty string)
            # if eea_sampling_point != '':
            #     cut_external_metadata_AIDE_D = self.external_metadata_AIDE_D[np.where(self.external_metadata_AIDE_D['SamplingPointId'] == eea_sampling_point)[0]]
            # else:
            #     cut_external_metadata_AIDE_D = []        

            # #if cut_external_metadata_AIDE_D is empty or not specific - next try doing join with "EoICode" and "Pollutant" (if not empty strings)
            # if (eoicode != '') & (pollutant_AIDE_D_format != ''):
            #     if len(cut_external_metadata_AIDE_D) == 0:
            #         cut_external_metadata_AIDE_D = self.external_metadata_AIDE_D[np.intersect1d(np.where(self.external_metadata_AIDE_D['EoICode'] == eoicode)[0], np.where(self.external_metadata_AIDE_D['Pollutant'] == pollutant_AIDE_D_format)[0])]
            #     elif len(cut_external_metadata_AIDE_D) > 1:
            #         test_cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[np.intersect1d(np.where(cut_external_metadata_AIDE_D['EoICode'] == eoicode)[0], np.where(cut_external_metadata_AIDE_D['Pollutant'] == pollutant_AIDE_D_format)[0])]
            #         if len(test_cut_external_metadata_AIDE_D) > 0:
            #             cut_external_metadata_AIDE_D = copy.deepcopy(test_cut_external_metadata_AIDE_D)           

            #check if have specific AIDE D metadata
            # if (len(cut_external_metadata_AIDE_D) != 0):
            #     #if not specific, preferentially take first index
            #     cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[0]
            #     #if eoicode still empty string, and have made join with AIDE D meta file, then take eoicode retrieved for station (if not empty string)
            #     if eoicode == '':
            #         if cut_external_metadata_AIDE_D['EoICode'] != '':
            #             #only take data after underscores/dashes to remove issues with incorrectly formatted codes
            #             eoicode = cut_external_metadata_AIDE_D['EoICode'].split('-')[-1].split('_')[-1] 


            #AIRBASE v8 METADATA FILE 
            #first try making data/metadata join with AIRBASE external metadata file using the "station_european_code" and "component_code" (if not empty strings)
            # if (eoicode != '') & (data_species_code_AIRBASE_format != ''):
            #     cut_external_metadata_AIRBASE = self.external_metadata_AIRBASE[np.intersect1d(np.where(self.external_metadata_AIRBASE['station_european_code'] == eoicode)[0], np.where(self.external_metadata_AIRBASE['component_code'] == data_species_code_AIRBASE_format)[0])]
            # else:
            #     cut_external_metadata_AIRBASE = []
            # if (len(cut_external_metadata_AIRBASE) == 1):
            #     cut_external_metadata_AIRBASE = cut_external_metadata_AIRBASE[0]
            # #if cut_external_metadata_AIRBASE is not specific - next try doing join taking into account measurement dates
            # elif (len(cut_external_metadata_AIRBASE) > 1):
            #     #select the metadata row with the latest "measurement_end_date"
            #     #convert string metadata dates to datetime
            #     #*first convert empty strings to a date much later in future (i.e. empty string = no end date)
            #     measurement_end_dates = cut_external_metadata_AIRBASE['measurement_end_date']
            #     measurement_end_dates[np.where(measurement_end_dates == '')[0]] = '3000-01-01'
            #     measurement_end_dates = [datetime.datetime.strptime(date, '%Y-%m-%d').date() for date in measurement_end_dates]
            #     #finally get index of latest end date and cut cut_external_metadata_AIRBASE
            #     cut_external_metadata_AIRBASE = cut_external_metadata_AIRBASE[np.argmax(measurement_end_dates)]


            #UNIFY 2 METADATA files information into standardised dict
            cut_unified_metadata = {}

            #first load in MITECO metadata variables
            if len(cut_external_metadata_MITECO) > 0:
                for var in list(self.external_metadata_MITECO.keys()):                    
                    if var in list(MITECO_external_meta_map.keys()):
                        #get standard_name of variable
                        standard_var = MITECO_external_meta_map[var]
                        #get variable value
                        var_value = cut_external_metadata_MITECO[var]['values'][-1]
                        #do not write if empty string, or 'unknown' or '-999'
                        if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                            cut_unified_metadata[standard_var] = var_value

            #next load in EEA metadata variables
            if len(cut_external_metadata_eea) > 0:
                for var in list(cut_external_metadata_eea.keys()):                    
                    if var in list(EEA_external_meta_map.keys()):
                        #get standard_name of variable
                        standard_var = EEA_external_meta_map[var]
                        #do not proceed if already have variable written in cut_unified_metadata dict
                        if standard_var not in list(cut_unified_metadata.keys()):
                            #get variable value
                            var_value = cut_external_metadata_eea[var]['values'][-1]
                            #do not write if empty string, or 'unknown' or '-999'
                            if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                                cut_unified_metadata[standard_var] = var_value

            if ('longitude' not in list(cut_unified_metadata.keys())) or ('latitude' not in list(cut_unified_metadata.keys())): 
                self.run_details['log_file_str'] += 'NO OBLIGATORY METADATA VARIABLES FOUND FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)
                continue

            #next load in AIDE D metadata variables
            # if len(cut_external_metadata_AIDE_D) > 0:
            #     for var in self.external_metadata_AIDE_D.dtype.names:                    
            #         if var in list(EEA_AQ_eReporting_AIDE_D_external_meta_map.keys()):
            #             #get standard_name of variable
            #             standard_var = EEA_AQ_eReporting_AIDE_D_external_meta_map[var]
            #             #do not proceed if already have variable written in cut_unified_metadata dict
            #             if standard_var not in list(cut_unified_metadata.keys()):
            #                 #get variable value
            #                 var_value = cut_external_metadata_AIDE_D[var]
            #                 #do not write if empty string, or 'unknown' or '-999'
            #                 if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
            #                     cut_unified_metadata[standard_var] = var_value
                
            #next load in AIRBASE metadata variables
            # if len(cut_external_metadata_AIRBASE) > 0:
            #     for var in self.external_metadata_AIRBASE.dtype.names:                    
            #         if var in list(EEA_AIRBASE_external_meta_map.keys()):
            #             #get standard_name of variable
            #             standard_var = EEA_AIRBASE_external_meta_map[var]
            #             #do not proceed if already have variable written in cut_unified_metadata dict
            #             if standard_var not in list(cut_unified_metadata.keys()):
            #                 #get variable value
            #                 var_value = cut_external_metadata_AIRBASE[var]
            #                 #do not write if empty string, or 'unknown' or '-999'
            #                 if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
            #                     cut_unified_metadata[standard_var] = var_value

            #------------------------------------------------------------------------#
            #PARSE TIME - always given in GMT/UTC (no conversion neccessary)
            #------------------------------------------------------------------------#

            #convert each start measurement datetime to minutes from reference time
            frac_time = [datetime_to_fractional_minutes_from_reference(round_to_minute(dt)) for dt in start_dt]

            #------------------------------------------------------------------------#
            #PARSE METADATA
            #------------------------------------------------------------------------#

            #GHOST version
            site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

            #initialise string which will store all GHOST process warnings as empty            
            site_metadata['process_warnings']['value'].append('')
    
            #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
            site_metadata['meta_update_stamp']['value'].append(sorted(frac_time)[0])

            #write known metadata  
            site_metadata['network']['value'].append(self.run_details['network'])
            site_metadata['data_level']['value'].append('official')
            site_metadata['data_licence']['value'].append('Public domain where not explicitly stated otherwise: https://www.eea.europa.eu/legal/eea-data-policy') 

            #add reported measurement unit
            #convert raw reported units given to standard name
            reported_units = standard_reporting_units[self.run_details['parameter']]
            conv_obj = unit_converter.convert_units(reported_units, reported_units, 1, measured_species = self.parameter_details['chemical_formula'])
            site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

            #define list of variable names that require no processing that can be simply added to site_metadata
            metadata_to_add = ['latitude','longitude','altitude','sampling_height','distance_to_kerb','distance_to_building',
                              'distance_to_junction','distance_to_source','street_type','street_width','daytime_traffic_speed',
                              'daily_passing_vehicles','station_name','representative_radius','principal_investigator_institution',
                              'principal_investigator_email_address','contact_institution','contact_email_address']

            #iterate through all metadata variable names and put values into standard dictionary
            for variable_name in metadata_to_add:
                #check variable is in unified metadata dict
                if variable_name in list(cut_unified_metadata.keys()):
                    site_metadata[variable_name]['value'].append(cut_unified_metadata[variable_name].replace('"',''))

            #convert spanish style number format for longitude,latitude,altitude to english style (if necessary)
            site_metadata['longitude']['value'][-1] = site_metadata['longitude']['value'][-1].replace(',','.')
            site_metadata['latitude']['value'][-1] = site_metadata['latitude']['value'][-1].replace(',','.')
            if len(site_metadata['altitude']['value']) > 0:
                site_metadata['altitude']['value'][-1] = site_metadata['altitude']['value'][-1].replace(',','.')

            #parse associated networks 
            associated_network_string = ''
            #add EEA AQ e-Reporting with eeacode (if not empty string)
            if eeacode != '':
                associated_network_string = 'EEA AQ e-Reporting:{}'.format(eeacode)
            #next add AIRBASE with EoICode (if not empty string)
            if eoicode != '':
                airbase_string = 'EEA AIRBASE:{}'.format(eoicode)
                if associated_network_string == '':   
                    associated_network_string = copy.deepcopy(airbase_string)
                else:
                    associated_network_string = '{};{}'.format(associated_network_string, airbase_string)
                #add EBAS, if have 'emep_station' variable and is 'yes'
                if 'emep_station' in list(cut_unified_metadata.keys()):
                    if cut_unified_metadata['emep_station'] == 'yes':
                        #add EBAS with EoICode
                        ebas_string = 'EBAS:{}'.format(eoicode)
                        associated_network_string = '{};{}'.format(associated_network_string, ebas_string)
            #finally add local network with local code
            if ('local_network_code' in list(cut_unified_metadata.keys())) & ('local_reference' in list(cut_unified_metadata.keys())):
                local_string = '{}:{}'.format(cut_unified_metadata['local_network_code'],cut_unified_metadata['local_reference'])
                if associated_network_string == '':   
                    associated_network_string = copy.deepcopy(local_string)
                else:
                    associated_network_string = '{};{}'.format(associated_network_string, local_string)
            #append associated_network_string if not empty
            if associated_network_string != '':
                site_metadata['associated_networks']['value'].append(associated_network_string)

            #parse horizontal geodetic datum - take from parsed spatial reference system information (certain spatial reference systems use specific datums - i.e. 'EPSG:4979' uses 'WSG84' datum)
            if 'spatial_reference_system' in list(cut_unified_metadata.keys()):
                if cut_unified_metadata['spatial_reference_system'] == 'EPSG:4979':
                   site_metadata['ellipsoid']['value'].append('WGS 84')  
                   site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984') 
                   site_metadata['vertical_datum']['value'].append('Ellipsoidal - WGS 84') 
                   site_metadata['projection']['value'].append('geographic')
                #if spatial reference system is unknown then write handled error logfile (2)
                else:
                    self.run_details['log_file_str'] += 'UNKNOWN SPATIAL REFERENCE SYSTEM:{}'.format(cut_unified_metadata['spatial_reference_system'])
                    create_GHOST_stage_logfile(2,self.run_details)
            #if no defined spatial reference system, assume datum
            else:
                site_metadata['ellipsoid']['value'].append('WGS 84')  
                site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984') 
                site_metadata['vertical_datum']['value'].append('Ellipsoidal - WGS 84') 
                site_metadata['projection']['value'].append('geographic')            

            #parse population (need to check to remove non-numeric characters)
            if 'population' in list(cut_unified_metadata.keys()):
                population = cut_unified_metadata['population']
                population = re.sub("[^0-9]", "", population)
                site_metadata['population']['value'].append(population) 
    
            #parse 'principal_investigator_name' (concatenate title, first name and last name, if given)
            principal_investigator_name = ''
            if 'principal_investigator_title' in list(cut_unified_metadata.keys()):
                principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_title']
            if 'principal_investigator_first_name' in list(cut_unified_metadata.keys()):
                if principal_investigator_name == '':
                    principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_first_name']
                else:
                    principal_investigator_name = '{} {}'.format(principal_investigator_name, cut_unified_metadata['principal_investigator_first_name'])
            if 'principal_investigator_last_name' in list(cut_unified_metadata.keys()):
                if principal_investigator_name == '':
                    principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_last_name']
                else:
                    principal_investigator_name = '{} {}'.format(principal_investigator_name, cut_unified_metadata['principal_investigator_last_name'])
            if principal_investigator_name != '':
                site_metadata['principal_investigator_name']['value'].append(principal_investigator_name) 
     
            #parse 'contact_name' (concatenate title, first name and last name, if given)
            contact_name = ''
            if 'contact_title' in list(cut_unified_metadata.keys()):
                contact_name = contact_name + cut_unified_metadata['contact_title']
            if 'contact_first_name' in list(cut_unified_metadata.keys()):
                if contact_name == '':
                    contact_name = contact_name + cut_unified_metadata['contact_first_name']
                else:
                    contact_name = '{} {}'.format(contact_name, cut_unified_metadata['contact_first_name'])
            if 'contact_last_name' in list(cut_unified_metadata.keys()):
                if contact_name == '':
                    contact_name = contact_name + cut_unified_metadata['contact_last_name']
                else:
                    contact_name = '{} {}'.format(contact_name, cut_unified_metadata['contact_last_name'])
            if contact_name != '':
                site_metadata['contact_name']['value'].append(contact_name) 
    
            #parse miscellaneous details
            #deal with miscellaneous sampling details
            if 'sample_location_miscellaneous' in list(cut_unified_metadata.keys()):
                miscellaneous_sampling = 'Sampling location: {}'.format(cut_unified_metadata['sample_location_miscellaneous'])
                site_metadata['network_sampling_details']['value'].append(miscellaneous_sampling) 
    
            #deal with miscellaneous sampling inlet information
            if 'length_sampling_line' in list(cut_unified_metadata.keys()):
                inlet_info = 'Length: {}'.format(cut_unified_metadata['length_sampling_line'])
                site_metadata['measuring_instrument_inlet_information']['value'].append(inlet_info)
                    
            #deal with miscellaneous maintenance details
            miscellaneous_maintenance = ''
            if 'calibration_method' in list(cut_unified_metadata.keys()):
                string_to_add = 'Calibration methodology: {}'.format(cut_unified_metadata['calibration_method'])
                miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
            if 'calibration_frequency' in list(cut_unified_metadata.keys()):
                string_to_add = 'Calibration frequency: {}'.format(cut_unified_metadata['calibration_frequency'])
                if miscellaneous_maintenance == '':
                    miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
                else:
                    miscellaneous_maintenance = '{};{}'.format(miscellaneous_maintenance, string_to_add) 
            if 'calibration_description' in list(cut_unified_metadata.keys()):
                string_to_add = 'Calibration description: {}'.format(cut_unified_metadata['calibration_description'])
                if miscellaneous_maintenance == '':
                    miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
                else:
                    miscellaneous_maintenance = '{};{}'.format(miscellaneous_maintenance, string_to_add)            
            if miscellaneous_maintenance != '':
                site_metadata['network_maintenance_details']['value'].append(miscellaneous_maintenance)
    
            #deal with other miscellaneous details
            station_description = ''
            if 'station_description' in list(cut_unified_metadata.keys()):
                station_description = 'Station description: {}'.format(cut_unified_metadata['station_description'])
            if 'station_address' in list(cut_unified_metadata.keys()):
                if station_description == '':
                    station_description = 'Station address: {}'.format(cut_unified_metadata['station_address'])
                else:   
                    station_description = '{};{}'.format(station_description, cut_unified_metadata['station_address'])
            if station_description != '':
                site_metadata['network_miscellaneous_details']['value'].append(station_description)

            #parse measurement information (from external metadata) - use all possible variables of: 'sampling_equipment', 'measurement_equipment', 'analytical_technique', 'measurement_method', 'measurement_type', 'measurement_automatic'
            if 'sampling_equipment' in list(cut_unified_metadata.keys()):
                sampling_equipment = cut_unified_metadata['sampling_equipment']
            else:
                sampling_equipment = ''
            if 'measurement_equipment' in list(cut_unified_metadata.keys()):
                #remove 'unnessary long string from variable
                measurement_equipment = cut_unified_metadata['measurement_equipment'].split('measurementequipment/')[-1]
            else:
                measurement_equipment = ''
            if 'analytical_technique' in list(cut_unified_metadata.keys()):
                analytical_technique = cut_unified_metadata['analytical_technique']
            else:
                analytical_technique = ''
            if 'measurement_method' in list(cut_unified_metadata.keys()):
                #remove 'unnessary long string from variable
                measurement_method = cut_unified_metadata['measurement_method'].split('measurementmethod/')[-1]
            else:
                measurement_method = ''
            if 'measurement_type' in list(cut_unified_metadata.keys()):
                measurement_type = cut_unified_metadata['measurement_type']
            else:
                measurement_type = ''
            #only parse 'measurement_automatic' if don't have 'measurement_type'
            if ('measurement_automatic' in list(cut_unified_metadata.keys())) & (measurement_type == ''):
                measurement_automatic = cut_unified_metadata['measurement_automatic']
                if measurement_automatic == 'yes':
                    measurement_type = 'automatic'
                elif measurement_automatic == 'no':
                    measurement_type = 'not-automatic'
            else:
                measurement_automatic = ''
            #concatenate measurement information together
            measurement_information = '{} {} {} {} {}'.format(sampling_equipment, measurement_equipment, analytical_technique, measurement_method, measurement_type)
            #strip leading/trailing white spaces, and change multiple white spaces to be singular
            measurement_information = measurement_information.strip()
            measurement_information = ' '.join(measurement_information.split())
            site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(measurement_information, site_metadata, self.run_details, self.parameter_details)
            #set measurement qa codes which are True as joined metadata str
            site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
            measurement_qa_flag_codes = []
            for standard_flag_name in measurement_qa_flags.keys():
                if measurement_qa_flags[standard_flag_name]:
                    measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
            site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))

            #parse station classifications (from external metadata) - use all possible variables of: 'area_classification', 'station_classification', 'main_emission_source', 'land_use', 'area_classification_rural_subdivision', 'representative_radius'
            if 'area_classification' in list(cut_unified_metadata.keys()):
                area_classification = cut_unified_metadata['area_classification']
            else:
                area_classification = ''
            if 'station_classification' in list(cut_unified_metadata.keys()):
                station_classification = cut_unified_metadata['station_classification']
            else:
                station_classification = ''
            if 'main_emission_source' in list(cut_unified_metadata.keys()):
                main_emission_source = cut_unified_metadata['main_emission_source']
            else:
                main_emission_source = ''
            if 'land_use' in list(cut_unified_metadata.keys()):
                land_use = cut_unified_metadata['land_use']
            else:
                land_use = ''
            if 'area_classification_rural_subdivision' in list(cut_unified_metadata.keys()):
                area_classification_rural_subdivision = cut_unified_metadata['area_classification_rural_subdivision']
            else:
                area_classification_rural_subdivision = ''   
            if 'representative_radius' in list(cut_unified_metadata.keys()):
                representative_radius = cut_unified_metadata['representative_radius']
            else:
                representative_radius = ''   
            native_classifications = {'area_classification':area_classification, 'station_classification':station_classification, 'main_emission_source':main_emission_source, 'land_use':land_use, 'area_classification_rural_subdivision':area_classification_rural_subdivision, 'representative_radius':representative_radius}
            site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)
    
            #iterate through metadata variables, if any columns have not been appended to add a NaN
            for column_header in list(site_metadata.keys()):
                n_column = len(site_metadata[column_header]['value'])
                #if column is empty, then append NaN
                if n_column < 1:
                    site_metadata[column_header]['value'].append(np.NaN)
            
            #set station_reference (provided network ID + standard measurement method abbreviation)
            site_metadata['station_reference']['value'][-1] = '{}_{}'.format(unique_station_reference,standard_method_abbreviation) 

            #get data file download timestamp
            site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

            #add data revision timestamp, set as 0001-01-01 00:00 for all files
            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(1,1,1,0,0))
            site_metadata['data_revision_stamp']['value'][-1] = data_revision_timestamp

            #------------------------------------------------------------------------#
            #PARSE DATA
            #------------------------------------------------------------------------#

            #measurement values may need converting (i.e. mass density to mixing ratio)
    
            #measurement temperature and pressure values are never provided so are always assumed to be the EEA AQ eReporting standard pressure and temperatures (as report data to EEA)
            #these are defined as for 293 K temperature, and 1013 hPa for pressure here for all gas-phase compounds: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32008L0050&from=en and here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32004L0107&from=EN 
            #for particulate matter and substances to be analysed in particulate matter (e.g. lead) the sampling volume refers to ambient conditions in terms of temperature and atmospheric pressure at the date of measurements.
            #if for some reason particulate measurements are given as mixing ratio and need converting to mass density, then the conversion is done using the same standard temperature/pressure as reported for gases.
            
            measurements = []
            qa = [[] for i in range(len(frac_time))]

            standard_network_gas_volume = False
            unknown_gas_volume = False
            if self.parameter_details['matrix'] == 'gas':
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '293.0'
                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.0'
                standard_network_gas_volume = True
            else:
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN
                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN
                unknown_gas_volume = True
            
            #get standard reported units/quantity
            standard_reported_units = conv_obj.output_standard_units
            measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity  

            #set flags for writing QA flags associated with assumptions in unit conversion
            unit_conversion_standard_network = False
            unit_conversion_educated_guess = False

            #check if reported standard units, match required standard units
            if standard_reported_units != site_data[self.parameter_details['bsc_parameter_name']]['units']:

                #mass density to mixing ratio?
                if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                    #get conversion factor
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards for gases)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
        
                #mixing ratio to mass density?
                elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':standard_reported_units}     
                    #do conversion with network provided sample gas temperature/pressure values
                    #if these are not set, then make educated guess (assume network standards)
                    if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                        input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_standard_network = True
                    else:
                        input_vals = {'temperature':'293.0', 'pressure':'1013.0', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                        unit_conversion_educated_guess = True
                    conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
       
                #otherwise simple unit conversion, i.e. pptv --> ppbv, ng/m3 --> ug/m3
                else:
                    conv_obj = unit_converter.convert_units(standard_reported_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])

                #process through each measurement and convert value using derived conversion factor
                for n, meas in enumerate(cut_measurements):
                    #only do conversion if measurement value is above 0 and not infinite
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    if (float_meas > 0.0) & (np.isfinite(float_meas)):
                        measurements.append(float_meas*conv_obj.conversion_factor)
                    else:
                        measurements.append(float_meas)

                    #append sample gas volume/unit_conversion assumption QA flags if necessary
                    if standard_network_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
                    if unknown_gas_volume:
                        qa[n].append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])
                    if unit_conversion_standard_network:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                    if unit_conversion_educated_guess:
                        qa[n].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

            #otherwise do nothing to measured values
            else:
                for n, meas in enumerate(cut_measurements):
                    try:
                        float_meas = float(meas)
                    except:
                        float_meas = np.NaN
                    measurements.append(float_meas)

            #keep only measurement windows that are >= 0 minute in temporal resolution, as well as all other associated variables across other fields
            valid_temporal_resolution_inds = [ii for ii, res in enumerate(temporal_resolutions) if res >= 0]
            if len(valid_temporal_resolution_inds) != len(frac_time):
                frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
                temporal_resolutions = [temporal_resolutions[x] for x in valid_temporal_resolution_inds]
                qa = [qa[x] for x in valid_temporal_resolution_inds]
                measurements = [measurements[x] for x in valid_temporal_resolution_inds]

            #----------------------------------------------------------------------------#
            #WRITE OUT DATA
            #----------------------------------------------------------------------------# 
            site_data['time']['value'] = frac_time
            site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
            site_data['temporal_resolution']['value'] = temporal_resolutions   
            #set data flags as all un-flagged
            site_data['flag']['value'] = [[standard_data_flag_name_to_data_flag_code['Un-Flagged Data']] for i in range(len(site_data['time']['value']))]
            site_data['qa']['value'] = qa
            #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
            for column_header in list(site_data.keys()):
                n_column = len(site_data[column_header]['value'])
                #if column is empty, then append NaN
                if n_column == 0:
                    if site_data[column_header]['data_type'] != np.uint8:
                        site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                    else:
                        site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
            #------------------------------------------------------

            #standardise metadata/data formats and write out station data to numpy file
            write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)
            if write_obj.written == False:
                self.run_details['log_file_str'] += 'DATA ALL MISSING FOR STATION: {}. SKIPPING.\n'.format(unique_station_reference)