import copy
import datetime
import time

import numpy as np
import pandas as pd

from crossnetwork_processing_functions import get_data_files, create_GHOST_stage_logfile
from GHOST_standards import get_standard_data, get_standard_metadata, country_ISO3166_to_name, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code, standard_measurement_methodologies, standard_networks
from network_file_standardise import NetworkFileStandardise
from network_parsers import ParseNASAAmesFile
from standardisation_mapping_functions import map_network_classifications_to_standard_classifications, map_network_measurement_information_to_standard_measurement_information, network_data_flag_to_standard_data_flag_mapping
from time_functions import round_to_minute, datetime_to_fractional_minutes_from_reference, get_datafile_download_timestamp
import unit_converter


class StandardiseNetworkData:

    '''Convert one data provider original file for a specific station into a GHOST standard numpy station specific file'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):
        
        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- PARSING RAW DATA\n'.format(self.station_reference)
        
    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #gather data files of station reference to parse
        data_files = self.gather_data_files()

        #parse station reference data files
        for data_file in data_files:
            self.parse_file(data_file)

        #if stage has completed successfully, with some valid data, write a valid process log file
        if self.written_data:
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)
        #else, if no data has been written for station write an error log file to inform station is not continuing in the pipeline
        elif not self.written_data:
            self.run_details['log_file_str'] += 'NO DATA HAS BEEN WRITTEN FOR STATION'
            create_GHOST_stage_logfile(1,self.run_details)
    
    def gather_data_files(self):

        '''Function to gather data files to parse per station'''

        #get data files specific to station
        data_files = get_data_files(self.run_details, self.parameter_details, specific_station=self.station_reference)

        #sort files in date order
        data_file_start_dates = [int(data_file.split('/')[-1].split('.')[1]) for data_file in data_files]
        data_files = np.array([x for _,x in sorted(zip(data_file_start_dates,data_files))]) 

        #get standard measured unit quantity
        if self.parameter_details['standard_units'] != 'unitless':
            conv_obj = unit_converter.convert_units(self.parameter_details['standard_units'],self.parameter_details['standard_units'],1)
            self.measured_unit_quantity = conv_obj.output_represented_quantity

        #set flag to know when have written data for station    
        self.written_data = False
   
        return data_files


    def parse_file(self, data_file):

        '''Function to parse data file to GHOST standard numpy station specific file'''

        self.run_details['log_file_str'] += 'PROCESSING FILE: {}\n'.format(data_file) 
        
        #create site instances of metadata/data dictionaries
        site_data =     get_standard_data(self.parameter_details)
        site_metadata = get_standard_metadata(self.parameter_details)
 
        #Get data level - this is given in both in filename and inside file as metadata field. 
        #If exists in metadata, write this out preferentially. If not there, write from filename
        #if data level is not given anywhere, assume level 2
        data_level = ''
        if '.lev' in data_file:
            data_level_string = data_file.split('.lev')[-1].split('.nas')[0]
            data_level_string = data_level_string.split('_')[0]
            if data_level_string != '':
                data_level = str(data_level_string).strip()              

        # get EBAS parameter name from data filename
        ebas_parameter_name = data_file.split('/')[-1].split('.')[4]

        #determine if variable is corrected or not
        corrected_variable = False
        if ('corrected' in ebas_parameter_name) & ('sulphate' not in ebas_parameter_name):
            corrected_variable = True

        #----------------------------------------------------------------------------#
        #READ FILE
        #----------------------------------------------------------------------------#

        site_nasa_ames_obj = ParseNASAAmesFile(data_file,self.station_reference,self.run_details)
        site_nasa_ames_obj.read_header()
        site_nasa_ames_obj.read_data()
    
        #get file reference start date
        start_date_reference = site_nasa_ames_obj.date_start

        #get start/end time arrays
        s_times = site_nasa_ames_obj.structured_data[0]
        e_times = site_nasa_ames_obj.structured_data[1]
    
        #if length of s_times == 0, then do not process file, as zero valid data
        if len(s_times) == 0:
            self.run_details['log_file_str'] += 'NO DATA IN FILE: {}. SKIPPING.\n'.format(data_file)
            return

        #create datetime from YYYYMMDD start date file reference 
        dt_start = datetime.datetime(int(start_date_reference[:4]),int(start_date_reference[4:6]),int(start_date_reference[6:]))
        
        #iterate through file start/end times (in units of days from start date reference), and calculate the time in integer minutes from the desired start year
        #create start/end datetimes for each time given by adding to datetime date file reference, round to nearest minute to avoid issues due to floating point estimation
        start_dt = [round_to_minute(dt_start+datetime.timedelta(days=float(s_time))) for s_time in s_times]
        end_dt = [round_to_minute(dt_start+datetime.timedelta(days=float(e_time))) for e_time in e_times]

        #convert each start measurement datetime to minutes from reference time 
        #do not need to shift time base as already in UTC         
        frac_time = [datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

        #get timedeltas between start and end of measurement window
        #get list of valid temporal resolution indices, where temporal resolution of measurement window is >= 1 minute
        temporal_resolutions = []
        valid_temporal_resolution_inds = []
        for temp_ii, (start, end) in enumerate(zip(start_dt,end_dt)):
            dt_window = end - start
            #convert total seconds to total minutes
            dt_window_minutes = int(dt_window.total_seconds()/60.)
            #only append measurement window resolution if >= 0 minute in length
            if dt_window_minutes >= 0:
                temporal_resolutions.append(dt_window_minutes)    
                valid_temporal_resolution_inds.append(temp_ii)

        #cut start time data array to be just those times for measurement windows, where temporal resolution is >= 0 minute
        if len(valid_temporal_resolution_inds) != len(frac_time):
            frac_time = [frac_time[x] for x in valid_temporal_resolution_inds]
        
        #set variable flags for parsing header of data file
        var_ind = -999
        flag = -999
        multiple_stat_types = False
    
        #get desired measured parameter data
        #check how many inds there are matched for relevant parameter
        relevant_var_inds = np.where(np.array(site_nasa_ames_obj.primary_names) == ebas_parameter_name)[0]
        #typically there will be more than 1 set of units for only gas phase species (mixing ratio and mass density) 
        if len(relevant_var_inds) > 1:
            #if more than one valid ind for wanted parameter, cut for preferential units 
            #each parameter has defined preferential units as a list (first element is preferred), iterate list until have units desired 
            priority_units_list = self.parameter_details['ebas_priority_units']
            for priority_units in priority_units_list:
                unit_inds = [idx for idx,s in enumerate(site_nasa_ames_obj.primary_units) if priority_units in s]
                #if have got desired units in file, break out of loop
                if len(unit_inds) > 0:
                    break
            #if have not found any desired units, write error log file to inform of this horrible development (caught error)
            if len(unit_inds) == 0:
                self.run_details['log_file_str'] += 'CANNOT FIND ANY DESIRED UNITS IN FILE'
                create_GHOST_stage_logfile(2,self.run_details)            

            #next check overlap of desired var and units
            intersect_inds = np.array([val for val in relevant_var_inds if val in unit_inds])
            #if more than 1 valid intersect ind, must be due to reporting different metrics (mean, min, max etc.) 
            #cut to just get arithmetic mean
            if len(intersect_inds) > 1:
            
                #have to check for duplicate column data, as there some instances where there are repeats (e.g. - CH0002R, 2007, for NO)
                #check if variable names, units and extra details are same for all intersects
                #if so, just take first intersect ind preferentially
            
                if (len(np.unique(np.array(site_nasa_ames_obj.primary_names)[intersect_inds])) == 1) & (len(np.unique(np.array(site_nasa_ames_obj.primary_units)[intersect_inds])) == 1) & (len(np.unique(np.array(site_nasa_ames_obj.primary_details)[intersect_inds])) < 2):
                    var_ind =  intersect_inds[0]
                
                #otherwise cut for arithmetic mean
                else:
                    #set flag informing that have multiple statistic types
                    multiple_stat_types = True
                    for int_i in intersect_inds:
                        for detail in site_nasa_ames_obj.primary_details[int_i]:
                            if 'arithmetic mean' in detail:
                                var_ind = int_i
                                break       
                    #if var_ind is still -999 then there is no arithmetic mean stat, so skip file
                    if var_ind == -999:
                        self.run_details['log_file_str'] += 'NO ARITHMETIC MEAN STATISTIC IN FILE: {}. SKIPPING.\n'.format(data_file)
                        return       
            else:
                var_ind = intersect_inds[0]
        else:
            var_ind = relevant_var_inds[0]

        # for some carbon measurements, the measurements are given as a specific fraction or as the artifacts
        # do not process files for these measurements
        if len(site_nasa_ames_obj.primary_details[var_ind]) > 0:
            for line in site_nasa_ames_obj.primary_details[var_ind]:
                line_lower = line.lower().replace(" ", "")
                if ('artifact=' in line_lower) or ('fraction=' in line_lower):
                    self.run_details['log_file_str'] += 'PARAMETER IS ARTIFACT/FRACTION, IN FILE: {}. SKIPPING.\n'.format(data_file)
                    return 
                    
        #if aerosol coefficient or AOD parameter, then check for specific wavelength currently processing
        if ('_COEFF_' in self.run_details['parameter']) or (self.parameter_details['matrix'] == 'aod'):
            parameter_wavelength = float(self.run_details['parameter'].split('_')[-1].split('nm')[0])
            file_wavelength = 'unknown'
            for line in site_nasa_ames_obj.primary_details[var_ind]:
                line_lower = line.lower().replace(" ", "")
                if 'wavelength=' in line_lower:
                    file_wavelength = float(line_lower.split('wavelength=')[-1].split('nm')[0])
                    break 

            # if file wavelength is not same as that of parameter, then do not process file
            if (parameter_wavelength != file_wavelength) & (file_wavelength != 'unknown'):
                self.run_details['log_file_str'] += 'PARAMETER WAVELENGTH ({}) NOT SAME AS FILE WAVELENGTH ({}), IN FILE: {}. SKIPPING.\n'.format(parameter_wavelength,file_wavelength,data_file)
                return

        #get parameter measurements
        file_measurements = np.array(site_nasa_ames_obj.structured_data[var_ind+1])
        
        #get data flags
        #check how many inds there are for flags
        relevant_flag_inds = [idx for idx,s in enumerate(site_nasa_ames_obj.primary_names) if 'numflag' in s]
    
        #if more than 1 flag in file then must be because there multiple different metrics reported (mean, min, max etc.), each with associated flag
        #in this case simply add 1 to determined var indice, as the flag is always reported immediately after 
        if len(relevant_flag_inds) > 1:
            flag_ind = var_ind+1 
        else:
            flag_ind = relevant_flag_inds[0]
    
        #add 1 to allow for starttime field (not one of primary names)
        raw_file_flags = np.array(site_nasa_ames_obj.structured_data[flag_ind+1])
        
        #----------------------------------------------------------------------------#
        #PROCESS METADATA
        #----------------------------------------------------------------------------#
        
        #GHOST version
        site_metadata['GHOST_version']['value'].append(self.run_details['ghost_version'])

        #initialise string which will store all GHOST process warnings as empty            
        site_metadata['process_warnings']['value'].append('')

        #save meta_update_stamp, set as first sorted time processed (as minutes relative to reference time)
        site_metadata['meta_update_stamp']['value'].append(sorted(frac_time)[0])
   
        #create dictionary that metadata that can be appended to in both details and comments sections 
        #no parameter specific metadata overlaps between details and comments, however this is useful when compling metadata from different fields into 1 standardised meta field.
        self.concat_meta = {}
   
        #dictionary for storing raw metadata units with quantity name
        self.units_dict = {}
   
        #process metadata in details
        self.process_metadata(site_nasa_ames_obj.primary_details[var_ind],'details')

        #process metadata in comments
        self.process_metadata(site_nasa_ames_obj.comments,'comments')

        #merge certain saved metadata fields into 1 field
        #delete merged fields from concat_meta afterwards
        
        #associated_networks
        variables_to_merge = ['wdca_id','gaw_id','airs_id','other_ids']
        for var in variables_to_merge:
            #if variable not in concat_meta, make it empty string
            if var not in list(self.concat_meta.keys()):
                self.concat_meta[var] = ''
        #make merged variable
        self.concat_meta['associated_networks'] = '{}:{};{};{};{};{}'.format(site_nasa_ames_obj.campaign_name.strip(),self.station_reference,self.concat_meta['wdca_id'],self.concat_meta['gaw_id'],self.concat_meta['airs_id'],self.concat_meta['other_ids'])
        #delete merged fields
        for var in variables_to_merge:
            del self.concat_meta[var]
        
        #measurement information
        variables_to_merge = ['instrument_type','instrument_manufacturer','instrument_model','analytical_measurement_technique','analytical_instrument_manufacturer','analytical_instrument_model']
        for var in variables_to_merge:
            #if variable not in concat_meta, make it empty string
            if var not in list(self.concat_meta.keys()):
                self.concat_meta[var] = ''
        #make merged variable
        self.concat_meta['measurement_information'] = '{},{},{},{},{},{}'.format(self.concat_meta['instrument_type'], self.concat_meta['instrument_manufacturer'], self.concat_meta['instrument_model'], self.concat_meta['analytical_measurement_technique'], self.concat_meta['analytical_instrument_manufacturer'], self.concat_meta['analytical_instrument_model'])
        #delete merged fields
        for var in variables_to_merge:
            del self.concat_meta[var]
        
        #inlet information
        variables_to_merge = ['inlet_type','inlet_description','inlet_tube_material','inlet_outer_diameter','inlet_inner_diameter','inlet_length']
        for var in variables_to_merge:
            #if variable not in concat_meta, make it empty string
            if var not in list(self.concat_meta.keys()):
                self.concat_meta[var] = ''
        #make merged variable
        self.concat_meta['measuring_instrument_inlet_information'] = '{};{};{};{};{};{}'.format(self.concat_meta['inlet_type'], self.concat_meta['inlet_description'], self.concat_meta['inlet_tube_material'], self.concat_meta['inlet_outer_diameter'], self.concat_meta['inlet_inner_diameter'], self.concat_meta['inlet_length'])
        #change " to inches in merged string if there - causes formatting issues further down pipe
        self.concat_meta['measuring_instrument_inlet_information'] = self.concat_meta['measuring_instrument_inlet_information'].replace('"','inch')
        #delete merged fields
        for var in variables_to_merge:
            del self.concat_meta[var]
        
        #sampling details
        variables_to_merge = ['filter_face_velocity','exposed_filter_area','filter_description','medium','filter_prefiring','filter_conditioning','coating','sample_preparation']
        for var in variables_to_merge:
            #if variable not in concat_meta, make it empty string
            if var not in list(self.concat_meta.keys()):
                self.concat_meta[var] = ''
        #make merged variable
        self.concat_meta['network_sampling_details'] = '{};{};{};{};{};{};{};{}'.format(self.concat_meta['filter_face_velocity'],self.concat_meta['exposed_filter_area'],self.concat_meta['filter_description'],self.concat_meta['medium'],self.concat_meta['filter_prefiring'],self.concat_meta['filter_conditioning'],self.concat_meta['coating'],self.concat_meta['sample_preparation'])
        #delete merged fields
        for var in variables_to_merge:
            del self.concat_meta[var]

        #uncertainty details
        variables_to_merge = ['measurement_uncertainty_explained','detection_limit_explained','zero_values']
        for var in variables_to_merge:
            #if variable not in concat_meta, make it empty string
            if var not in list(self.concat_meta.keys()):
                self.concat_meta[var] = ''
        #make merged variable
        self.concat_meta['network_uncertainty_details'] = '{};{};{}'.format(self.concat_meta['measurement_uncertainty_explained'],self.concat_meta['detection_limit_explained'],self.concat_meta['zero_values'])
        #delete merged fields
        for var in variables_to_merge:
            del self.concat_meta[var]
        
        #maintenance details
        variables_to_merge = ['maintenance_description','zero_type','zero_interval','blank_correction','artifact_correction_description','charring_correction','humidity_control','humidity_description']
        for var in variables_to_merge:
            #if variable not in concat_meta, make it empty string
            if var not in list(self.concat_meta.keys()):
                self.concat_meta[var] = ''
        #make merged variable
        self.concat_meta['network_maintenance_details'] = '{};{};{};{};{};{};{};{}'.format(self.concat_meta['maintenance_description'],self.concat_meta['zero_type'],self.concat_meta['zero_interval'],self.concat_meta['blank_correction'],self.concat_meta['artifact_correction_description'],self.concat_meta['charring_correction'],self.concat_meta['humidity_control'],self.concat_meta['humidity_description'])
        #delete merged fields
        for var in variables_to_merge:
            del self.concat_meta[var]
        
        #Network QA details
        variables_to_merge = ['qa_date','qa_outcome','qa_bias','qa_variability','qa_document_url']  
        for var in variables_to_merge:
            #if variable not in concat_meta, make it empty string
            if var not in list(self.concat_meta.keys()):
                self.concat_meta[var] = ''
        #make merged variable
        self.concat_meta['network_qa_details'] = '{};{};{};{};{}'.format(self.concat_meta['qa_date'],self.concat_meta['qa_outcome'],self.concat_meta['qa_bias'],self.concat_meta['qa_variability'],self.concat_meta['qa_document_url'])
        #delete merged fields
        for var in variables_to_merge:
            del self.concat_meta[var]

        #iterate through concatenated metadata and remove formatting inconsistencies, and deal with empty strings
        for key in list(self.concat_meta.keys()):
            #strip white space from ends of string
            self.concat_meta[key] = self.concat_meta[key].strip()
            #split string by commas, removing parts which are empty, and rejoining string
            comma_split = self.concat_meta[key].split(',')
            self.concat_meta[key] = ','.join([part.strip() for part in comma_split if part.strip() != ''])
            #split string by semi-colons, removing parts which are empty, and rejoining string
            semicolon_split = self.concat_meta[key].split(';')
            self.concat_meta[key] = ';'.join([part.strip() for part in semicolon_split if part.strip() != ''])
            #replace double space with single spaces
            self.concat_meta[key] = self.concat_meta[key].replace('  ',' ')
            #if first or last character is semicolon, colon or comma, cut it off
            if len(self.concat_meta[key]) > 1:
                if (self.concat_meta[key][0] == ';') or (self.concat_meta[key][0] == ':') or (self.concat_meta[key][0] == ','):
                    self.concat_meta[key] = self.concat_meta[key][1:]
                if (self.concat_meta[key][-1] == ';') or (self.concat_meta[key][-1] == ':') or (self.concat_meta[key][-1] == ','):
                    self.concat_meta[key] = self.concat_meta[key][:-1]
            #strip white space from ends of string
            self.concat_meta[key] = self.concat_meta[key].strip()
            #convert empty string, or solo commas/semicolons/colons to NaN
            if (self.concat_meta[key] == '') or (self.concat_meta[key] == ';') or (self.concat_meta[key] == ',') or (self.concat_meta[key] == ':'):
                self.concat_meta[key] = np.NaN

        #if do not have multiple statistic types provided then check if stat type is 'arithmetic mean'
        if multiple_stat_types == False:
            #if statistic type is not 'arithmetic mean' or have no information about statistic then skip file
            if 'statistic_type' not in self.concat_meta:
                self.run_details['log_file_str'] += 'NO ARITHMETIC MEAN STATISTIC IN FILE: {}. SKIPPING.\n'.format(data_file)
                return  
            if self.concat_meta['statistic_type'] != 'arithmetic mean':
                self.run_details['log_file_str'] += 'NO ARITHMETIC MEAN STATISTIC IN FILE: {}. SKIPPING.\n'.format(data_file)
                return
        if 'statistic_type' in self.concat_meta:
            del self.concat_meta['statistic_type']

        #process data revision timestamp (original format YYYYMMDDHHMMSS)
        if 'revision_date' in self.concat_meta:
            data_revision_timestamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(int(self.concat_meta['revision_date'][:4]),int(self.concat_meta['revision_date'][4:6]),int(self.concat_meta['revision_date'][6:8]),int(self.concat_meta['revision_date'][8:10]),int(self.concat_meta['revision_date'][10:12]),int(self.concat_meta['revision_date'][12:14])))
            site_metadata['data_revision_stamp']['value'].append(data_revision_timestamp)
            del self.concat_meta['revision_date']
            
        #process station classifications
        #if both file area_classification and land use are present in file, then process standardised classifications
        if ('area_classification' in self.concat_meta) & ('land_use' in self.concat_meta):
            #get standardised classifications
            native_classifications = {'area_classification':self.concat_meta['area_classification'],'land_use':self.concat_meta['land_use']}
            site_metadata = map_network_classifications_to_standard_classifications(native_classifications, site_metadata, self.run_details)
        #delete classification parameters from concat_meta
        if 'area_classification' in self.concat_meta:
            del self.concat_meta['area_classification']
        if 'land_use' in self.concat_meta:
            del self.concat_meta['land_use']

        #process measurement information
        #do a little of extra work to get get thermal optical analysis protocol if available
        if self.run_details['parameter'] in ['C','EC','OC','PM10_C','PM10_EC','PM10_OC','PM2.5_C','PM2.5_EC','PM2.5_OC','PM1_C','PM1_EC','PM1_OC']:
            if 'IMPROVE' in data_file.upper():
                self.concat_meta['measurement_information'] = '{}_IMPROVE'.format(self.concat_meta['measurement_information'])
            elif 'NIOSH' in data_file.upper():
                self.concat_meta['measurement_information'] = '{}_NIOSH'.format(self.concat_meta['measurement_information'])
            elif 'EUSAAR' in data_file.upper():
                self.concat_meta['measurement_information'] = '{}_EUSAAR'.format(self.concat_meta['measurement_information'])

        site_metadata, measurement_qa_flags, standard_method_abbreviation = map_network_measurement_information_to_standard_measurement_information(self.concat_meta['measurement_information'], site_metadata, self.run_details, self.parameter_details)
        #set measurement qa codes which are True as joined metadata str
        site_metadata['measurement_qa_flags'] = {'value':[], 'data_type':object, 'string_format':'lower_short'}
        measurement_qa_flag_codes = []
        for standard_flag_name in measurement_qa_flags.keys():
            if measurement_qa_flags[standard_flag_name]:
                measurement_qa_flag_codes.append(str(standard_QA_name_to_QA_code[standard_flag_name]))
        site_metadata['measurement_qa_flags']['value'].append(','.join(measurement_qa_flag_codes))
        #delete parameter from concat_meta
        del self.concat_meta['measurement_information']

        #write rest of saved metadata out
        for column_header in list(self.concat_meta.keys()):
            n_column = len(site_metadata[column_header]['value'])
            #if length of column is less than number of references, then hasn't been written to yet, therefore append
            if n_column < 1:
                site_metadata[column_header]['value'].append(self.concat_meta[column_header])
            #otherwise it has been written as NaN already, therefore overwrite
            else:
                site_metadata[column_header]['value'][-1] = self.concat_meta[column_header]

        #write known network specific metadata  
        site_metadata['ellipsoid']['value'].append('WGS 84') 
        site_metadata['horizontal_datum']['value'].append('World Geodetic System 1984')
        site_metadata['vertical_datum']['value'].append('tidal - mean sea level')
        site_metadata['projection']['value'].append('geographic')
        #site_metadata['country']['value'].append(country_ISO3166_to_name[self.station_reference[:2]]) 
        site_metadata['network']['value'].append('EBAS')
        #add data licence from raw providing network
        site_metadata['data_licence']['value'].append('')
                    
        #iterate through metadata variables, if any columns have not been appended to add a NaN
        for column_header in list(site_metadata.keys()):
            n_column = len(site_metadata[column_header]['value'])
            #if column is empty, then append NaN
            if n_column == 0:
                site_metadata[column_header]['value'].append(np.NaN)

        #if do not have data level from metadata, try writing out data level from filename
        #if no level in filename either, assume it to be level 2 (highest level)  
        #do not continue if data level is not accepted        
        if pd.isnull(site_metadata['data_level']['value'][-1]):
            if data_level == '':
                site_metadata['data_level']['value'][-1] = '2.0'
            elif data_level in ['3a','3b','3c','3d']:
                self.run_details['log_file_str'] += 'DATA LEVEL ({}) IS NOT ACCEPTED IN FILE: {}. SKIPPING.\n'.format(data_level, data_file)
                return
            elif str(float(data_level)) in list(standard_networks['EBAS']['data_level_hierarchy'].keys()):
                site_metadata['data_level']['value'][-1] = str(float(data_level))            
            else:        
                self.run_details['log_file_str'] += 'DATA LEVEL ({}) IS UNKNOWN IN FILE: {}. SKIPPING.\n'.format(data_level, data_file)
                create_GHOST_stage_logfile(2,self.run_details)
        else:
            if site_metadata['data_level']['value'][-1] in ['3a','3b','3c','3d']:
                self.run_details['log_file_str'] += 'DATA LEVEL ({}) IS NOT ACCEPTED IN FILE: {}. SKIPPING.\n'.format(site_metadata['data_level']['value'][-1], data_file)
                return
            elif str(float(site_metadata['data_level']['value'][-1])) in list(standard_networks['EBAS']['data_level_hierarchy'].keys()):
                site_metadata['data_level']['value'][-1] = str(float(site_metadata['data_level']['value'][-1]))  
            else:
                self.run_details['log_file_str'] += 'DATA LEVEL ({}) IS UNKNOWN IN FILE: {}. SKIPPING.\n'.format(str(float(site_metadata['data_level']['value'][-1])), data_file)
                create_GHOST_stage_logfile(2,self.run_details)

        #set station_reference (provided network ID + standard measurement method abbreviation)
        site_metadata['station_reference']['value'][-1] = '{}_{}'.format(self.station_reference,standard_method_abbreviation)

        #get data file download timestamp
        site_metadata['data_download_stamp']['value'][-1] = get_datafile_download_timestamp(data_file)

        #-----------------------------------------------------
        #if processing a parameter with a specific wavelength, and could not determine it from the file then try determine it from measurement information
        if ('_COEFF_' in self.run_details['parameter']) or (self.parameter_details['matrix'] == 'aod'):
            if file_wavelength == 'unknown':
                # if have instrument information, try to obtain specific wavelength from it
                if not pd.isnull(site_metadata['measuring_instrument_name']['value'][-1]):
                    method = site_metadata['measurement_methodology']['value'][-1]
                    instrument = site_metadata['measuring_instrument_name']['value'][-1]
                    method_dict = standard_measurement_methodologies[method]
                    instrument_dict = method_dict['instruments'][instrument]
                    if 'measured_parameters' in instrument_dict:
                        measured_parameters = instrument_dict['measured_parameters']
                    else:
                        measured_parameters = method_dict['measured_parameters']
                    # get number of parameters with wavelength
                    measured_parameters_wavelength = np.unique([measured_parameter.split('_')[-1] for measured_parameter in measured_parameters if measured_parameter.split('_')[-1][-2:] == 'nm'])
                    # if have only 1 remaining wavelength, then now know wavelength
                    # if not, then cannot isolate wavelength, so skip file
                    if len(measured_parameters_wavelength) > 1:
                        self.run_details['log_file_str'] += 'CANNOT DETERMINE WAVELENGTH FOR PARAMETER PROCESSING FROM FILE, & WAVELENGTH IS NOT SPECIFIC FOR INSTRUMENT: {}. SKIPPING.\n'.format(data_file)
                        return
                    
                #if have no specific instrument, do not have enough information to obtain wavelength information, so skip file
                else:
                    self.run_details['log_file_str'] += 'CANNOT DETERMINE WAVELENGTH FOR PARAMETER PROCESSING FROM FILE, & NO INSTRUMENT INFORMATION: {}. SKIPPING.\n'.format(data_file)
                    return

        #-----------------------------------------------------
        #convert metadata data with units to standard units
        
        #get measurement units
        units = site_nasa_ames_obj.primary_units[var_ind].strip()

        #convert aerosol coefficients to parsable units
        if units == '1/Mm':
            units = 'Mm-1'

        #if measurements are unitless, do not do any conversion
        if units == 'no unit':
            site_metadata['measuring_instrument_reported_units']['value'][-1] = 'unitless'
        else:            
            #convert raw units given to standard name (if not unitless)
            try:
                conv_obj = unit_converter.convert_units(units,units,1, measured_species = self.parameter_details['chemical_formula'])
            #if cannot convert units to standard units, then then a chemical formula is needed for conversion of units but do not have it: ug C/m3 for NMHC
            #therefore do not process file
            except SystemExit:
                self.run_details['log_file_str'] += 'NO VALID UNITS IN FILE: {}. SKIPPING.\n'.format(data_file)
                return
            site_metadata['measuring_instrument_reported_units']['value'][-1] = conv_obj.output_standard_units
            measuring_instrument_reported_unit_quantity = conv_obj.output_represented_quantity        

        #process reported sampled gas temperature and pressure first 
        #use EBAS stated standards if temperature/pressure are not stated explicitly with measurements, UNLESS these are stated to be 'ambient' or 'internal instrument' in which case they are set as NaN
        #standards are 293.15 K / 1013.25 hPa for O3, & 273.15 K / 1013.25 hPa for all other parameters
        #see here: https://ebas-submit.nilu.no/Submit-Data/Data-Reporting/Comments/Generic-metadata-comments/Volume-std-temperature
        
        #set unit conversion QA flags as default False (only one can be True --> unknown takes priority)
        standard_network_gas_volume = False
        unknown_gas_volume = False
        
        #reported sampled gas temperature is 'unknown'? set as 'ambient' or 'internal instrument' by EBAS
        #overwrite as NaN
        if site_metadata['network_provided_volume_standard_temperature']['value'][-1] == 'unknown':
            unknown_gas_volume = True
            site_metadata['network_provided_volume_standard_temperature']['value'][-1] = np.NaN  
        #have reported sampled gas temperature? convert units if necessary
        elif not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1]):
            #if metadata units not equal to standard units, convert!
            if self.units_dict['network_provided_volume_standard_temperature'] != site_metadata['network_provided_volume_standard_temperature']['units']:
                conv_obj = unit_converter.convert_units(self.units_dict['network_provided_volume_standard_temperature'],site_metadata['network_provided_volume_standard_temperature']['units'],site_metadata['network_provided_volume_standard_temperature']['value'][-1], measured_species = self.parameter_details['chemical_formula'])
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = conv_obj.converted_value
            #remove from units dict 
            del self.units_dict['network_provided_volume_standard_temperature']
        #else if do not have reported sampled gas temperature, AND not stated as 'unknown', then set as EBAS stated standard for component
        else:
            standard_network_gas_volume = True
            if self.run_details['parameter'] == 'O3':
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '293.15'
            else:
                site_metadata['network_provided_volume_standard_temperature']['value'][-1] = '273.15'
        
        #reported sampled gas pressure is 'unknown'? set as 'ambient' or 'internal instrument' by EBAS
        #overwrite as NaN
        if site_metadata['network_provided_volume_standard_pressure']['value'][-1] == 'unknown':
            unknown_gas_volume = True
            standard_network_gas_volume = False 
            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = np.NaN 
        #have reported sampled gas pressure? convert units if necessary
        elif not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1]):
            #if metadata units not equal to standard units, convert!
            if self.units_dict['network_provided_volume_standard_pressure'] != site_metadata['network_provided_volume_standard_pressure']['units']:
                conv_obj = unit_converter.convert_units(self.units_dict['network_provided_volume_standard_pressure'],site_metadata['network_provided_volume_standard_pressure']['units'],site_metadata['network_provided_volume_standard_pressure']['value'][-1], measured_species = self.parameter_details['chemical_formula'])
                site_metadata['network_provided_volume_standard_pressure']['value'][-1] = conv_obj.converted_value
             #remove from units dict 
            del self.units_dict['network_provided_volume_standard_pressure']
        #else if do not have reported sampled gas pressure, AND not stated as 'unknown', then set as EBAS stated standard for component
        else:
            if not unknown_gas_volume:
                standard_network_gas_volume = True
            site_metadata['network_provided_volume_standard_pressure']['value'][-1] = '1013.25'

        #process rest of meta units
        for meta_name in self.units_dict:

            #convert aerosol coefficients to parsable units
            if self.units_dict[meta_name] == '1/Mm':
                self.units_dict[meta_name] = 'Mm-1'

            #if measurements are unitless, do not do any conversion
            if self.units_dict[meta_name] not in ['no unit','unitless','']:

                #if metadata units not equal to standard units, convert!
                if self.units_dict[meta_name] != site_metadata[meta_name]['units']:
                    
                    #get standard reported units/quantity
                    conv_obj = unit_converter.convert_units(self.units_dict[meta_name],self.units_dict[meta_name],1, measured_species = self.parameter_details['chemical_formula'])
                    standard_reported_units = conv_obj.output_standard_units
                    reported_unit_quantity = conv_obj.output_represented_quantity

                    #mass density to mixing ratio?
                    if (meta_name in ['measuring_instrument_reported_lower_limit_of_detection','measuring_instrument_reported_uncertainty']) & (reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':self.units_dict[meta_name]}
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':site_metadata[meta_name]['value'][-1]}
                        else:
                            if self.run_details['parameter'] == 'O3':
                                input_vals = {'temperature':'293.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':site_metadata[meta_name]['value'][-1]}
                            else:
                                input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':site_metadata[meta_name]['value'][-1]}
                        conv_obj = unit_converter.convert_units(input_units,site_metadata[meta_name]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
                    
                    #mixing ratio to mass density?
                    elif (meta_name in ['measuring_instrument_reported_lower_limit_of_detection','measuring_instrument_reported_uncertainty']) & (reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':self.units_dict[meta_name]}
                        #do conversion with network provided sample gas temperature/pressure values
                        #if these are not set, then make educated guess (assume network standards)
                        if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):
                            input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':site_metadata[meta_name]['value'][-1]}
                        else:
                            if self.run_details['parameter'] == 'O3':
                                input_vals = {'temperature':'293.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':site_metadata[meta_name]['value'][-1]}
                            else:
                                input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':site_metadata[meta_name]['value'][-1]}
                        conv_obj = unit_converter.convert_units(input_units,site_metadata[meta_name]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
                    #quantities which just need converting i.e. cm2/s --> m2/min
                    else:
                        conv_obj = unit_converter.convert_units(self.units_dict[meta_name],site_metadata[meta_name]['units'],site_metadata[meta_name]['value'][-1], measured_species = self.parameter_details['chemical_formula'])
                        
                    #get converted value
                    site_metadata[meta_name]['value'][-1] = conv_obj.converted_value
        
        #----------------------------------------------------------------------------#
        #PROCESS DATA
        #----------------------------------------------------------------------------#

        #flags/qa lists to write out
        flags = []
        qa = []
    
        #get dictionary necessary for mapping of EBAS file data flags to standard data flag codes
        data_flag_mapping_dictionary = network_data_flag_to_standard_data_flag_mapping['EBAS']

        #iterate through EBAS file data flags and map to standard flag codes
        for raw_file_flag in raw_file_flags:

            #lists to append qa/data flags per measurement
            flags_per_measurement = []
            qa_per_measurement = []
            
            #convert file flag into integer number, equivalent to format as in standard flag mapping dictionary 
            if '.' in raw_file_flag:
                raw_file_flag = raw_file_flag.split('.')[-1]
            #split raw file flag after every 3rd element into list of flags (as can be multiple flags per measurement)
            file_flags = [raw_file_flag[i:i+3] for i in range(0, len(raw_file_flag), 3)]            
            #get unique file flags
            file_flags = np.unique(file_flags)

            #set GHOST decreed validity/network decreed validity variables initially as 'V'
            ghost_decreed_validity = 'V'
            network_decreed_validity = 'V'

            #iterate through all file flags per measurement
            for file_flag in file_flags:
                # map data flag/s and append
                standard_data_flag_dict = data_flag_mapping_dictionary[file_flag]
                #get mapped standard data flags
                standard_data_flag_names = standard_data_flag_dict['standard_data_flag_name']
                #iterate through standard data flags 
                for standard_data_flag_ii, standard_data_flag_name in enumerate(standard_data_flag_names):
                    #check if standard data flag is empty (i.e. unmapped as non-useful)
                    if standard_data_flag_name != '': 
                        #get corresponding data flag code for standard data flag
                        flag_code = standard_data_flag_name_to_data_flag_code[standard_data_flag_name]
                        #append standard data flag (if not already appended)
                        if flag_code not in flags_per_measurement:
                            flags_per_measurement.append(flag_code)
                        #determine if GHOST/network decreed valid
                        if standard_data_flag_dict['GHOST_decreed_validity'][standard_data_flag_ii] == 'I':
                            ghost_decreed_validity = 'I'
                        if standard_data_flag_dict['network_decreed_validity'][standard_data_flag_ii] == 'I':
                            network_decreed_validity = 'I'

            #if ghost decreed validity, or network decreed validity is Invalid, then set QA flags
            if ghost_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed'])
            if network_decreed_validity == 'I':
                qa_per_measurement.append(standard_QA_name_to_QA_code['Invalid Data Provider Flags - Network Decreed'])

            #write preliminary data flag if data level == 0.0
            #described by EBAS as:
            # -- Annotated raw data
            # -- Format instrument specific
            # -- Contains all parameters provided by instrument as provided
            # -- Contains all parameters / info needed for processing to final value.
            # -- "Native" time resolution
            if site_metadata['data_level']['value'][-1] == '0.0':
                if standard_data_flag_name_to_data_flag_code['Preliminary Data'] not in flags_per_measurement:
                    flags_per_measurement.append(standard_data_flag_name_to_data_flag_code['Preliminary Data'])

            #write corrected parameter qa flag if necessary
            if corrected_variable:
                qa_per_measurement.append(standard_QA_name_to_QA_code['Corrected Parameter'])
        
            #if have assumed sampled gas volume from network standard, set QA flag statting this
            if standard_network_gas_volume:
                qa_per_measurement.append(standard_QA_name_to_QA_code['Sample Gas Volume - Network Standard'])
            #if sampled gas volume is unknown, set QA flag stating this
            if unknown_gas_volume: 
                qa_per_measurement.append(standard_QA_name_to_QA_code['Sample Gas Volume - Unknown'])

            #append qa_per_measurement, flags_per_measurement to lists to write out
            flags.append(flags_per_measurement)
            qa.append(qa_per_measurement)
            
        #--------------------------------------
        #measurements
        #measurement values may need converting (i.e. mass density to mixing ratio)
        #--------------------------------------
        
        measurements = []

        #check if reported standard units, match required standard units
        if site_metadata['measuring_instrument_reported_units']['value'][-1] != site_data[self.parameter_details['bsc_parameter_name']]['units']:
            
            #mass density to mixing ratio?
            if (measuring_instrument_reported_unit_quantity == 'mass_density') & (self.measured_unit_quantity == 'mole_fraction'):
                #get conversion factor
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':site_metadata['measuring_instrument_reported_units']['value'][-1]}
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume network standards)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])):   
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                else:
                    if self.run_details['parameter'] == 'O3':
                        input_vals = {'temperature':'293.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                    else:
                        input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mass_density':1.0}
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
                #if have assumed sampled gas volume from network standard, set QA flag that are assuming network standard sampled gas volume when converting units 
                if standard_network_gas_volume:
                    for qa_ii, qa_codes in enumerate(qa):
                        qa[qa_ii].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                #if sampled gas volume is unknown, set QA flag that are making educated guess at sampled gas volume when converting units
                if unknown_gas_volume: 
                    for qa_ii, qa_codes in enumerate(qa):
                        qa[qa_ii].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])

            #mixing ratio to mass density?
            elif (measuring_instrument_reported_unit_quantity == 'mole_fraction') & (self.measured_unit_quantity == 'mass_density'):
                #get conversion factor
                input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mole_fraction':site_metadata['measuring_instrument_reported_units']['value'][-1]}     
                #do conversion with network provided sample gas temperature/pressure values
                #if these are not set, then make educated guess (assume network standards)
                if (not pd.isnull(site_metadata['network_provided_volume_standard_temperature']['value'][-1])) & (not pd.isnull(site_metadata['network_provided_volume_standard_pressure']['value'][-1])): 
                    input_vals = {'temperature':site_metadata['network_provided_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['network_provided_volume_standard_pressure']['value'][-1], 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                else:
                    if self.run_details['parameter'] == 'O3':
                        input_vals = {'temperature':'293.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                    else:
                        input_vals = {'temperature':'273.15', 'pressure':'1013.25', 'molar_mass':unit_converter.get_molecular_mass(self.parameter_details['chemical_formula']), 'mole_fraction':1.0}
                conv_obj = unit_converter.convert_units(input_units,site_data[self.parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = self.parameter_details['chemical_formula'],conversion_input_quantity = 'mole_fraction')
                #if have assumed sampled gas volume from network standard, set QA flag that am assuming network standard sampled gas volume when converting units 
                if standard_network_gas_volume:
                    for qa_ii, qa_codes in enumerate(qa):
                        qa[qa_ii].append(standard_QA_name_to_QA_code['Unit Conversion - Network Standard Sample Gas Volume Assumption'])
                #if sampled gas volume is unknown, set QA flag that are making educated guess at sampled gas volume when converting units
                if unknown_gas_volume: 
                    for qa_ii, qa_codes in enumerate(qa):
                        qa[qa_ii].append(standard_QA_name_to_QA_code['Unit Conversion - Educated Guess Sample Gas Volume Assumption'])
 
            #otherwise simple unit conversion, i.e. pptv to ppbv or mass density to mass density
            else:
                conv_obj = unit_converter.convert_units(site_metadata['measuring_instrument_reported_units']['value'][-1],site_data[self.parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = self.parameter_details['chemical_formula'])  
            
            #process through each measurement and convert value using derived conversion factor
            for n, meas in enumerate(file_measurements):
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                #only do conversion if measurement value is above 0, is not NaN/infinite, and do not have data flag stating the measurement is missing 
                if (float_meas > 0.0) & (np.isfinite(float_meas)) & (standard_data_flag_name_to_data_flag_code['Missing Data'] not in flags[n]):
                    measurements.append(float_meas*conv_obj.conversion_factor)
                else:
                    measurements.append(float_meas)
            
        #otherwise do nothing to measured values
        else:
            for n, meas in enumerate(file_measurements):
                try:
                    float_meas = float(meas)
                except:
                    float_meas = np.NaN
                measurements.append(float_meas)
        
        #------------------------------------------------------
        #cut data variables to only keep those associated with measurement windows of >= 0 minute (time and temporal resolution fields already cut)
        if len(valid_temporal_resolution_inds) != len(measurements):
            flags = [flags[x] for x in valid_temporal_resolution_inds]
            qa = [qa[x] for x in valid_temporal_resolution_inds]        
            measurements = [measurements[x] for x in valid_temporal_resolution_inds]

        #----------------------------------------------------------------------------#
        #WRITE OUT DATA
        #----------------------------------------------------------------------------#
        site_data['time']['value'] = frac_time
        site_data[self.parameter_details['bsc_parameter_name']]['value'] = measurements
        site_data['temporal_resolution']['value'] = temporal_resolutions   
        site_data['flag']['value'] = flags
        site_data['qa']['value'] = qa
        #iterate through data variables, if any columns have not been appended to, add a list of NaNs (equal to length of time array)
        for column_header in list(site_data.keys()):
            n_column = len(site_data[column_header]['value'])
            #if column is empty, then append NaN
            if n_column == 0:
                if site_data[column_header]['data_type'] != np.uint8:
                    site_data[column_header]['value'] = [np.NaN]*len(site_data['time']['value'])
                else:
                    site_data[column_header]['value'] = [255]*len(site_data['time']['value'])
        #------------------------------------------------------
            
        #standardise metadata/data formats and write out station data to numpy file
        write_obj = NetworkFileStandardise(site_metadata, site_data, self.run_details, self.parameter_details)

        #update flag to state that have written data for station (if have done so)
        if write_obj.written:
            self.written_data = True
        else:
            self.run_details['log_file_str'] += 'DATA ALL MISSING IN FILE: {}. SKIPPING.\n'.format(data_file)

    def process_metadata(self, meta, meta_type):

        '''Function that takes metadata from header details/comments and sorts it.
           Many parameters can appear in both header and comments, it handles these cases. 
        '''    

        line_titles = []
        line_data = []
    
        #get splitter for different section of datafile metadata is reported in
        splitter_dict = {'comments':':','details':'='}
        splitter = splitter_dict[meta_type]
    
        for line in meta: 
            line_split = line.split(splitter)
            if len(line_split) >= 2:
                line_titles.append(line_split[0].strip())
                line_data.append(''.join([element.strip() for element in line_split[1:]]))
    
        #data_level
        if 'Data level' in line_titles:
            data_level_string = line_data[line_titles.index('Data level')]
            if (data_level_string != '') & ('data_level' not in self.concat_meta):
                self.concat_meta['data_level'] = data_level_string

        #statistic_type
        if 'Statistics' in line_titles:
            statistic = line_data[line_titles.index('Statistics')]
            if 'statistic_type' not in self.concat_meta:
                self.concat_meta['statistic_type'] = statistic

        #revision_date     
        if 'Revision date' in line_titles:
            revision_date = line_data[line_titles.index('Revision date')]
            if (revision_date != '') & ('revision_date' not in self.concat_meta):
                self.concat_meta['revision_date'] = revision_date
    
        #data_timezone
        if 'Timezone' in line_titles:
            data_timezone = line_data[line_titles.index('Timezone')]
            if (data_timezone != '') & ('timezone' not in self.concat_meta):
                #if timezone data is reported in is not UTC, write error log file as it should be (caught error)
                if data_timezone == 'UTC':
                    pass
                else:
                    self.run_details['log_file_str'] += 'DATA NOT IN UTC TIME'
                    create_GHOST_stage_logfile(2,self.run_details) 

        #latitude
        if 'Station latitude' in line_titles:
            latitude = line_data[line_titles.index('Station latitude')]
            if (latitude != '') & ('latitude' not in self.concat_meta):
                self.concat_meta['latitude'] = latitude

        #longitude
        if 'Station longitude' in line_titles:
            longitude = line_data[line_titles.index('Station longitude')]
            if (longitude != '') & ('longitude' not in self.concat_meta):
                self.concat_meta['longitude'] = longitude

        #altitude 
        if 'Station altitude' in line_titles:
            altitude = line_data[line_titles.index('Station altitude')]
            if (altitude != '') & ('altitude' not in self.concat_meta):
                if 'altitude' not in list(self.concat_meta.keys()):
                    self.concat_meta['altitude'] = altitude.split(' ')[0]
                    self.units_dict['altitude'] = ''.join(altitude.split(' ')[1:])

        #sampling_height 
        if 'Measurement height' in line_titles:
            sampling_height = line_data[line_titles.index('Measurement height')]
            if (sampling_height != '') & ('sampling_height' not in self.concat_meta):
                self.concat_meta['sampling_height'] = sampling_height.split(' ')[0]
                self.units_dict['sampling_height'] = ''.join(sampling_height.split(' ')[1:])

        #measurement_altitude --> this is actually the altitude from sea level at GROUND level at point of measurement, so more accurate than altitude field
        #                         therefore overwrite the altitude field when it is available
        if 'Measurement altitude' in line_titles:
            measurement_altitude = line_data[line_titles.index('Measurement altitude')]
            if (measurement_altitude != '') & ('measurement_altitude' not in self.concat_meta):
                self.concat_meta['altitude'] = measurement_altitude.split(' ')[0]
                self.units_dict['altitude'] = ''.join(measurement_altitude.split(' ')[1:])

        #station_name
        if 'Station name' in line_titles:
            station_name = line_data[line_titles.index('Station name')]
            if (station_name != '') & ('station_name' not in self.concat_meta):
                self.concat_meta['station_name'] = station_name

        #padministrative_country_division_1
        #if 'Station state/province' in line_titles:
        #    province = line_data[line_titles.index('Station state/province')]
        #    if (province != '') & ('administrative_country_division_1' not in self.concat_meta):
        #        self.concat_meta['administrative_country_division_1'] = province

        #flow_rate
        if ('Flow rate' in line_titles):
            reported_flow_rate = line_data[line_titles.index('Flow rate')]
            if (reported_flow_rate != '') & ('reported_flow_rate' not in self.concat_meta):
                self.concat_meta['measuring_instrument_reported_flow_rate'] = reported_flow_rate.split(' ')[0]
                self.units_dict['measuring_instrument_reported_flow_rate'] = ''.join(reported_flow_rate.split(' ')[1:])

        #reported_lower_limit_of_detection 
        if ('Detection limit' in line_titles):
            reported_lower_limit_of_detection = line_data[line_titles.index('Detection limit')]
            if (reported_lower_limit_of_detection != '') & ('measuring_instrument_reported_lower_limit_of_detection' not in self.concat_meta):
                self.concat_meta['measuring_instrument_reported_lower_limit_of_detection'] = reported_lower_limit_of_detection.split(' ')[0]
                self.units_dict['measuring_instrument_reported_lower_limit_of_detection'] = ''.join(reported_lower_limit_of_detection.split(' ')[1:])

        #reported_uncertainty (given either as a percent or absolute value) 
        if ('Measurement uncertainty' in line_titles):
            reported_uncertainty = line_data[line_titles.index('Measurement uncertainty')]
            if (reported_uncertainty != '') & ('measuring_instrument_reported_uncertainty' not in self.concat_meta):
                self.concat_meta['measuring_instrument_reported_uncertainty'] = reported_uncertainty.split(' ')[0]
                uncertainty_unit = ''.join(reported_uncertainty.split(' ')[1:])
                if uncertainty_unit != '%':
                    self.units_dict['measuring_instrument_reported_uncertainty'] = uncertainty_unit
    
        #absorption_cross_section 
        if ('Absorption cross section' in line_titles):
            reported_absorption_cross_section = line_data[line_titles.index('Absorption cross section')]
            if (reported_absorption_cross_section != '') & ('measuring_instrument_reported_absorption_cross_section' not in self.concat_meta):
                #convert known strings to cross section value
                if 'Hearn' and '1961' in reported_absorption_cross_section:
                    self.concat_meta['measuring_instrument_reported_absorption_cross_section'] = '1.147e-17'
                    self.units_dict['measuring_instrument_reported_absorption_cross_section'] = 'cm2'
                elif 'CCQM' and '2019' in reported_absorption_cross_section:
                    self.concat_meta['measuring_instrument_reported_absorption_cross_section'] = '1.1329e-17'
                    self.units_dict['measuring_instrument_reported_absorption_cross_section'] = 'cm2'
                else:
                    self.concat_meta['measuring_instrument_reported_absorption_cross_section'] = reported_absorption_cross_section.split(' ')[0]
                    self.units_dict['measuring_instrument_reported_absorption_cross_section'] = ''.join(reported_absorption_cross_section.split(' ')[1:]) 

        #calibration_scale
        if ('Calibration scale' in line_titles):
            calibration_scale = line_data[line_titles.index('Calibration scale')]
            if (calibration_scale != '') & ('measuring_instrument_calibration_scale' not in self.concat_meta):
                self.concat_meta['measuring_instrument_calibration_scale'] = calibration_scale
        
        #volume_standard_temperature 
        if ('Volume std. temperature' in line_titles):
            volume_standard_temperature = line_data[line_titles.index('Volume std. temperature')]
            #volume standard temperature set as ambient/instrument internal, write as 'unknown'
            if (volume_standard_temperature != 'ambient') or (volume_standard_temperature != 'instrument internal'):      
                self.concat_meta['network_provided_volume_standard_temperature'] = 'unknown'
            #have standard temperature number?
            elif (volume_standard_temperature != '') & ('network_provided_volume_standard_temperature' not in self.concat_meta):
                #must have units given for temperature, otherwise it is useless. If no units are given, do not write anything
                try:
                    self.concat_meta['network_provided_volume_standard_temperature'] = volume_standard_temperature.split(' ')[0]
                    self.units_dict['network_provided_volume_standard_temperature'] = ''.join(volume_standard_temperature.split(' ')[1:]) 
                except:
                    pass
    
        #volume_standard_pressure 
        if ('Volume std. pressure' in line_titles):
            volume_standard_pressure = line_data[line_titles.index('Volume std. pressure')]
            #volume standard pressure set as ambient/instrument internal, write as 'unknown'
            if (volume_standard_pressure != 'ambient') or (volume_standard_pressure != 'instrument internal'):
                self.concat_meta['network_provided_volume_standard_pressure'] = 'unknown'
            #have standard pressure number?
            elif (volume_standard_pressure != '') & ('network_provided_volume_standard_pressure' not in self.concat_meta):
                #must have units given for pressure, otherwise it is useless. If no units are given, do not write anything
                try:
                    self.concat_meta['network_provided_volume_standard_pressure'] = volume_standard_pressure.split(' ')[0]
                    self.units_dict['network_provided_volume_standard_pressure'] = ''.join(volume_standard_pressure.split(' ')[1:]) 
                except:
                    pass
    
        #associated_networks
        if 'Station WDCA-ID' in line_titles:
            wdca_id = line_data[line_titles.index('Station WDCA-ID')].strip()
            if (wdca_id != '') & ('wdca_id' not in self.concat_meta):
                self.concat_meta['wdca_id'] = 'WDCA:'+wdca_id
        if 'Station GAW-ID' in line_titles:
            gaw_id = line_data[line_titles.index('Station GAW-ID')].strip()
            if (gaw_id != '') & ('gaw_id' not in self.concat_meta):
                self.concat_meta['gaw_id'] = 'GAW:'+gaw_id
        if 'Station AIRS-ID' in line_titles:
            airs_id = line_data[line_titles.index('Station AIRS-ID')].strip()
            if (airs_id != '') & ('airs_id' not in self.concat_meta):
                self.concat_meta['airs_id'] = 'AIRS:'+airs_id
        if 'Station other IDs' in line_titles:
            other_ids = line_data[line_titles.index('Station other IDs')].strip()
            if (other_ids != '') & ('other_ids' not in self.concat_meta):
                self.concat_meta['other_ids'] = ''
                other_ids = other_ids.split(',')
                for o_id in other_ids:
                    o_id_split = o_id.split('(')
                    self.concat_meta['other_ids'] = self.concat_meta['other_ids'] + '{}:{};'.format(o_id_split[1][:-1].strip(),o_id_split[0].strip())
                #remove end semi-colon
                self.concat_meta['other_ids'] = self.concat_meta['other_ids'][:-1] 

        #area_classification/land_use
        if 'Station setting' in line_titles:
            area_classification = line_data[line_titles.index('Station setting')].replace(" ", "").lower()
            if (area_classification != '') & ('area_classification' not in self.concat_meta):
                self.concat_meta['area_classification'] = area_classification
        if 'Station land use' in line_titles:  
            land_use = line_data[line_titles.index('Station land use')].replace(" ", "").lower()
            if (land_use != '') & ('land_use' not in self.concat_meta):
                self.concat_meta['land_use'] = land_use

        #measurement information
        #instrument_type, instrument_manufacturer, instrument_model, analytical_measurement_technique, analytical_instrument_manufacturer, analytical_instrument_model
        if 'Instrument type' in line_titles:
            instrument_type = line_data[line_titles.index('Instrument type')].lower()
            if (instrument_type != '') & ('instrument_type' not in self.concat_meta):
                self.concat_meta['instrument_type'] = instrument_type
        if 'Instrument manufacturer' in line_titles:
            instrument_manufacturer = line_data[line_titles.index('Instrument manufacturer')].lower()
            if (instrument_manufacturer != '') & ('instrument_manufacturer' not in self.concat_meta):
                self.concat_meta['instrument_manufacturer'] = instrument_manufacturer
        if 'Instrument model' in line_titles:
            instrument_model = line_data[line_titles.index('Instrument model')].lower()
            if (instrument_model != '') & ('instrument_model' not in self.concat_meta):
                self.concat_meta['instrument_model'] = instrument_model
        if 'Analytical measurement technique' in line_titles:
            analytical_measurement_technique = line_data[line_titles.index('Analytical measurement technique')].lower()
            if (analytical_measurement_technique != '') & ('analytical_measurement_technique' not in self.concat_meta):
                self.concat_meta['analytical_measurement_technique'] = analytical_measurement_technique
        if 'Analytical instrument manufacturer' in line_titles:
            analytical_instrument_manufacturer = line_data[line_titles.index('Analytical instrument manufacturer')].lower()
            if (analytical_instrument_manufacturer != '') & ('analytical_instrument_manufacturer' not in self.concat_meta):
                self.concat_meta['analytical_instrument_manufacturer'] = analytical_instrument_manufacturer
        if 'Analytical instrument model' in line_titles:
            analytical_instrument_model = line_data[line_titles.index('Analytical instrument model')].lower()
            if (analytical_instrument_model != '') & ('analytical_instrument_model' not in self.concat_meta):
                self.concat_meta['analytical_instrument_model'] = analytical_instrument_model

        #inlet_information  
        if ('Inlet type' in line_titles):
            inlet_type = line_data[line_titles.index('Inlet type')].lower()
            if (inlet_type != '') & ('inlet_type' not in self.concat_meta):
                self.concat_meta['inlet_type'] = 'Inlet type: '+inlet_type
        if ('Inlet description' in line_titles):
            inlet_description = line_data[line_titles.index('Inlet description')].lower()
            if (inlet_description != '') & ('inlet_description' not in self.concat_meta):
                self.concat_meta['inlet_description'] = 'Description: '+inlet_description
        if ('Inlet tube material' in line_titles):
            inlet_tube_material = line_data[line_titles.index('Inlet tube material')].lower().strip()
            if (inlet_tube_material != '') & ('inlet_tube_material' not in self.concat_meta):
                self.concat_meta['inlet_tube_material'] = 'Tube material: '+inlet_tube_material
        if ('Inlet tube outer diameter' in line_titles):
            inlet_outer_diameter = line_data[line_titles.index('Inlet tube outer diameter')].lower().strip()
            if (inlet_outer_diameter != '') & ('inlet_outer_diameter' not in self.concat_meta):
                self.concat_meta['inlet_outer_diameter'] = 'Outer diameter: '+inlet_outer_diameter
        if ('Inlet tube inner diameter' in line_titles):
            inlet_inner_diameter = line_data[line_titles.index('Inlet tube inner diameter')].lower().strip()
            if (inlet_inner_diameter != '') & ('inlet_inner_diameter' not in self.concat_meta):
                self.concat_meta['inlet_inner_diameter'] = 'Inner diameter: '+inlet_inner_diameter
        if ('Inlet tube length' in line_titles):
            inlet_length = line_data[line_titles.index('Inlet tube length')].lower().strip()
            if (inlet_length != '') & ('inlet_length' not in self.concat_meta):
                self.concat_meta['inlet_length'] = 'Length: '+inlet_length

        #principal_investigator information
        #can be multiple - so take first instance
        if 'Originator' in line_titles:
            principal_investigator_split = line_data[line_titles.index('Originator')].split(',')
        
            principal_investigator_name =  '{} {}'.format(principal_investigator_split[1].strip(),principal_investigator_split[0].strip())
            principal_investigator_name = principal_investigator_name.strip()
            if (principal_investigator_name != '') & ('principal_investigator_name' not in self.concat_meta):
                self.concat_meta['principal_investigator_name'] = principal_investigator_name
            
            principal_investigator_email_address = principal_investigator_split[2].strip()
            if (principal_investigator_email_address != '') & ('principal_investigator_email_address' not in self.concat_meta):
                self.concat_meta['principal_investigator_email_address'] = principal_investigator_email_address
            
            principal_investigator_institution = principal_investigator_split[3].strip()
            if (principal_investigator_institution != '') & ('principal_investigator_institution' not in self.concat_meta):
                self.concat_meta['principal_investigator_institution'] = principal_investigator_institution
        
       #contact information
       #can be multiple - so take first instance
        if 'Submitter' in line_titles:
            contact_split = line_data[line_titles.index('Submitter')].split(',')
        
            contact_name = '{} {}'.format(contact_split[1].strip(),contact_split[0].strip())
            contact_name = contact_name.strip()
            if (contact_name != '') & ('contact_name' not in self.concat_meta):
                self.concat_meta['contact_name'] = contact_name
        
            contact_email_address = contact_split[2].strip()
            if (contact_email_address != '') & ('contact_email_address' not in self.concat_meta):
                self.concat_meta['contact_email_address'] = contact_email_address
        
            contact_institution = contact_split[3].strip()
            if (contact_institution != '') & ('contact_institution' not in self.concat_meta):
                self.concat_meta['contact_institution'] = contact_institution
            
        #uncertainty details
        if ('Measurement uncertainty expl.' in line_titles):
            measurement_uncertainty_explained = line_data[line_titles.index('Measurement uncertainty expl.')].strip()   
            if (measurement_uncertainty_explained != '') & ('measurement_uncertainty_explained' not in self.concat_meta):
                self.concat_meta['measurement_uncertainty_explained'] = 'Measurement uncertainty explained: '+measurement_uncertainty_explained

        if ('Detection limit expl.' in line_titles):
            detection_limit_explained = line_data[line_titles.index('Detection limit expl.')].strip() 
            if (detection_limit_explained != '') & ('detection_limit_explained' not in self.concat_meta):
                self.concat_meta['detection_limit_explained'] = 'Detection limit explained: '+detection_limit_explained

        if ('Zero/negative values' in line_titles):
            zero_values = line_data[line_titles.index('Zero/negative values')].strip() 
            if (zero_values != '') & ('zero_values' not in self.concat_meta):
                self.concat_meta['zero_values'] = 'Zero/negative values: '+zero_values

        #measurement maintenance details
        if ('Maintenance description' in line_titles):
            maintenance_description = line_data[line_titles.index('Maintenance description')].strip() 
            if (maintenance_description != '') & ('maintenance_description' not in self.concat_meta):
                self.concat_meta['maintenance_description'] = 'Maintenance description: '+maintenance_description

        if ('Zero/span check type' in line_titles):
            zero_type = line_data[line_titles.index('Zero/span check type')].strip() 
            if (zero_type != '') & ('zero_type' not in self.concat_meta):
                self.concat_meta['zero_type'] = 'Zero/span check type: '+zero_type

        if ('Zero/span check interval' in line_titles):
            zero_interval = line_data[line_titles.index('Zero/span check interval')].strip() 
            if (zero_interval != '') & ('zero_interval' not in self.concat_meta):
                self.concat_meta['zero_interval'] = 'Zero/span check interval: '+zero_interval

        if ('Blank correction' in line_titles):
            blank_correction = line_data[line_titles.index('Blank correction')].strip()
            if (blank_correction != '') & ('blank_correction' not in self.concat_meta):
                self.concat_meta['blank_correction'] = 'Blank correction: '+blank_correction
    
        if ('Artifact correction description' in line_titles):
            artifact_correction_description = line_data[line_titles.index('Artifact correction description')].strip()
            if (artifact_correction_description != '') & ('artifact_correction_description' not in self.concat_meta):
                self.concat_meta['artifact_correction_description'] = 'Artifact correction description: '+artifact_correction_description
    
        if ('Charring correction' in line_titles):
            charring_correction = line_data[line_titles.index('Charring correction')].strip()
            if (charring_correction != '') & ('charring_correction' not in self.concat_meta):
                self.concat_meta['charring_correction'] = 'Charring correction: '+charring_correction
    
        if ('Humidity/temperature control' in line_titles):
            humidity_control = line_data[line_titles.index('Humidity/temperature control')].strip() 
            if (humidity_control != '') & ('humidity_control' not in self.concat_meta):
                self.concat_meta['humidity_control'] = 'Humidity/temp. control: '+humidity_control

        if ('Humidity/temperature control description' in line_titles):
            humidity_description = line_data[line_titles.index('Humidity/temperature control description')].strip() 
            if (humidity_description != '') & ('humidity_description' not in self.concat_meta):
                self.concat_meta['humidity_description'] = 'Humidity/temp. control description: '+humidity_description
    
        #sampling details 
        if ('Filter face velocity' in line_titles):
            filter_face_velocity = line_data[line_titles.index('Filter face velocity')].strip() 
            if (filter_face_velocity != '') & ('filter_face_velocity' not in self.concat_meta):
                self.concat_meta['filter_face_velocity'] = 'Filter face velocity: '+filter_face_velocity
    
        if ('Exposed filter area' in line_titles):
            exposed_filter_area = line_data[line_titles.index('Exposed filter area')].strip() 
            if (exposed_filter_area != '') & ('exposed_filter_area' not in self.concat_meta):
                self.concat_meta['exposed_filter_area'] = 'Exposed filter area: '+exposed_filter_area
    
        if ('Filter description' in line_titles):
            filter_description = line_data[line_titles.index('Filter description')].strip()
            if (filter_description != '') & ('filter_description' not in self.concat_meta):
                self.concat_meta['filter_description'] = 'Filter description: '+filter_description
    
        if ('Medium' in line_titles):
            medium = line_data[line_titles.index('Medium')].strip()
            if (medium != '') & ('medium' not in self.concat_meta):
                self.concat_meta['medium'] = 'Medium: '+medium
    
        if ('Filter prefiring' in line_titles):
            filter_prefiring = line_data[line_titles.index('Filter prefiring')].strip()
            if (filter_prefiring != '') & ('Filter prefiring' not in self.concat_meta):
                self.concat_meta['filter_prefiring'] = 'Filter prefiring: '+filter_prefiring
    
        if ('Filter conditioning' in line_titles):
            filter_conditioning = line_data[line_titles.index('Filter conditioning')].strip()
            if (filter_conditioning != '') & ('filter_conditioning' not in self.concat_meta):
                self.concat_meta['filter_conditioning'] = 'Filter conditioning: '+filter_conditioning
    
        if ('Coating/Solution' in line_titles):
            coating = line_data[line_titles.index('Coating/Solution')].strip()
            if (coating != '') & ('coating' not in self.concat_meta):
                self.concat_meta['coating'] = 'Coating/Solution: '+coating
    
        if ('Sample preparation' in line_titles):
            sample_preparation = line_data[line_titles.index('Sample preparation')].strip()
            if (sample_preparation != '') & ('sample_preparation' not in self.concat_meta):
                self.concat_meta['sample_preparation'] = 'Sample preparation: '+sample_preparation
    
        #network QA details
        if ('QA date' in line_titles):  
            qa_date = line_data[line_titles.index('QA date')].strip()
            if (qa_date != '') & ('qa_date' not in self.concat_meta):
                self.concat_meta['qa_date'] = 'QA date: '+qa_date
        
        if ('QA outcome' in line_titles):
            qa_outcome = line_data[line_titles.index('QA outcome')].strip()
            if (qa_outcome != '') & ('qa_outcome' not in self.concat_meta):
                self.concat_meta['qa_outcome'] = 'QA outcome: '+qa_outcome
    
        if ('QA bias' in line_titles):
            qa_bias = line_data[line_titles.index('QA bias')].strip()
            if (qa_bias != '') & ('qa_bias' not in self.concat_meta):
                self.concat_meta['qa_bias'] = 'QA bias: '+qa_bias
    
        if ('QA variability' in line_titles):
            qa_variability = line_data[line_titles.index('QA variability')].strip()
            if (qa_variability != '') & ('qa_variability' not in self.concat_meta):
                self.concat_meta['qa_variability'] = 'QA variability: '+qa_variability
    
        if ('QA document URL' in line_titles):
            qa_document_url = line_data[line_titles.index('QA document URL')].strip()
            if (qa_document_url != '') & ('qa_document_url' not in self.concat_meta):
                self.concat_meta['qa_document_url'] = 'QA document URL: '+qa_document_url    
    
        #miscellaneous details
        if ('Comment' in line_titles):
            comment = line_data[line_titles.index('Comment')].strip() 
            if (comment != '') & ('network_miscellaneous_details' not in self.concat_meta):
                self.concat_meta['network_miscellaneous_details'] = comment