#!/usr/bin/env python
# -*- coding: utf-8 -*-

#WRITTEN BY DENE BOWDALO

###------------------------------------------------------------------------------------###
###IMPORT MODULES
###------------------------------------------------------------------------------------###

from collections import Counter
import converter
import crossnetwork_processing_modules
import csv
import datetime
from dateutil.relativedelta import *
import instrument_processing_modules
import itertools
import numpy as np
import pandas as pd
import standardisation_modules

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###

def read_data(zipped_list):

    #unzip variables    
    station_n, ref, data_files, external_metadata = zipped_list[0],zipped_list[1],zipped_list[2],zipped_list[3]

    print '%s -- READING (%s)'%(ref, station_n)

    #create site instances of metadata/data dicts
    site_data =     standardisation_modules.create_standardised_data_frame()
    site_metadata = standardisation_modules.create_standardised_metadata_frame()
    
    #sort files in date order
    data_file_start_dates = [int(data_file.split('/')[-1].split('.')[1]) for data_file in data_files]
    data_files = np.array([x for _,x in sorted(zip(data_file_start_dates,data_files))]) 
    
    #standard_resolution_dict (in minutes)
    standard_resolution_dict = {60:'H',1440:'D',10080:'W'}
    
    #----------------------------------------------------------------------------------
    #check if there are multiple files whose timespans overlap
    #if so, find reason for clash, and determine files to keep (and subsequently prevent clash)
    matching_inds = []
    
    start_time_bases = []
    measure_methods = []
    matrixes = []
    ranges = []
    resolutions = []
    instru_names = []
    data_levels = []
    duplicate_files = []
    for data_file in data_files:
        data_split = data_file.split('/')[-1].split('.')
        start_time_bases=np.append(start_time_bases,data_split[1])
        measure_methods=np.append(measure_methods,data_split[3])
        matrixes=np.append(matrixes,data_split[5])
        ranges=np.append(ranges,data_split[6])
        resolutions=np.append(resolutions,data_split[7])
        instru_names=np.append(instru_names,data_split[8])
        if len(data_split) == 12:
            data_levels=np.append(data_levels,data_split[10].split('_')[0])
        else:
            data_levels=np.append(data_levels,data_split[10]+'.'+data_split[11].split('_')[0])
        if '_dup' in data_file:
            duplicate_files = np.append(duplicate_files,True)
        else:
            duplicate_files = np.append(duplicate_files,False)
            
    #if any of the matrixes are not correct for parameter processing, break processing!
    for x, matrix in enumerate(matrixes):
        if matrix != parameter_details['ebas_matrix']:
            print 'Breaking processing. Matrix for %s file is incorrect for current parameter processing. Should be %s.'%(data_files[x],parameter_details['ebas_matrix']) 
            1+'a'
    
    #iterate through each file
    #get start and end file times (in minutes from reference date)
    #find files whose times overlap 
    start_times = []
    end_times = []
    for t, start_time_base in enumerate(start_time_bases):
        start_dt = datetime.datetime(int(start_time_base[:4]),int(start_time_base[4:6]),int(start_time_base[6:8]),int(start_time_base[8:10]),int(start_time_base[10:12]))
        if 'y' in ranges[t]:
            end_dt = (start_dt + relativedelta(years=+int(ranges[t].replace('y','')))) - relativedelta(hours=+1) 
        elif 'mo' in ranges[t]:
            end_dt = (start_dt + relativedelta(months=+int(ranges[t].replace('mo','')))) - relativedelta(hours=+1) 
        elif 'w' in ranges[t]:
            end_dt = (start_dt + relativedelta(weeks=+int(ranges[t].replace('w','')))) - relativedelta(hours=+1)
        elif 'd' in ranges[t]:
            end_dt = (start_dt + relativedelta(days=+int(ranges[t].replace('d','')))) - relativedelta(hours=+1) 
        elif 'h' in ranges[t]:
            end_dt = (start_dt + relativedelta(hours=+int(ranges[t].replace('h','')))) - relativedelta(hours=+1)     
            
        #convert start_dt/ end_dt to minutes since reference time 
        start_times.append(crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(start_dt))
        end_times.append(crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(end_dt))
        
    #test time overlaps between each file pair combination
    #if there is time overlap, save both file inds out
    for i,start_time in enumerate(start_times):
        base_start_time = start_times[i]
        base_end_time = end_times[i]
        
        other_file_inds = range(0,i)+range(i+1,len(data_files))
        
        base_match_inds = [i]
        for o in other_file_inds:
            percent_inside_reference = 0
            
            start_time = start_times[o]
            end_time = end_times[o]
            test_time_diff = end_time - start_time
            
            #tested time window entirely contained within reference window (i.e. no new information) --> always return match 
            if (start_time >= base_start_time) & (end_time < base_end_time):
                base_match_inds.append(o) 
                continue
                
            #some overlap between tested time window and reference on right edge of reference window
            elif (start_time >= base_start_time) & (start_time < base_end_time):
                #how much of tested file time is inside reference time window (i.e. how much information is new or duplicated?)
                time_inside_reference = base_end_time - start_time
                percent_inside_reference = (time_inside_reference/test_time_diff)*100.
                
            #some overlap between tested time window and reference on left edge of reference window
            elif (end_time >= base_start_time) & (end_time < base_end_time):
                #how much of tested file time is inside reference time window (i.e. how much information is new or duplicated?)
                time_inside_reference = end_time - base_start_time
                percent_inside_reference = (time_inside_reference/test_time_diff)*100.
                
            #if percent of tested time window inside reference window is >= 50% class a match
            if (percent_inside_reference >= 50.0):
                base_match_inds.append(o) 

        if len(base_match_inds) > 1:
            matching_inds.append(base_match_inds)
            
    #if have overlapping files, try to determine why, and subsequently resolve duplication by excluding certain files
    #this exclusion is first done by if any files are flagged as duplicates, secondly by data level, thirdly by data resolution, fourthly by the length of the measurement range, next by prioritising a consistent measurement methodology through the time record, and lastly by prioritising a consistent instrument name through the file record 
    if len(matching_inds) > 0:
        
        #remove duplicate lists from matching_inds list of lists
        for i in range(len(matching_inds)):
            matching_inds[i] = sorted(matching_inds[i])
        matching_inds = sorted(matching_inds)
        matching_inds = list(matching_inds for matching_inds,_ in itertools.groupby(matching_inds))
        #remove lists which are just subsets of other lists
        d_match_inds = []
        for i in range(len(matching_inds)):
            for lol in matching_inds[:i]+matching_inds[i+1:]:
                if all(elem in lol for elem in matching_inds[i]):
                    d_match_inds=np.append(d_match_inds,i).astype(int)
        matching_inds = np.delete(matching_inds,d_match_inds,axis=0)
        
        #iterate for each matching file pair to determine why there is a duplicate
        #if determine reason for duplicate, prioritise particular file and write the other ind to remove_inds, for exclusion from processing
        
        total_remove_inds = []
        
        for x, match_inds in enumerate(matching_inds):
    
            #remove already remove indices from being erroneously matched later
            cut_match_inds = np.array([x for x in match_inds if x not in total_remove_inds]).astype(int)
            
            #if current cut_match_inds list has been reduced to a list of 1 or smaller, skip to next iteration
            if len(cut_match_inds) <= 1:
                continue
            
            #check if any files are reported as duplicates, if so remove them preferentially
            if len(np.unique(duplicate_files[cut_match_inds])) != 1:
                print 'Some files flagged as duplicates - fantastic EBAS.. Keeping files not flagged as duplicates'
                
                remove_inds = cut_match_inds[np.where(duplicate_files[cut_match_inds] == True)[0]]
                
                total_remove_inds = np.append(total_remove_inds, remove_inds).astype(int)
                #remove already remove indices from being erroneously matched later
                cut_match_inds = np.array([x for x in cut_match_inds if x not in total_remove_inds]).astype(int)
            
            
            #check if file have different data levels - remove lowest level data files
            if len(np.unique(data_levels[cut_match_inds])) != 1:
                print 'Files with different data levels. Keeping highest data levels'
                
                data_levels_cut = []
                for i,lev in enumerate(data_levels[cut_match_inds]):
                    try:
                        data_levels_cut=np.append(data_levels_cut,lev.split('lev')[1])
                    except:
                        data_levels_cut=np.append(data_levels_cut,2.0)
                    
                level_max = np.max(data_levels_cut.astype(np.float32))
                remove_inds = cut_match_inds[np.where(data_levels_cut != level_max)[0]]
            
                total_remove_inds = np.append(total_remove_inds, remove_inds).astype(int)
                #remove already remove indices from being erroneously matched later
                cut_match_inds = np.array([x for x in cut_match_inds if x not in total_remove_inds]).astype(int)
            
            #check if files have different stated resolution - remove lowest resolution files
            if len(np.unique(resolutions[cut_match_inds])) != 1:
                remove_inds = []
                print 'Files with different resolution. Keeping highest resolution files.'
                pref_resolutions = ['h','d','w','mo']
                #iterate through preferential file resolutions, keeping highest resolution files preferentially
                for p_res in pref_resolutions:
                    res_num = []
                    res_i = []
                    bad_inds = []
                    for j, res in enumerate(resolutions[cut_match_inds]):
                        #if resolution code is stated resolution, proceed
                        if p_res in res:
                            #get n hours/days/weeks/months
                            res_num.append(int(filter(str.isdigit, res)))
                            res_i.append(cut_match_inds[j])
                        else:
                            bad_inds.append(cut_match_inds[j])
                        
                    #if have matched resolutions, get min value (highest resolution)
                    if len(res_num) >  0:
                        highest_res = np.min(res_num)
                        #if any of the n hours/days etc. in array differs from miniumum value (highest resolution), then add to remove files list
                        for r,res_n in enumerate(res_num):
                            if res_n != highest_res:
                                remove_inds=np.append(remove_inds,res_i[r]).astype(int)
                        #add match_inds not matched for current resolution to remove inds (i.e. bad inds)
                        remove_inds=np.append(remove_inds,bad_inds).astype(int)
                        #then break out of for loop
                        break
            
                total_remove_inds = np.append(total_remove_inds, remove_inds).astype(int)
                #remove already remove indices from being erroneously matched later
                cut_match_inds = np.array([x for x in cut_match_inds if x not in total_remove_inds]).astype(int)
        
            #check if files have different measurement ranges - remove smallest ranges
            if len(np.unique(ranges[cut_match_inds])) != 1:
                remove_inds = []
                print 'Files with different measurement ranges. Keeping biggest ranges.'
                pref_ranges = ['y','mo','w','d','h']
                #iterate through preferential measurement ranges, keeping biggest ranges preferentially
                for p_range in pref_ranges:
                    range_num = []
                    range_i = []
                    bad_inds = []
                    for j, ran in enumerate(ranges[cut_match_inds]):
                        #if range code is stated range, proceed
                        if p_range in ran:
                            #get n hours/days/weeks/months
                            range_num.append(int(filter(str.isdigit, ran)))
                            range_i.append(cut_match_inds[j])
                        else:
                            bad_inds.append(cut_match_inds[j])
                        
                    #if have matched ranges, get max value (maximum range)
                    if len(range_num) > 0:
                        biggest_range = np.max(range_num)
                        #if any of the n hours/days etc. in array differs from maximum value (biggest range), then add to remove files list
                        for r,range_n in enumerate(range_num):
                            if range_n != biggest_range:
                                remove_inds=np.append(remove_inds,range_i[r]).astype(int)
                        #add match_inds not matched for current range to remove inds (i.e. bad inds)
                        remove_inds=np.append(remove_inds,bad_inds).astype(int)
                        #then break out of for loop
                        break
                        
                total_remove_inds = np.append(total_remove_inds, remove_inds).astype(int)
                #remove already remove indices from being erroneously matched later
                cut_match_inds = np.array([x for x in cut_match_inds if x not in total_remove_inds]).astype(int)


            #check if have files have different measurement methods - keep method which is most utilised over all of time record (for consistency)
            #if methods have same count, keep first method
            if len(np.unique(measure_methods[cut_match_inds])) != 1:
                print 'Files with different measurement methods. Keeping most utilised method across time record.'
                measure_counts = Counter(measure_methods)
                n_measures = []
                for m in measure_methods[cut_match_inds]:
                    n_measures=np.append(n_measures,measure_counts[m])
                remove_inds = np.array(list(cut_match_inds[:np.argmax(n_measures)])+list(cut_match_inds[np.argmax(n_measures)+1:]))

                total_remove_inds = np.append(total_remove_inds, remove_inds).astype(int)
                #remove already remove indices from being erroneously matched later
                cut_match_inds = np.array([x for x in cut_match_inds if x not in total_remove_inds]).astype(int)

            #check if have files have different instrument names - keep instrument name which is most referenced over all of time record (for consistency)
            #if names have same count, keep first name
            if len(np.unique(instru_names[cut_match_inds])) != 1:
                print 'Files with different instrument names. Keeping most referenced instrument name across time record.'
                name_counts = Counter(instru_names)
                n_names = []
                for m in instru_names[cut_match_inds]:
                    n_names=np.append(n_names,name_counts[m])
                remove_inds = np.array(list(cut_match_inds[:np.argmax(n_names)])+list(cut_match_inds[np.argmax(n_names)+1:]))

                total_remove_inds = np.append(total_remove_inds, remove_inds).astype(int)
                #remove already remove indices from being erroneously matched later
                cut_match_inds = np.array([x for x in cut_match_inds if x not in total_remove_inds]).astype(int)
                
            #if have no indices to remove - cannot determine discernible difference between files
            #keep first file preferentially
            if len(cut_match_inds) > 1:
                print 'Time overlap - but cannot determine differences between files. Keeping first file preferentially'
                total_remove_inds = np.append(total_remove_inds, cut_match_inds[1:]).astype(int)
                 
        #exclude determined files from further processing
        if len(total_remove_inds) > 0:
            print 'old files = ', data_files
            print 'removing files = ', data_files[total_remove_inds]
            data_files = np.delete(data_files,total_remove_inds)
            print 'remaining files = ', data_files

        
    #----------------------------------------------------------------------------------
    
    #iterate through site files
    for file_n, data_file in enumerate(data_files):
        
        print data_file
        
        #empty lists to append site data variables for file
        times = []
        measurements = []
        temporal_resolutions = []
        flags = []
        qa = []
        
        #flag for exiting from badly formatted file
        process_valid = True
        
        #flag to indicate if are processing last file to do some final activities 
        if file_n == len(data_files)-1:
            final_file = True
        else:
            final_file = False
        
        while process_valid == True:
        
            print data_file
        
            #Get data level - this is given in both in filename and inside file as metadata field. 
            #Level 2 data is the maximum quality assured data - class all other levels as preliminary data
            #If exists in metadata, write this out preferentially. If not there, write from filename
            #if data level is not given anywhere, assume level 2
            if '.lev' in data_file:
                data_level = data_file.split('.lev')[-1].split('.nas')[0]
                data_level = data_level.split('_')[0]
                if data_level != '':
                    data_level_string = str(data_level)
                
            #get temporal resolution of measurements in file
            temporal_resolution = data_file.split('/')[-1].split('.')[7]
            #calculate timedelta of standard temporal resolution of datafile
            if 'h' in temporal_resolution: 
                temporal_resolution_timedelta = relativedelta(hours=+int(temporal_resolution.replace('h','')))
            elif 'd' in temporal_resolution:
                temporal_resolution_timedelta = relativedelta(days=+int(temporal_resolution.replace('d','')))
            elif 'w' in temporal_resolution:
                temporal_resolution_timedelta = relativedelta(weeks=+int(temporal_resolution.replace('w','')))    
            elif 'mo' in temporal_resolution:
                temporal_resolution_timedelta = relativedelta(months=+int(temporal_resolution.replace('mo','')))               

            #----------------------------------------------------------------------------#
            #READ FILE
            #----------------------------------------------------------------------------#
    
            site_nasa_ames_obj = crossnetwork_processing_modules.process_nasa_ames(data_file)
            site_nasa_ames_obj.read_header()
            site_nasa_ames_obj.read_data()
        
            #get file reference start date
            start_date_reference = site_nasa_ames_obj.date_start
    
            #get start/end time arrays
            s_time = site_nasa_ames_obj.structured_data[0]
            e_time = site_nasa_ames_obj.structured_data[1]
        
            #create datetime from YYYYMMDD start date file reference 
            dt_start = datetime.datetime(int(start_date_reference[:4]),int(start_date_reference[4:6]),int(start_date_reference[6:]))
            #iterate through file times (in units of days from start date reference), and calculate the time in integer minutes from the desired start year
            #array to append file times
            file_times = []
            for ii in range(len(s_time)):
                #create datetimes for each time given by adding to datetime date file reference, round to nearest minute to avoid issues due to floating point estimation
                dt_now_s = crossnetwork_processing_modules.round_to_minute(dt_start+datetime.timedelta(days=float(s_time[ii])))
                dt_now_e = crossnetwork_processing_modules.round_to_minute(dt_start+datetime.timedelta(days=float(e_time[ii])))

                #convert each start measurement datetime to minutes from reference time
                frac_time = crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(dt_now_s)                
            
                #for first time of file, save it out for meta_update_stamp as minutes relative to reference time
                if ii == 0:
                    site_metadata['meta_update_stamp']['value'].append(frac_time)
            
                #get timedelta between start and end of measurement window
                dt_window = dt_now_e - dt_now_s
                
                #does measurement window equal standard time resolution file is reported in? 
                #if so, write out standard resolution code
                if dt_window == temporal_resolution_timedelta:
                    temporal_resolution_code = re.sub('[^a-zA-Z]+', "", temporal_resolution)[0].upper()
                    
                #if not, write out time in minutes
                #check if time in minutes equals one of standard resolutions      
                else:
                    #convert hours to minutes
                    dt_window_minutes = dt_window.total_seconds()/60.
                    

                    try:
                        temporal_resolution_code = standard_resolution_dict[dt_window_minutes]
                    except KeyError:
                        temporal_resolution_code = str(int(dt_window_minutes))
                temporal_resolutions.append(temporal_resolution_code)
                    
                #append processed time - do not need to shift time base as already in UTC
                file_times.append(frac_time)               
            
            #if length of current appended times == 0, then do not process file, as zero valid data
            if len(file_times) == 0:
                process_valid = False
            else:
                for n in range(len(file_times)):
                    times.append(file_times[n])
            
            #variable flags for parsing file
            var_ind = -999
            flag = -999
        
            #get desired measured parameter data
            #check how many inds there are matched for relevant parameter
            relevant_var_inds = np.where(np.array(site_nasa_ames_obj.primary_names) == parameter_details['ebas_parameter_name'])[0]
            #typically there will be more than 1 set of units for only gas phase species (mixing ratio and mass density) 
            if len(relevant_var_inds) > 1:
                #if more than one valid ind for wanted parameter, cut for preferential units 
                #each parameter has defined preferential units as a list (first element is preferred), iterate list until have units desired 
                priority_units_list = parameter_details['ebas_priority_units']
                for priority_units in priority_units_list:
                    unit_inds = [idx for idx,s in enumerate(site_nasa_ames_obj.primary_units) if priority_units in s]
                    #if have got desired units in file, break out of loop
                    if len(unit_inds) > 0:
                        break
                #if have not found any desired units, break processing to inform of this horrible development
                if len(unit_inds) == 0:
                    print 'Cannot find any desired units in file'
                    1+'a'
                
                #next check overlap of desired var and units
                intersect_inds = np.array([val for val in relevant_var_inds if val in unit_inds])
                #if more than 1 valid intersect ind, must be due to reporting different metrics (mean, min, max etc.) 
                #cut to just get arithmetic mean
                if len(intersect_inds) > 1:
                
                    #have to check for duplicate column data, as there some instances where there are repeats (e.g. - CH0002R, 2007, for NO)
                    #check if variable names, units and extra details are same for all intersects
                    #if so, just take first intersect ind preferentially
                
                    if (len(np.unique(np.array(site_nasa_ames_obj.primary_names)[intersect_inds])) == 1) & (len(np.unique(np.array(site_nasa_ames_obj.primary_units)[intersect_inds])) == 1) & (len(np.unique(np.array(site_nasa_ames_obj.primary_details)[intersect_inds])) < 2):
                        var_ind =  intersect_inds[0]
                    
                    #otherwise cut for arithmetic mean
                    else:
                        for int_i in intersect_inds:
                            for detail in site_nasa_ames_obj.primary_details[int_i]:
                                if 'arithmetic mean' in detail:
                                    var_ind = int_i
                                    break       
                else:
                    var_ind = intersect_inds[0]
            else:
                var_ind = relevant_var_inds[0]
                
            #get parameter measurements
            file_measurements = np.array(site_nasa_ames_obj.structured_data[var_ind+1])
            
            #get data flags
            #check how many inds there are for flags
            relevant_flag_inds = [idx for idx,s in enumerate(site_nasa_ames_obj.primary_names) if 'numflag' in s]
        
            #if more than 1 flag in file then must be because there multiple different metrics reported (mean, min, max etc.), each with associated flag
            #in this case simply add 1 to determined var indice, as the flag is always reported immediately after 
            if len(relevant_flag_inds) > 1:
                flag_ind = var_ind+1 
            else:
                flag_ind = relevant_flag_inds[0]
        
            #add 1 to allow for starttime field (not one of primary names)
            file_flags = np.array(site_nasa_ames_obj.structured_data[flag_ind+1])
            
            #----------------------------------------------------------------------------#
            #PROCESS METADATA
            #----------------------------------------------------------------------------#
            
            #GHOST version
            site_metadata['GHOST_version']['value'].append(run_details['ghost_version'])
            
            #reference
            site_metadata['station_reference']['value'].append(ref)
        
            #get length of current metadata lists written 
            n_meta = len(site_metadata['station_reference']['value'])
       
            #create dictionary that metadata that can be appended to in both details and comments sections 
            #no parameter specific metadata overlaps between details and comments, however this is useful when compling metadata from different fields into 1 standardised meta field.
            concat_meta = {}
       
            #dictionary for storing raw metadata units with quantity name
            units_dict = {}
       
            #process metadata in details
            concat_meta,units_dict = process_metadata(site_nasa_ames_obj.primary_details[var_ind],'details',concat_meta,units_dict)

            #process metadata in comments
            concat_meta,units_dict = process_metadata(site_nasa_ames_obj.comments,'comments',concat_meta,units_dict)
    
            #merge certain saved metadata fields into 1 field
            #delete merged fields from concat_meta afterwards
            
            #associated_networks
            variables_to_merge = ['wdca_id','gaw_id','airs_id','other_ids']
            for var in variables_to_merge:
                #if variable not in concat_meta, make it empty string
                if var not in concat_meta.keys():
                    concat_meta[var] = ''
            #make merged variable
            concat_meta['associated_networks'] = '%s;%s;%s;%s'%(concat_meta['wdca_id'],concat_meta['gaw_id'],concat_meta['airs_id'],concat_meta['other_ids'])
            #delete merged fields
            for var in variables_to_merge:
                del concat_meta[var]
            
            #measurement information
            variables_to_merge = ['instrument_type','instrument_manufacturer','instrument_model','analytical_measurement_technique','analytical_instrument_manufacturer','analytical_instrument_model']
            for var in variables_to_merge:
                #if variable not in concat_meta, make it empty string
                if var not in concat_meta.keys():
                    concat_meta[var] = ''
            #make merged variable
            concat_meta['measurement_information'] = '%s,%s,%s,%s,%s,%s'%(concat_meta['instrument_type'], concat_meta['instrument_manufacturer'], concat_meta['instrument_model'],concat_meta['analytical_measurement_technique'], concat_meta['analytical_instrument_manufacturer'], concat_meta['analytical_instrument_model'])
            #delete merged fields
            for var in variables_to_merge:
                del concat_meta[var]
            
            #inlet information
            variables_to_merge = ['inlet_type','inlet_description','inlet_tube_material','inlet_outer_diameter','inlet_inner_diameter','inlet_length']
            for var in variables_to_merge:
                #if variable not in concat_meta, make it empty string
                if var not in concat_meta.keys():
                    concat_meta[var] = ''
            #make merged variable
            concat_meta['measuring_instrument_inlet_information'] = '%s;%s;%s;%s;%s;%s'%(concat_meta['inlet_type'], concat_meta['inlet_description'], concat_meta['inlet_tube_material'], concat_meta['inlet_outer_diameter'], concat_meta['inlet_inner_diameter'], concat_meta['inlet_length'])
            #change " to inches in merged string if there - causes formatting issues further down pipe
            concat_meta['measuring_instrument_inlet_information'] = concat_meta['measuring_instrument_inlet_information'].replace('"','inch')
            #delete merged fields
            for var in variables_to_merge:
                del concat_meta[var]
            
            #sampling details
            variables_to_merge = ['filter_face_velocity','exposed_filter_area','filter_description','medium','filter_prefiring','filter_conditioning','coating','sample_preparation']
            for var in variables_to_merge:
                #if variable not in concat_meta, make it empty string
                if var not in concat_meta.keys():
                    concat_meta[var] = ''
            #make merged variable
            concat_meta['network_sampling_details'] = '%s;%s;%s;%s;%s;%s;%s;%s'%(concat_meta['filter_face_velocity'],concat_meta['exposed_filter_area'],concat_meta['filter_description'],concat_meta['medium'],concat_meta['filter_prefiring'],concat_meta['filter_conditioning'],concat_meta['coating'],concat_meta['sample_preparation'])
            #delete merged fields
            for var in variables_to_merge:
                del concat_meta[var]
    
            #uncertainty details
            variables_to_merge = ['measurement_uncertainty_explained','detection_limit_explained','zero_values']
            for var in variables_to_merge:
                #if variable not in concat_meta, make it empty string
                if var not in concat_meta.keys():
                    concat_meta[var] = ''
            #make merged variable
            concat_meta['network_uncertainty_details'] = '%s;%s;%s'%(concat_meta['measurement_uncertainty_explained'],concat_meta['detection_limit_explained'],concat_meta['zero_values'])
            #delete merged fields
            for var in variables_to_merge:
                del concat_meta[var]
            
            #maintenance details
            variables_to_merge = ['maintenance_description','zero_type','zero_interval','blank_correction','artifact_correction_description','charring_correction','humidity_control','humidity_description']
            for var in variables_to_merge:
                #if variable not in concat_meta, make it empty string
                if var not in concat_meta.keys():
                    concat_meta[var] = ''
            #make merged variable
            concat_meta['network_maintenance_details'] = '%s;%s;%s;%s;%s;%s;%s;%s'%(concat_meta['maintenance_description'],concat_meta['zero_type'],concat_meta['zero_interval'],concat_meta['blank_correction'],concat_meta['artifact_correction_description'],concat_meta['charring_correction'],concat_meta['humidity_control'],concat_meta['humidity_description'])
            #delete merged fields
            for var in variables_to_merge:
                del concat_meta[var]
            
            #Network QA details
            variables_to_merge = ['qa_date','qa_outcome','qa_bias','qa_variability','qa_document_url']  
            for var in variables_to_merge:
                #if variable not in concat_meta, make it empty string
                if var not in concat_meta.keys():
                    concat_meta[var] = ''
            #make merged variable
            concat_meta['network_qa_details'] = '%s;%s;%s;%s;%s'%(concat_meta['qa_date'],concat_meta['qa_outcome'],concat_meta['qa_bias'],concat_meta['qa_variability'],concat_meta['qa_document_url'])
            #delete merged fields
            for var in variables_to_merge:
                del concat_meta[var]
    
            #iterate through concatenated metadata and remove formatting inconsistencies, and deal with empty strings
            for key in concat_meta.keys():
                #strip white space from ends of string
                concat_meta[key] = concat_meta[key].strip()
                #if have empty commas or semi colons, replace with blank space
                concat_meta[key] = concat_meta[key].replace(',,','')
                concat_meta[key] = concat_meta[key].replace(';;','')
                concat_meta[key] = concat_meta[key].replace('  ',' ')
                #if first or last character is semicolon, colon or comma, cut it off
                if len(concat_meta[key]) > 1:
                    if (concat_meta[key][0] == ';') or (concat_meta[key][0] == ':') or (concat_meta[key][0] == ','):
                        concat_meta[key] = concat_meta[key][1:]
                    if (concat_meta[key][-1] == ';') or (concat_meta[key][-1] == ':') or (concat_meta[key][-1] == ','):
                        concat_meta[key] = concat_meta[key][:-1]
                #strip white space from ends of string
                concat_meta[key] = concat_meta[key].strip()
                #convert empty string, or solo commas/semicolons/colons to NaN
                if (concat_meta[key] == '') or (concat_meta[key] == ';') or (concat_meta[key] == ',') or (concat_meta[key] == ':'):
                    concat_meta[key] = np.NaN
    
            #calculate proximity to coast
    
            #process station classifications
            #if both file area_classification and land use are present in file, then process standardised classifications
            if ('area_classification' in concat_meta) & ('land_use' in concat_meta):
                #get standardised classifications
                native_classifications = {'area_classification':concat_meta['area_classification'],'land_use':concat_meta['land_use']}
                area_classification,station_classification,main_emission_source,land_use,terrain,measurement_scale = standardisation_modules.mapping_network_classifications(native_classifications)
                site_metadata['standardised_network_provided_area_classification']['value'].append(area_classification)
                site_metadata['standardised_network_provided_station_classification']['value'].append(station_classification)
                site_metadata['standardised_network_provided_main_emission_source']['value'].append(main_emission_source)
                site_metadata['standardised_network_provided_land_use']['value'].append(land_use)
                site_metadata['standardised_network_provided_terrain']['value'].append(terrain)
                site_metadata['standardised_network_provided_measurement_scale']['value'].append(measurement_scale)
            #delete parameters from concat_meta
            if 'area_classification' in concat_meta:
                del concat_meta['area_classification']
            if 'land_use' in concat_meta:
                del concat_meta['land_use']
    
            #process measurement information
            if pd.isnull(concat_meta['measurement_information']) == False:
                measurment_standard_dict, measurement_method_qa_flags = instrument_processing_modules.network_instrument_mapping(concat_meta['measurement_information'],site_metadata['meta_update_stamp']['value'][-1])
                #if have correctly mapped standardised measurement information, write it out 
                if measurement_method_qa_flags['un_mapped_method_flag'] == False:
                    site_metadata['primary_sampling_instrument_name']['value'].append(measurment_standard_dict['primary_sampling_instrument'])
                    site_metadata['primary_sampling_type']['value'].append(measurment_standard_dict['primary_sampling_type'])
                    site_metadata['primary_sampling_instrument_documented_flow_rate']['value'].append(measurment_standard_dict['primary_sampling_instrument_documented_flow_rate'])
                    site_metadata['primary_sampling_instrument_reported_flow_rate']['value'].append(measurment_standard_dict['primary_sampling_instrument_reported_flow_rate'])
                    site_metadata['primary_sampling_process_details']['value'].append(measurment_standard_dict['primary_sampling_process_details'])
                    site_metadata['primary_sampling_instrument_manual']['value'].append(measurment_standard_dict['primary_sampling_instrument_manual_name'])
                    site_metadata['primary_sampling_further_details']['value'].append(measurment_standard_dict['primary_sampling_instrument_further_details'])
                    site_metadata['sample_preparation_methodology_names']['value'].append(measurment_standard_dict['sample_preparation'])
                    site_metadata['sample_preparation_methodology_types']['value'].append(measurment_standard_dict['preparation_type'])
                    site_metadata['sample_preparation_process_details']['value'].append(measurment_standard_dict['preparation_process_details'])
                    site_metadata['sample_preparation_further_details']['value'].append(measurment_standard_dict['preparation_further_details'])
                    site_metadata['measuring_instrument_name']['value'].append(measurment_standard_dict['measuring_instrument'])
                    site_metadata['measurement_methodology']['value'].append(measurment_standard_dict['methodology'])
                    site_metadata['measuring_instrument_sampling_type']['value'].append(measurment_standard_dict['measuring_instrument_sampling_type'])
                    site_metadata['measuring_instrument_documented_flow_rate']['value'].append(measurment_standard_dict['measuring_instrument_documented_flow_rate'])
                    site_metadata['measuring_instrument_reported_flow_rate']['value'].append(measurment_standard_dict['measuring_instrument_reported_flow_rate'])
                    site_metadata['measuring_instrument_process_details']['value'].append(measurment_standard_dict['measuring_instrument_process_details'])
                    site_metadata['measuring_instrument_manual_name']['value'].append(measurment_standard_dict['measuring_instrument_manual_name'])
                    site_metadata['measuring_instrument_further_details']['value'].append(measurment_standard_dict['measuring_instrument_further_details'])
                    site_metadata['measuring_instrument_reported_lower_limit_of_detection']['value'].append(measurment_standard_dict['reported_lower_limit_of_detection'])
                    site_metadata['measuring_instrument_documented_lower_limit_of_detection']['value'].append(measurment_standard_dict['documented_lower_limit_of_detection'])
                    site_metadata['measuring_instrument_reported_upper_limit_of_detection']['value'].append(measurment_standard_dict['reported_upper_limit_of_detection'])
                    site_metadata['measuring_instrument_documented_upper_limit_of_detection']['value'].append(measurment_standard_dict['documented_upper_limit_of_detection'])
                    site_metadata['measuring_instrument_reported_uncertainty']['value'].append(measurment_standard_dict['reported_uncertainty'])
                    site_metadata['measuring_instrument_documented_uncertainty']['value'].append(measurment_standard_dict['documented_uncertainty'])
                    site_metadata['measuring_instrument_reported_accuracy']['value'].append(measurment_standard_dict['reported_accuracy'])
                    site_metadata['measuring_instrument_documented_accuracy']['value'].append(measurment_standard_dict['documented_accuracy'])
                    site_metadata['measuring_instrument_reported_precision']['value'].append(measurment_standard_dict['reported_precision'])
                    site_metadata['measuring_instrument_documented_precision']['value'].append(measurment_standard_dict['documented_precision'])
                    site_metadata['measuring_instrument_reported_zero_drift']['value'].append(measurment_standard_dict['reported_zero_drift'])
                    site_metadata['measuring_instrument_documented_zero_drift']['value'].append(measurment_standard_dict['documented_zero_drift'])
                    site_metadata['measuring_instrument_reported_span_drift']['value'].append(measurment_standard_dict['reported_span_drift'])
                    site_metadata['measuring_instrument_documented_span_drift']['value'].append(measurment_standard_dict['documented_span_drift'])
                    site_metadata['measuring_instrument_reported_measurement_resolution']['value'].append(measurment_standard_dict['reported_resolution'])
                    site_metadata['measuring_instrument_documented_measurement_resolution']['value'].append(measurment_standard_dict['documented_resolution'])
                    site_metadata['measuring_instrument_reported_absorption_cross_section']['value'].append(measurment_standard_dict['reported_absorption_cross_section'])
                    site_metadata['measuring_instrument_documented_absorption_cross_section']['value'].append(measurment_standard_dict['documented_absorption_cross_section'])
            #delete parameter from concat_meta
            del concat_meta['measurement_information']
            
            #write rest of saved metadata out
            for column_header in concat_meta.keys():
                n_column = len(site_metadata[column_header]['value'])
                #if length of column is less than number of references, then hasn't been written to yet, therefore append
                if n_column < n_meta:
                    site_metadata[column_header]['value'].append(concat_meta[column_header])
                #otherwise it has been written as NaN already, therefore overwrite
                else:
                    site_metadata[column_header]['value'][-1] = concat_meta[column_header]
    
            #write known network specific metadata  
            site_metadata['geodetic_datum']['value'].append('WSG84')
            site_metadata['country']['value'].append(crossnetwork_processing_modules.country_ISO3166_to_name(ref[:2])) 
            site_metadata['network']['value'].append('EBAS')
                        
                        
            #iterate through metadata variables, if any columns have not been appended to add a NaN
            #check number of refs in list
            for column_header in site_metadata.keys():
                n_column = len(site_metadata[column_header]['value'])
                #if length of column is less than number of references, then append NaN
                if n_column < n_meta:
                    site_metadata[column_header]['value'].append(np.NaN)
                    
            #process non-network specific metadata (station_timezone, data_download_stamp)
            site_metadata = crossnetwork_processing_modules.process_non_network_specific_metadata(site_metadata, data_file)
            
            #if do not have data level from metadata, try writing out data level from filename
            #if no level in filename either, assume it to be level 2 (highest level) 
            if pd.isnull(site_metadata['data_level']['value'][-1]) == True:
                try:
                    site_metadata['data_level']['value'][-1] = str(float(data_level_string))
                except:
                    site_metadata['data_level']['value'][-1] = '2.0'
            else:
                site_metadata['data_level']['value'][-1] = str(float(site_metadata['data_level']['value'][-1]))            
            
                        
            #-----------------------------------------------------
            #convert metadata data with units to standard units
            
            #get measurement units
            units = site_nasa_ames_obj.primary_units[var_ind].strip()
            #convert raw units given to standard name
            conv_obj = converter.convert_units(units,units,1, measured_species = parameter_details['chemical_formula'])
            conv_obj.do_conversion()
            site_metadata['measuring_instrument_reported_units']['value'][-1] = conv_obj.output_standard_units
            
            #process standard temperature and pressure first - potentially need later for converting gas concentrations
            #use EBAS stated standards if not given - also set flag to know have assumed gas volume, if later converting units
            #given as 293.15 K / 1013.25 hPa for O3, & 273.15 K / 1013.25 hPa for all other parameters:
            assume_gas_volume = False
            
            if 'measuring_instrument_volume_standard_temperature' in units_dict:
                
                #if metadata units not equal to standard units, convert!
                if units_dict['measuring_instrument_volume_standard_temperature'] != site_metadata['measuring_instrument_volume_standard_temperature']['units']:
                    conv_obj = converter.convert_units(units_dict['measuring_instrument_volume_standard_temperature'],site_metadata['measuring_instrument_volume_standard_temperature']['units'],site_metadata['measuring_instrument_volume_standard_temperature']['value'][-1], measured_species = parameter_details['chemical_formula'])
                    conv_obj.do_conversion()
                    site_metadata['measuring_instrument_volume_standard_temperature']['value'][-1] = conv_obj.converted_value
                #remove from units dict 
                del units_dict['measuring_instrument_volume_standard_temperature']
            else:
                assume_gas_volume = True
                if run_details['parameter'] == 'O3':
                    site_metadata['measuring_instrument_volume_standard_temperature']['value'][-1] = '293.15'
                else:
                    site_metadata['measuring_instrument_volume_standard_temperature']['value'][-1] = '273.15'
            
            if 'measuring_instrument_volume_standard_pressure' in units_dict:
            
                #if metadata units not equal to standard units, convert!
                if units_dict['measuring_instrument_volume_standard_pressure'] != site_metadata['measuring_instrument_volume_standard_pressure']['units']:
                    conv_obj = converter.convert_units(units_dict['measuring_instrument_volume_standard_pressure'],site_metadata['measuring_instrument_volume_standard_pressure']['units'],site_metadata['measuring_instrument_volume_standard_pressure']['value'][-1], measured_species = parameter_details['chemical_formula'])
                    conv_obj.do_conversion()
                    site_metadata['measuring_instrument_volume_standard_pressure']['value'][-1] = conv_obj.converted_value
                 #remove from units dict 
                del units_dict['measuring_instrument_volume_standard_pressure']
            else:
                assume_gas_volume = True
                site_metadata['measuring_instrument_volume_standard_pressure']['value'][-1] = '1013.25'

            #process rest of meta units
            for meta_name in units_dict:
                #if metadata units not equal to standard units, convert!
                if units_dict[meta_name] != site_metadata[meta_name]['units']:
                    
                    #some gas concentration values may need converting to mole fraction from mass density (using reported standard temperature and pressure)
                    if (meta_name in ['measuring_instrument_reported_lower_limit_of_detection','measuring_instrument_reported_uncertainty']) & ('/m3' in units_dict[meta_name]) & ('mol-1' in site_data[parameter_details['bsc_parameter_name']]['units']):
                        input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':units_dict[meta_name]}
                        input_vals = {'temperature':site_metadata['measuring_instrument_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['measuring_instrument_volume_standard_pressure']['value'][-1], 'molar_mass':crossnetwork_processing_modules.get_molecular_mass(parameter_details['chemical_formula']), 'mass_density':site_metadata[meta_name]['value'][-1]}
                        conv_obj  = converter.convert_units(input_units,site_metadata[meta_name]['units'],input_vals,measured_species = parameter_details['chemical_formula'])
                    #quantities which just need converting i.e. cm2/s --> m2/min
                    else:
                        conv_obj = converter.convert_units(units_dict[meta_name],site_metadata[meta_name]['units'],site_metadata[meta_name]['value'][-1], measured_species = parameter_details['chemical_formula'])
                        
                    #do conversion
                    conv_obj.do_conversion()
                    site_metadata[meta_name]['value'][-1] = conv_obj.converted_value
            
            #----------------------------------------------------------------------------#
            #PROCESS DATA
            #----------------------------------------------------------------------------#

            #flags
            #load standardised flags dict
            standard_flags = standardisation_modules.standard_flags()
            #map flags to standard flags
            for n, flag in enumerate(file_flags):
                
                #get flag into integer number, equivalent to format as in standard flag dictionary 
                if '.' in flag:
                    flag = flag.split('.')[-1]
                if len(flag) > 3:
                    flag = flag[:3]
                
                standard_flag = standard_flags[flag]['flag_type']
                qa_flag_science = standard_flags[flag]['self_decreed_validity']
                qa_flag_network = standard_flags[flag]['network_decreed_validity']
                
                #Should not have any empty flags, if so stop processing
                if standard_flag == '':
                    print 'Flag is empty, Do not know why. Breaking processing.'
                    1+'a'
                
                #write preliminary data flags (informs if data file contains preliminary data (lower than level 2))
                if (site_metadata['data_level']['value'][-1] != '2.0') & ('P' not in standard_flag):
                    standard_flag = standard_flag+'P'
                    qa_flag_science = 'I'
                    qa_flag_network = 'V'
                
                #append standard flag
                flags.append(standard_flag)
                
                #process qa information associated with flags
                qa_str = ''
                #append qa flag if standard flag is decreed scientifically invalid
                if qa_flag_science == 'I':
                    qa_str = qa_str+standardisation_modules.get_standard_qa_flag('Invalid Data Provider Flags - Scientifically Decreed')
                #append qa flag if standard flag is network decreed invalid
                if qa_flag_network == 'I':
                    qa_str = qa_str+standardisation_modules.get_standard_qa_flag('Invalid Data Provider Flags - Network Decreed')
                
                qa.append(qa_str)
                
            #--------------------------------------
            #measurements
            #measurement values may need converting (i.e. mass density to mixing ratio)
            #--------------------------------------
            
            #if gas volume not provided by data provider and units need converting, this conversion is done using assumed network standard pressure and temperatures - write qa data flag to inform of this assumption
            #this flag below alerts for writing of qa flag later
            write_volume_assumption_flag = False
            
            #check if reported standard units, match required standard units
            if site_metadata['measuring_instrument_reported_units']['value'][-1] != site_data[parameter_details['bsc_parameter_name']]['units']:
                
                #mass density to mixing ratio?
                if ('m-3' in site_metadata['measuring_instrument_reported_units']['value'][-1]) & ('mol-1' in site_data[parameter_details['bsc_parameter_name']]['units']):
                    
                    #set flag to instruct writing of flag that am assuming gas volume in converting units 
                    if assume_gas_volume == True:
                        write_volume_assumption_flag = True
                    
                    #get conversion factor
                    input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':site_metadata['measuring_instrument_reported_units']['value'][-1]}
                    input_vals = {'temperature':site_metadata['measuring_instrument_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['measuring_instrument_volume_standard_pressure']['value'][-1], 'molar_mass':crossnetwork_processing_modules.get_molecular_mass(parameter_details['chemical_formula']), 'mass_density':1.0}
                    conv_obj = converter.convert_units(input_units,site_data[parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
                    
                #otherwise simple unit conversion, i.e. pptv to ppbv or mass density to mass density
                else:
                    conv_obj = converter.convert_units(site_metadata['measuring_instrument_reported_units']['value'][-1],site_data[parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = parameter_details['chemical_formula'])
            
                #do conversion
                conv_obj.do_conversion()
                
                #process through each measurement and convert value using derived conversion factor
                for n, meas in enumerate(file_measurements):
                    #only do conversion if measurement value is above 0 and measurement is not missing (determined using standard flag)
                    if (float(meas) > 0.0) & ('M' not in flags[n]):
                        measurements.append(float(meas)*conv_obj.conversion_factor)
                    else:
                        measurements.append(meas)
                
            #otherwise do nothing to measured values
            else:
                for n, meas in enumerate(file_measurements):
                    measurements.append(meas)
                    
            #----------------------------------------------------------------------------#    
            #QA
            #----------------------------------------------------------------------------#
            #write instrument/methodology qa flags, using standard qa flags
            #dictionary maps variable names returned from methodology processing to standardised QA codes
            qa_mapping_dict = {'un_mapped_method_flag':'Methodology Not Mapped',
                               'primary_sampling_assumption_flag':'Assumed Primary Sampling',
                               'preparation_assumption_flag':'Assumed Sample Preparation',
                               'measuring_assumption_flag':'Assumed Measurement Methodology',
                               'unknown_primary_sampling_instrument_flag':'Unknown Primary Sampling Instrument',
                               'unknown_measuring_instrument_flag':'Unknown Measuring Instrument',
                               'erroneous_primary_sampling_flag':'Erroneous Primary Sampling',
                               'erroneous_sample_preparation_flag':'Erroneous Sample Preparation',
                               'erroneous_measurement_methodology_flag':'Erroneous Measurement Methodology',
                               'not_qc_accepted_method_flag':'Invalid QA Measurement Methodology'}
                               
            #iterate through instrument/methodology flags and write standard qa flag if required
            for variable_name in qa_mapping_dict.keys():
                if measurement_method_qa_flags[variable_name] == True:
                    for x, qa_str in enumerate(qa):
                        qa[x] = qa_str+standardisation_modules.get_standard_qa_flag(qa_mapping_dict[variable_name])
            
            #write assumed volume of gas qa flags (inform have assumed volume of gas for converting units)
            if write_volume_assumption_flag == True:
                for x, qa_str in enumerate(qa):
                    qa[x] = qa_str+standardisation_modules.get_standard_qa_flag('Assumed Gas Volume')
            
            #------------------------------------------------------
            #write out site variables
            #No LODs or measurement uncertainties per measurement - append list of NaNs    
            for n in range(len(times)):
                site_data['time']['value'].append(times[n])
                site_data[parameter_details['bsc_parameter_name']]['value'].append(measurements[n])
                site_data['temporal_resolution']['value'].append(temporal_resolutions[n])   
                site_data['reported_lower_limit_of_detection_per_measurement']['value'].append(np.NaN)
                site_data['reported_upper_limit_of_detection_per_measurement']['value'].append(np.NaN)
                site_data['reported_uncertainty_per_measurement']['value'].append(np.NaN)
                site_data['derived_uncertainty_per_measurement']['value'].append(np.NaN)
                site_data['flag']['value'].append(flags[n])
                site_data['qa']['value'].append(qa[n])
            #------------------------------------------------------
            
            #if final file, do some final processing
            #1. add associated network ids from manually compiled metadata, if do not already exist in metadata
            #2. write process standard classifications from manually compiled EBAS classes (if no other classes given for site)
            
            if final_file == True:
                #check if have all associated network ids for station from manually compiled metadata
                manual_other_network_reference_string = external_metadata['other_network_references']
                
                #if have some reference ids, split apart into list (on semicolon)
                if len(manual_other_network_reference_string) != 0:
                    manual_other_network_references = manual_other_network_reference_string.split(';')
                    
                    #iterate through ids and check if already have them saved in associated networks (check all strings through time record) 
                    for id in manual_other_network_references:
                        for ii, saved_ids in enumerate(site_metadata['associated_networks']['value']):
                            #if id not there, append it
                            if pd.isnull(saved_ids) == True:
                                site_metadata['associated_networks']['value'][ii] = id
                            else:
                                if id not in saved_ids:
                                    site_metadata['associated_networks']['value'][ii] = site_metadata['associated_networks']['value'][ii]+';'+id
                                
            
                #if do not have station classifications in any of the data files, then read and process manually compiled classifications (defined with EBAS own definitions)
                station_area_classes = site_metadata['standardised_network_provided_area_classification']['value']
                
                #iterate through currently processed standard_classes, for each station file
                for ii, station_area_class in enumerate(station_area_classes):
                    #if any of the classes is a string (i.e. has been processed), break, as do not need manual classes
                    if pd.isnull(station_area_class) == False:
                        break  
                        
                    #if last index, and are still processing, then no standard classes have been processed for the station, therefore read manual classes and process
                    if ii == (len(station_area_classes)-1):
                        
                        #Cut for manually classes index.   
                        #manual_class_ref_i = manual_meta_classes[manual_meta_variable_names.index('reference')].index(site_metadata['station_reference']['value'][-1])
                        manual_area_classification = external_metadata['setting'].replace(" ", "").lower()
                        manual_land_use = external_metadata['land_use'].replace(" ", "").lower()
                        manual_native_classifications = {'area_classification':manual_area_classification, 'land_use':manual_land_use}

                        #get standardised classifications and write out (for all of station files)
                        area_classification,station_classification,main_emission_source,land_use,terrain,measurement_scale = standardisation_modules.mapping_network_classifications(manual_native_classifications)            
                        
                        if pd.isnull(area_classification) == False:
                            site_metadata['standardised_network_provided_area_classification']['value'] = [area_classification]*len(data_files)
                        if pd.isnull(station_classification) == False:
                            site_metadata['standardised_network_provided_station_classification']['value'] = [station_classification]*len(data_files)
                        if pd.isnull(main_emission_source) == False:
                            site_metadata['standardised_network_provided_main_emission_source']['value'] = [main_emission_source]*len(data_files)
                        if pd.isnull(land_use) == False:
                            site_metadata['standardised_network_provided_land_use']['value'] = [land_use]*len(data_files)
                        if pd.isnull(terrain) == False:
                            site_metadata['standardised_network_provided_terrain']['value'] = [terrain]*len(data_files)
                        if pd.isnull(measurement_scale) == False:
                            site_metadata['standardised_network_provided_measurement_scale']['value'] = [measurement_scale]*len(data_files)
                
            #read file successfully. move to next file
            process_valid = False
    
    #standardise metadata/data and write out site data to temporary netcdf
    crossnetwork_processing_modules.post_process_station_data(site_metadata,site_data,ref)
         
    return 

#takes metadata from header details and comments and sorts it.
#many parmaters can appear in both header and comments, it handles these cases. 

def process_metadata(meta,meta_type,concat_meta,units_dict):
    
    line_titles = []
    line_data = []
    
    #get splitter for different section of datafile metadata is reported in
    splitter_dict = {'comments':':','details':'='}
    splitter = splitter_dict[meta_type]
    
    for line in meta: 
        line_split = line.split(splitter)
        if len(line_split) >= 2:
            line_titles.append(crossnetwork_processing_modules.remove_accents(line_split[0].strip()))
            line_data.append(crossnetwork_processing_modules.remove_accents(line_split[1].strip()))
    
    #data_level
    if 'Data level' in line_titles:
        data_level = line_data[line_titles.index('Data level')]
        if (data_level != '') & ('data_level' not in concat_meta):
            concat_meta['data_level'] = data_level

    #data_timezone
    if 'Timezone' in line_titles:
        data_timezone = line_data[line_titles.index('Timezone')]
        if (data_timezone != '') & ('timezone' not in concat_meta):
            #if timezone data is reported in is not UTC, break processing as it should be
            if data_timezone == 'UTC':
                concat_meta['data_timezone'] = '0.0'
            else:
                'Data not in UTC time!'
                1+'a'

    #latitude
    if 'Station latitude' in line_titles:
        latitude = line_data[line_titles.index('Station latitude')]
        if (latitude != '') & ('latitude' not in concat_meta):
            concat_meta['latitude'] = latitude

    #longitude
    if 'Station longitude' in line_titles:
        longitude = line_data[line_titles.index('Station longitude')]
        if (longitude != '') & ('longitude' not in concat_meta):
            concat_meta['longitude'] = longitude

    #altitude 
    if 'Station altitude' in line_titles:
        altitude = line_data[line_titles.index('Station altitude')]
        if (altitude != '') & ('altitude' not in concat_meta):
            concat_meta['altitude'] = altitude.split(' ')[0]
            units_dict['altitude'] = ''.join(altitude.split(' ')[1:])

    #sampling_height 
    if 'Measurement height' in line_titles:
        sampling_height = line_data[line_titles.index('Measurement height')]
        if (sampling_height != '') & ('sampling_height' not in concat_meta):
            concat_meta['sampling_height'] = sampling_height.split(' ')[0]
            units_dict['sampling_height'] = ''.join(sampling_height.split(' ')[1:])

    #station_name
    if 'Station name' in line_titles:
        station_name = line_data[line_titles.index('Station name')]
        if (station_name != '') & ('station_name' not in concat_meta):
            concat_meta['station_name'] = station_name

    #province
    if 'Station state/province' in line_titles:
        province = line_data[line_titles.index('Station state/province')]
        if (province != '') & ('province' not in concat_meta):
            concat_meta['province'] = province

    #flow_rate
    if ('Flow rate' in line_titles):
        reported_flow_rate = line_data[line_titles.index('Flow rate')]
        if (reported_flow_rate != '') & ('reported_flow_rate' not in concat_meta):
            concat_meta['measuring_instrument_reported_flow_rate'] = reported_flow_rate.split(' ')[0]
            units_dict['measuring_instrument_reported_flow_rate'] = ''.join(reported_flow_rate.split(' ')[1:])

    #reported_lower_limit_of_detection 
    if ('Detection limit' in line_titles):
        reported_lower_limit_of_detection = line_data[line_titles.index('Detection limit')]
        if (reported_lower_limit_of_detection != '') & ('measuring_instrument_reported_lower_limit_of_detection' not in concat_meta):
            concat_meta['measuring_instrument_reported_lower_limit_of_detection'] = reported_lower_limit_of_detection.split(' ')[0]
            units_dict['measuring_instrument_reported_lower_limit_of_detection'] = ''.join(reported_lower_limit_of_detection.split(' ')[1:])

    #reported_uncertainty (given either as a percent or absolute value) 
    if ('Measurement uncertainty' in line_titles):
        reported_uncertainty = line_data[line_titles.index('Measurement uncertainty')]
        if (reported_uncertainty != '') & ('measuring_instrument_reported_uncertainty' not in concat_meta):
            concat_meta['measuring_instrument_reported_uncertainty'] = reported_uncertainty.split(' ')[0]
            uncertainty_unit = ''.join(reported_uncertainty.split(' ')[1:])
            if uncertainty_unit != '%':
                units_dict['measuring_instrument_reported_uncertainty'] = uncertainty_unit
    
    #absorption_cross_section 
    if ('Absorption cross section' in line_titles):
        reported_absorption_cross_section = line_data[line_titles.index('Absorption cross section')]
        if (reported_absorption_cross_section != '') & ('measuring_instrument_reported_absorption_cross_section' not in concat_meta):
            concat_meta['measuring_instrument_reported_absorption_cross_section'] = reported_absorption_cross_section.split(' ')[0]
            units_dict['measuring_instrument_reported_absorption_cross_section'] = ''.join(reported_absorption_cross_section.split(' ')[1:]) 

    #calibration_scale
    if ('Calibration scale' in line_titles):
        calibration_scale = line_data[line_titles.index('Calibration scale')]
        if (calibration_scale != '') & ('measuring_instrument_calibration_scale' not in concat_meta):
            concat_meta['measuring_instrument_calibration_scale'] = calibration_scale
        
    #volume_standard_temperature 
    if ('Volume std. temperature' in line_titles):
        volume_standard_temperature = line_data[line_titles.index('Volume std. temperature')]
        if (volume_standard_temperature != '') & (volume_standard_temperature != 'ambient') & ('measuring_instrument_volume_standard_temperature' not in concat_meta):
            #must have units given for temperature, otherwise it is useless. If no units are given, write volume standard temperature out as NaN
            try:
                concat_meta['measuring_instrument_volume_standard_temperature'] = volume_standard_temperature.split(' ')[0]
                units_dict['measuring_instrument_volume_standard_temperature'] = ''.join(volume_standard_temperature.split(' ')[1:]) 
            except:
                concat_meta['measuring_instrument_volume_standard_temperature'] = np.NaN
    
    #volume_standard_pressure 
    if ('Volume std. pressure' in line_titles):
        volume_standard_pressure = line_data[line_titles.index('Volume std. pressure')]
        if (volume_standard_pressure != '') & (volume_standard_pressure != 'ambient') & ('measuring_instrument_volume_standard_pressure' not in concat_meta):
            #must have units given for pressure, otherwise it is useless. If no units are given, write volume standard pressure out as NaN
            try:
                concat_meta['measuring_instrument_volume_standard_pressure'] = volume_standard_pressure.split(' ')[0]
                units_dict['measuring_instrument_volume_standard_pressure'] = ''.join(volume_standard_pressure.split(' ')[1:]) 
            except:
                concat_meta['measuring_instrument_volume_standard_pressure'] = np.NaN
    
    #associated_networks
    if 'Station WDCA-ID' in line_titles:
        wdca_id = line_data[line_titles.index('Station WDCA-ID')].strip()
        if (wdca_id != '') & ('wdca_id' not in concat_meta):
            concat_meta['wdca_id'] = 'WDCA:'+wdca_id
    if 'Station GAW-ID' in line_titles:
        gaw_id = line_data[line_titles.index('Station GAW-ID')].strip()
        if (gaw_id != '') & ('gaw_id' not in concat_meta):
            concat_meta['gaw_id'] = 'GAW:'+gaw_id
    if 'Station AIRS-ID' in line_titles:
        airs_id = line_data[line_titles.index('Station AIRS-ID')].strip()
        if (airs_id != '') & ('airs_id' not in concat_meta):
            concat_meta['airs_id'] = 'AIRS:'+airs_id
    if 'Station other IDs' in line_titles:
        other_ids = line_data[line_titles.index('Station other IDs')].strip()
        if (other_ids != '') & ('other_ids' not in concat_meta):
            concat_meta['other_ids'] = ''
            other_ids = other_ids.split(',')
            for o_id in other_ids:
                o_id_split = o_id.split('(')
                concat_meta['other_ids'] = concat_meta['other_ids'] + '%s:%s;'%(o_id_split[1][:-1],o_id_split[0])
            #remove end semi-colon
            concat_meta['other_ids'] = concat_meta['other_ids'][:-1] 

    #area_classification/land_use
    if 'Station setting' in line_titles:
        area_classification = line_data[line_titles.index('Station setting')].replace(" ", "").lower()
        if (area_classification != '') & ('area_classification' not in concat_meta):
            concat_meta['area_classification'] = area_classification
    if 'Station land use' in line_titles:  
        land_use = line_data[line_titles.index('Station land use')].replace(" ", "").lower()
        if (land_use != '') & ('land_use' not in concat_meta):
            concat_meta['land_use'] = land_use

    #measurement information
    #instrument_type, instrument_manufacturer, instrument_model, analytical_measurement_technique, analytical_instrument_manufacturer, analytical_instrument_model
    if 'Instrument type' in line_titles:
        instrument_type = line_data[line_titles.index('Instrument type')].lower()
        if (instrument_type != '') & ('instrument_type' not in concat_meta):
            concat_meta['instrument_type'] = instrument_type
    if 'Instrument manufacturer' in line_titles:
        instrument_manufacturer = line_data[line_titles.index('Instrument manufacturer')].lower()
        if (instrument_manufacturer != '') & ('instrument_manufacturer' not in concat_meta):
            concat_meta['instrument_manufacturer'] = instrument_manufacturer
    if 'Instrument model' in line_titles:
        instrument_model = line_data[line_titles.index('Instrument model')].lower()
        if (instrument_model != '') & ('instrument_model' not in concat_meta):
            concat_meta['instrument_model'] = instrument_model
    if 'Analytical measurement technique' in line_titles:
        analytical_measurement_technique = line_data[line_titles.index('Analytical measurement technique')].lower()
        if (analytical_measurement_technique != '') & ('analytical_measurement_technique' not in concat_meta):
            concat_meta['analytical_measurement_technique'] = analytical_measurement_technique
    if 'Analytical instrument manufacturer' in line_titles:
        analytical_instrument_manufacturer = line_data[line_titles.index('Analytical instrument manufacturer')].lower()
        if (analytical_instrument_manufacturer != '') & ('analytical_instrument_manufacturer' not in concat_meta):
            concat_meta['analytical_instrument_manufacturer'] = analytical_instrument_manufacturer
    if 'Analytical instrument model' in line_titles:
        analytical_instrument_model = line_data[line_titles.index('Analytical instrument model')].lower()
        if (analytical_instrument_model != '') & ('analytical_instrument_model' not in concat_meta):
            concat_meta['analytical_instrument_model'] = analytical_instrument_model

    #inlet_information  
    if ('Inlet type' in line_titles):
        inlet_type = line_data[line_titles.index('Inlet type')].lower()
        if (inlet_type != '') & ('inlet_type' not in concat_meta):
            concat_meta['inlet_type'] = 'Inlet type: '+inlet_type
    if ('Inlet description' in line_titles):
        inlet_description = line_data[line_titles.index('Inlet description')].lower()
        if (inlet_description != '') & ('inlet_description' not in concat_meta):
            concat_meta['inlet_description'] = 'Description: '+inlet_description
    if ('Inlet tube material' in line_titles):
        inlet_tube_material = line_data[line_titles.index('Inlet tube material')].lower().strip()
        if (inlet_tube_material != '') & ('inlet_tube_material' not in concat_meta):
            concat_meta['inlet_tube_material'] = 'Tube material: '+inlet_tube_material
    if ('Inlet tube outer diameter' in line_titles):
        inlet_outer_diameter = line_data[line_titles.index('Inlet tube outer diameter')].lower().strip()
        if (inlet_outer_diameter != '') & ('inlet_outer_diameter' not in concat_meta):
            concat_meta['inlet_outer_diameter'] = 'Outer diameter: '+inlet_outer_diameter
    if ('Inlet tube inner diameter' in line_titles):
        inlet_inner_diameter = line_data[line_titles.index('Inlet tube inner diameter')].lower().strip()
        if (inlet_inner_diameter != '') & ('inlet_inner_diameter' not in concat_meta):
            concat_meta['inlet_inner_diameter'] = 'Inner diameter: '+inlet_inner_diameter
    if ('Inlet tube length' in line_titles):
        inlet_length = line_data[line_titles.index('Inlet tube length')].lower().strip()
        if (inlet_length != '') & ('inlet_length' not in concat_meta):
            concat_meta['inlet_length'] = 'Length: '+inlet_length

    #principal_investigator information
    #can be multiple - so take first instance
    if 'Originator' in line_titles:
        principal_investigator_split = line_data[line_titles.index('Originator')].split(',')
        
        principal_investigator_name =  '%s %s'%(principal_investigator_split[1].strip(),principal_investigator_split[0].strip())
        principal_investigator_name = principal_investigator_name.strip()
        if (principal_investigator_name != '') & ('principal_investigator_name' not in concat_meta):
            concat_meta['principal_investigator_name'] = principal_investigator_name
            
        principal_investigator_email_address = principal_investigator_split[2].strip()
        if (principal_investigator_email_address != '') & ('principal_investigator_email_address' not in concat_meta):
            concat_meta['principal_investigator_email_address'] = principal_investigator_email_address
            
        principal_investigator_institution = principal_investigator_split[3].strip()
        if (principal_investigator_institution != '') & ('principal_investigator_institution' not in concat_meta):
            concat_meta['principal_investigator_institution'] = principal_investigator_institution
        
   #contact information
   #can be multiple - so take first instance
    if 'Submitter' in line_titles:
        contact_split = line_data[line_titles.index('Submitter')].split(',')
        
        contact_name = '%s %s'%(contact_split[1].strip(),contact_split[0].strip())
        contact_name = contact_name.strip()
        if (contact_name != '') & ('contact_name' not in concat_meta):
            concat_meta['contact_name'] = contact_name
        
        contact_email_address = contact_split[2].strip()
        if (contact_email_address != '') & ('contact_email_address' not in concat_meta):
            concat_meta['contact_email_address'] = contact_email_address
        
        contact_institution = contact_split[3].strip()
        if (contact_institution != '') & ('contact_institution' not in concat_meta):
            concat_meta['contact_institution'] = contact_institution
            
    #uncertainty details
    if ('Measurement uncertainty expl.' in line_titles):
        measurement_uncertainty_explained =  line_data[line_titles.index('Measurement uncertainty expl.')].strip()   
        if (measurement_uncertainty_explained != '') & ('measurement_uncertainty_explained' not in concat_meta):
            concat_meta['measurement_uncertainty_explained'] = 'Measurement uncertainty explained: '+measurement_uncertainty_explained

    if ('Detection limit expl.' in line_titles):
        detection_limit_explained = line_data[line_titles.index('Detection limit expl.')].strip() 
        if (detection_limit_explained != '') & ('detection_limit_explained' not in concat_meta):
            concat_meta['detection_limit_explained'] = 'Detection limit explained: '+detection_limit_explained

    if ('Zero/negative values' in line_titles):
        zero_values = line_data[line_titles.index('Zero/negative values')].strip() 
        if (zero_values != '') & ('zero_values' not in concat_meta):
            concat_meta['zero_values'] = 'Zero/negative values: '+zero_values

    #measurement maintenance details
    if ('Maintenance description' in line_titles):
        maintenance_description = line_data[line_titles.index('Maintenance description')].strip() 
        if (maintenance_description != '') & ('maintenance_description' not in concat_meta):
            concat_meta['maintenance_description'] = 'Maintenance description: '+maintenance_description

    if ('Zero/span check type' in line_titles):
        zero_type = line_data[line_titles.index('Zero/span check type')].strip() 
        if (zero_type != '') & ('zero_type' not in concat_meta):
            concat_meta['zero_type'] = 'Zero/span check type: '+zero_type

    if ('Zero/span check interval' in line_titles):
        zero_interval = line_data[line_titles.index('Zero/span check interval')].strip() 
        if (zero_interval != '') & ('zero_interval' not in concat_meta):
            concat_meta['zero_interval'] = 'Zero/span check interval: '+zero_interval

    if ('Blank correction' in line_titles):
        blank_correction = line_data[line_titles.index('Blank correction')].strip()
        if (blank_correction != '') & ('blank_correction' not in concat_meta):
            concat_meta['blank_correction'] = 'Blank correction: '+blank_correction
    
    if ('Artifact correction description' in line_titles):
        artifact_correction_description = line_data[line_titles.index('Artifact correction description')].strip()
        if (artifact_correction_description != '') & ('artifact_correction_description' not in concat_meta):
            concat_meta['artifact_correction_description'] = 'Artifact correction description: '+artifact_correction_description
    
    if ('Charring correction' in line_titles):
        charring_correction = line_data[line_titles.index('Charring correction')].strip()
        if (charring_correction != '') & ('charring_correction' not in concat_meta):
            concat_meta['charring_correction'] = 'Charring correction: '+charring_correction
    
    if ('Humidity/temperature control' in line_titles):
        humidity_control = line_data[line_titles.index('Humidity/temperature control')].strip() 
        if (humidity_control != '') & ('humidity_control' not in concat_meta):
            concat_meta['humidity_control'] = 'Humidity/temp. control: '+humidity_control

    if ('Humidity/temperature control description' in line_titles):
        humidity_description = line_data[line_titles.index('Humidity/temperature control description')].strip() 
        if (humidity_description != '') & ('humidity_description' not in concat_meta):
            concat_meta['humidity_description'] = 'Humidity/temp. control description: '+humidity_description
    
    #sampling details 
    if ('Filter face velocity' in line_titles):
        filter_face_velocity = line_data[line_titles.index('Filter face velocity')].strip() 
        if (filter_face_velocity != '') & ('filter_face_velocity' not in concat_meta):
            concat_meta['filter_face_velocity'] = 'Filter face velocity: '+filter_face_velocity
    
    if ('Exposed filter area' in line_titles):
        exposed_filter_area = line_data[line_titles.index('Exposed filter area')].strip() 
        if (exposed_filter_area != '') & ('exposed_filter_area' not in concat_meta):
            concat_meta['exposed_filter_area'] = 'Exposed filter area: '+exposed_filter_area
    
    if ('Filter description' in line_titles):
        filter_description = line_data[line_titles.index('Filter description')].strip()
        if (filter_description != '') & ('filter_description' not in concat_meta):
            concat_meta['filter_description'] = 'Filter description: '+filter_description
    
    if ('Medium' in line_titles):
        medium = line_data[line_titles.index('Medium')].strip()
        if (medium != '') & ('medium' not in concat_meta):
            concat_meta['medium'] = 'Medium: '+medium
    
    if ('Filter prefiring' in line_titles):
        filter_prefiring = line_data[line_titles.index('Filter prefiring')].strip()
        if (zero_values != '') & ('zero_values' not in concat_meta):
            concat_meta['filter_prefiring'] = 'Filter prefiring: '+filter_prefiring
    
    if ('Filter conditioning' in line_titles):
        filter_conditioning = line_data[line_titles.index('Filter conditioning')].strip()
        if (filter_conditioning != '') & ('filter_conditioning' not in concat_meta):
            concat_meta['filter_conditioning'] = 'Filter conditioning: '+filter_conditioning
    
    if ('Coating/Solution' in line_titles):
        coating = line_data[line_titles.index('Coating/Solution')].strip()
        if (coating != '') & ('coating' not in concat_meta):
            concat_meta['coating'] = 'Coating/Solution: '+coating
    
    if ('Sample preparation' in line_titles):
        sample_preparation = line_data[line_titles.index('Sample preparation')].strip()
        if (sample_preparation != '') & ('sample_preparation' not in concat_meta):
            concat_meta['sample_preparation'] = 'Sample preparation: '+sample_preparation
    
    #network QA details
    if ('QA date' in line_titles):  
        qa_date = line_data[line_titles.index('QA date')].strip()
        if (qa_date != '') & ('qa_date' not in concat_meta):
            concat_meta['qa_date'] = 'QA date: '+qa_date
        
    if ('QA outcome' in line_titles):
        qa_outcome = line_data[line_titles.index('QA outcome')].strip()
        if (qa_outcome != '') & ('qa_outcome' not in concat_meta):
            concat_meta['qa_outcome'] = 'QA outcome: '+qa_outcome
    
    if ('QA bias' in line_titles):
        qa_bias = line_data[line_titles.index('QA bias')].strip()
        if (qa_bias != '') & ('qa_bias' not in concat_meta):
            concat_meta['qa_bias'] = 'QA bias: '+qa_bias
    
    if ('QA variability' in line_titles):
        qa_variability = line_data[line_titles.index('QA variability')].strip()
        if (qa_variability != '') & ('qa_variability' not in concat_meta):
            concat_meta['qa_variability'] = 'QA variability: '+qa_variability
    
    if ('QA document URL' in line_titles):
        qa_document_url = line_data[line_titles.index('QA document URL')].strip()
        if (qa_document_url != '') & ('qa_document_url' not in concat_meta):
            concat_meta['qa_document_url'] = 'QA document URL: '+qa_document_url    
    
    #miscellaneous details
    if ('Comment' in line_titles):
        comment = line_data[line_titles.index('Comment')].strip() 
        if (comment != '') & ('network_miscellaneous_details' not in concat_meta):
            concat_meta['network_miscellaneous_details'] = comment

    return concat_meta,units_dict
