#!/usr/bin/env python
# -*- coding: utf-8 -*-

#WRITTEN BY DENE BOWDALO

###------------------------------------------------------------------------------------###
###IMPORT MODULES
###------------------------------------------------------------------------------------###

import converter
import crossnetwork_processing_modules
import csv
import datetime
import dateutil
import instrument_processing_modules
import io
import numpy as np
import os
import pandas as pd
import re
import standardisation_modules

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###

def read_data(zipped_list):

    #unzip variables    
    station_n, ref, data_files, empty_external_metadata = zipped_list[0],zipped_list[1],zipped_list[2],zipped_list[3]

    print '%s -- READING (%s)'%(ref, station_n)
    
    #create site instances of metadata/data dicts
    site_data =     standardisation_modules.create_standardised_data_frame()
    site_metadata = standardisation_modules.create_standardised_metadata_frame()
    
    #sort files in date order
    data_file_years = [int(data_file.split('/')[-1].split('_')[3]) for data_file in data_files]
    data_files = np.array([x for _,x in sorted(zip(data_file_years,data_files))]) 

    #standard_resolution_dict (in minutes)
    standard_resolution_dict = {60:'H',1440:'D',10080:'W'}
    
    #load in dictionaries which map 3 different external metadata sources for EIONET which standardised metadata fields
    standard_external_meta_map, AIDE_D_external_meta_map, AIRBASE_external_meta_map = crossnetwork_processing_modules.EIONET_external_meta_mapper()

    #------------------------------------------------------------------------------------#
    #ITERATE THROUGH ALL YEARLY FILES FOR STATION
    #------------------------------------------------------------------------------------#    
    eoicode_data_extent = {}
    eoicode_metadata_extent = {}
    sampling_process_data_extent = {}
    sampling_process_metadata_extent = {}
    
    for file_i, data_file in enumerate(data_files):
        valid_file = True
    
        #----------------------------------------------------------------------------#
        #READ FILE
        #----------------------------------------------------------------------------#
    
        with io.open(data_file, 'r', encoding='utf-16') as f:
        
            for index,line in enumerate(f): 
            
                if valid_file == False:
                    break
            
                #get column names
                if index == 0:
                    column_names = line.split(',')
                    file_data = [[] for _ in xrange(len(column_names))]

                #get Data
                elif index >= 1:
                    line_split = line.split(',')
                    for column_name in column_names:
                        try:
                            file_data[column_names.index(column_name)].append(line_split[column_names.index(column_name)])   
                        except:
                            print 'FILE: %s IS NOT FORMATTED CORRECTED. SKIPPING'%(data_file)
                            valid_file = False
                            break
        
        if valid_file == True:
        
            #convert file_data to numpy record array
            file_data = np.rec.fromarrays(file_data, names=column_names)
            
            #check species codes inside file are the correct ones for the species processing
            file_species_codes = np.unique(file_data['AirPollutantCode'])
            if (int(file_species_codes[0].split('/')[-1]) != int(parameter_details['airbase_code'])) or (len(file_species_codes) > 1):
                print 'INCORRECT SPECIES CODE INSIDE FILE FOR SPECIES PROCESSING!'
                1+'a'
               
            #Group data in file by EoICode
            all_grouped_data = []
            unique_eoicodes = np.unique(file_data['AirQualityStationEoICode'])
            for unique_eoicode in unique_eoicodes:
                unique_eoicode = unique_eoicode.split('-')[-1] 
                grouped_data_by_eoicode = file_data[np.where(file_data['AirQualityStationEoICode'] == unique_eoicode)[0]]
            
                #Group data in file by SamplingProcess
                unique_sampling_processes = np.unique(file_data['SamplingProcess'])
                for unique_sampling_process in unique_sampling_processes:
                    all_grouped_data.append(grouped_data_by_eoicode[np.where(grouped_data_by_eoicode['SamplingProcess'] == unique_sampling_process)[0]])
            
            
            #iterate through grouped data
            for grouped_data in all_grouped_data:
                eoicode = grouped_data['AirQualityStationEoICode'][0]
                data_sampling_process = grouped_data['SamplingProcess'][0] 
        
                #create a dictionary with unique units associated with indices in the file for conversion 
                unique_measurement_units = np.unique(grouped_data['UnitOfMeasurement'])
                unit_dict = {}
                #make sure have some valid units, otherwise skip file
                for units in unique_measurement_units:
                    if units == '':
                        continue
                    try:
                        encoded_units = units.encode('ascii')
                    except UnicodeEncodeError:
                        print "Non-ascii micro sign in reported units - changing %s to ug/m3"%(units) 
                        encoded_units = 'ug/m3'
                    unit_dict[encoded_units] = np.where(grouped_data['UnitOfMeasurement'] == units)[0]
                if unit_dict == {}:
                    print 'FILE: %s REPORTS NO UNITS. SKIPPING'%(data_file)
                    continue
            
                #get external metadata specific for station & species (data is retrieved from 3 files)
        
                #ready key data/metadata variables that will use for making join between data and metadata
                data_sampling_point = grouped_data['SamplingPoint'][0]
                data_species_code = grouped_data['AirPollutantCode'][0]
                pollutant_AIDE_D_format = parameter_details['airbase_name']
                data_species_code_AIRBASE_format = data_species_code.split('/')[-1]
        
                #STANDARD EXTERNAL METADATA FILE              
                #first try making data/metadata join with standard external meta file using the "SamplingProcess"
                cut_external_metadata_standard = external_metadata[np.where(external_metadata['SamplingProcess'] == data_sampling_process)[0]]
        
                #if cut_external_metadata_standard is empty or not specific - next try doing join with "SamplingPoint"
                if len(cut_external_metadata_standard) == 0:
                     cut_external_metadata_standard = external_metadata[np.where(external_metadata['SamplingPoint'] == data_sampling_point)[0]]
                elif type(cut_external_metadata_standard) == np.core.records.recarray:
                    cut_external_metadata_standard = cut_external_metadata_standard[np.where(cut_external_metadata_standard['SamplingPoint'] == data_sampling_point)[0]]
        
                #if cut_external_metadata_standard is empty or not specific - next try doing join with "AirQualityStationEoICode" and "AirPollutantCode"
                if len(cut_external_metadata_standard) == 0:
                    cut_external_metadata_standard = external_metadata[np.intersect1d(np.where(external_metadata['AirQualityStationEoICode'] == eoicode)[0], np.where(external_metadata['AirPollutantCode'] == data_species_code)[0])]
                elif type(cut_external_metadata_standard) == np.core.records.recarray:
                    cut_external_metadata_standard = cut_external_metadata_standard[np.intersect1d(np.where(cut_external_metadata_standard['AirQualityStationEoICode'] == eoicode)[0], np.where(cut_external_metadata_standard['AirPollutantCode'] == data_species_code)[0])]
        
                #check if have specific standard metadata
                if (len(cut_external_metadata_standard) != 0) & (type(cut_external_metadata_standard) == np.core.records.recarray):
                    #if not specific, preferentially take first index
                    cut_external_metadata_standard = cut_external_metadata_standard[0]
        
    
                #AIDE D METADATA FILE
                #first try making data/metadata join with AIDE D external metadata file using the "SamplingProcess"
                cut_external_metadata_AIDE_D = external_metadata_AIDE_D[np.where(external_metadata_AIDE_D['SamplingProcess'] == data_sampling_process)[0]]
        
                #if cut_external_metadata_AIDE_D is empty or not specific - next try doing join with "SamplingPointId"
                if len(cut_external_metadata_AIDE_D) == 0:
                    cut_external_metadata_AIDE_D = external_metadata_AIDE_D[np.where(external_metadata_AIDE_D['SamplingPointId'] == data_sampling_point)[0]]
                elif type(cut_external_metadata_AIDE_D) == np.core.records.recarray:
                    cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[np.where(cut_external_metadata_AIDE_D['SamplingPointId'] == data_sampling_point)[0]]
        
                #if cut_external_metadata_AIDE_D is empty or not specific - next try doing join with "EoICode" and "Pollutant"
                if len(cut_external_metadata_AIDE_D) == 0:
                    cut_external_metadata_AIDE_D = external_metadata_AIDE_D[np.intersect1d(np.where(external_metadata_AIDE_D['EoICode'] == eoicode)[0], np.where(external_metadata_AIDE_D['Pollutant'] == pollutant_AIDE_D_format)[0])]
                elif type(cut_external_metadata_AIDE_D) == np.core.records.recarray:
                    cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[np.intersect1d(np.where(cut_external_metadata_AIDE_D['EoICode'] == eoicode)[0], np.where(cut_external_metadata_AIDE_D['Pollutant'] == pollutant_AIDE_D_format)[0])]
        
                #check if have specific AIDE D metadata
                if (len(cut_external_metadata_AIDE_D) != 0) & (type(cut_external_metadata_AIDE_D) == np.core.records.recarray):
                    #if not specific, preferentially take first index
                    cut_external_metadata_AIDE_D = cut_external_metadata_AIDE_D[0]

        
                #AIRBASE v8 METADATA FILE 
                #first try making data/metadata join with AIRBASE external metadata file using the "station_european_code" and "component_code"
                cut_external_metadata_AIRBASE = external_metadata_AIRBASE[np.intersect1d(np.where(external_metadata_AIRBASE['station_european_code'] == eoicode)[0], np.where(external_metadata_AIRBASE['component_code'] == data_species_code_AIRBASE_format)[0])]
           
                #if cut_external_metadata_AIRBASE is not specific - next try doing join taking into account measurement dates
                if (len(cut_external_metadata_AIRBASE) != 0) & (type(cut_external_metadata_AIRBASE) == np.ndarray): 
                    #select the metadata row with the latest "measurement_end_date"
                    #convert string metadata dates to datetime
                    #*first convert empty strings to a date much later in future (i.e. empty string = no end date)
                    measurement_end_dates = cut_external_metadata_AIRBASE['measurement_end_date']
                    measurement_end_dates[np.where(measurement_end_dates == '')[0]] = '3000-01-01'
                    measurement_end_dates = [datetime.datetime.strptime(date, '%Y-%m-%d').date() for date in measurement_end_dates]
                    #finally get index of latest end date and cut cut_external_metadata_AIRBASE
                    cut_external_metadata_AIRBASE = cut_external_metadata_AIRBASE[np.argmax(measurement_end_dates)]

                #print 'standard: ', cut_external_metadata_standard['Altitude']
                #print 'aide d: ', cut_external_metadata_AIDE_D['Altitude']
                #print 'airbase: ', cut_external_metadata_AIRBASE['station_altitude']
            
                #UNIFY 3 METADATA files information into standardised dict
                cut_unified_metadata = {}
                #first load in standard metadata variables
                if len(cut_external_metadata_standard) > 0:
                    for var in external_metadata.dtype.names:                    
                        if var in standard_external_meta_map.keys():
                            #get standard_name of variable
                            standard_var = standard_external_meta_map[var]
                            #get variable value
                            var_value = cut_external_metadata_standard[var]
                            #do not write if empty string, or 'unknown' or '-999'
                            if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                                cut_unified_metadata[standard_var] = var_value

                #next load in AIDE D metadata variables
                if len(cut_external_metadata_AIDE_D) > 0:
                    for var in external_metadata_AIDE_D.dtype.names:                    
                        if var in AIDE_D_external_meta_map.keys():
                            #get standard_name of variable
                            standard_var = AIDE_D_external_meta_map[var]
                            #do not proceed if already have variable written in cut_unified_metadata dict
                            if standard_var not in cut_unified_metadata.keys():
                                #get variable value
                                var_value = cut_external_metadata_AIDE_D[var]
                                #do not write if empty string, or 'unknown' or '-999'
                                if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                                    cut_unified_metadata[standard_var] = var_value
                    
                #next load in AIRBASE metadata variables
                if len(cut_external_metadata_AIRBASE) > 0:
                    for var in external_metadata_AIRBASE.dtype.names:                    
                        if var in AIRBASE_external_meta_map.keys():
                            #get standard_name of variable
                            standard_var = AIRBASE_external_meta_map[var]
                            #do not proceed if already have variable written in cut_unified_metadata dict
                            if standard_var not in cut_unified_metadata.keys():
                                #get variable value
                                var_value = cut_external_metadata_AIRBASE[var]
                                #do not write if empty string, or 'unknown' or '-999'
                                if (var_value != '') & (var_value != 'unknown') & (var_value != '-999'): 
                                    cut_unified_metadata[standard_var] = var_value

                #Check have some valid metadata
                if len(cut_unified_metadata) == 0:
                    print 'NO VALID EXTERNAL METADATA FOUND FOR %s IN FILE: %s. SKIPPING.'%(data_file,eoicode)
                    continue
    
                #decode UTF coded metadata to ascii
                for var_name in cut_unified_metadata.keys():
                    cut_unified_metadata[var_name] = crossnetwork_processing_modules.remove_accents(cut_unified_metadata[var_name])
            
                #------------------------------------------------------------------------#
                #PARSE TIME/DATA
                #------------------------------------------------------------------------#
          
                #parse start datetime - "DatetimeBegin"  (Defines the start time: yyyy-mm-dd hh:mm:ss Z, of the measurement -- includes timezone)
                #first parse into datetime (taking into account timezone information), convert to UTC datetime, then round to nearest minute
                start_dt = [crossnetwork_processing_modules.round_to_minute(dateutil.parser.parse(date).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in grouped_data['DatetimeBegin']]

                #parse end datetime - "DatetimeEnd" (Defines the end time: yyyy-mm-dd hh:mm:ss Z, of the measurement -- includes timezone)
                #first parse into datetime (taking into account timezone information), convert to UTC datetime, then round to nearest minute
                end_dt = [crossnetwork_processing_modules.round_to_minute(dateutil.parser.parse(date).astimezone(dateutil.tz.tzutc())).replace(tzinfo=None) for date in grouped_data['DatetimeEnd']]

                #convert each start measurement datetime to minutes from reference time
                frac_time = [crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(dt) for dt in start_dt]

                #for first time of file, save it out for meta_update_stamp as minutes relative to reference time
                site_metadata['meta_update_stamp']['value'].append(frac_time[0])
        
                #parse flags    
                #load standardised flags dict
                standard_flags = standardisation_modules.standard_flags()
                #get flags from data file - Validity
                data_flags = grouped_data['Validity']
                #get standard flag information
                flags = [standard_flags[flag]['flag_type'] for flag in data_flags]
                qa_flags_science = [standard_flags[flag]['self_decreed_validity'] for flag in data_flags]
                qa_flags_network = [standard_flags[flag]['network_decreed_validity'] for flag in data_flags]
                #get verification information
                verification_data = grouped_data['Verification']
        
                #process qa information associated with flags
                qa = []
                for ii in range(len(flags)):
        
                    #write preliminary data flags if "Verification" variable informs that data is preliminary 
                    #1 = Verified, 2 = Preliminary verified, 3 = Not verified
                    if (verification_data[ii] == '2') or (verification_data[ii] == '3'):
                        flags[ii] = flags[ii]+'P'
                        qa_flags_science[ii] = 'I'
                        qa_flag_network[ii] = 'V'
            
                    qa_str = ''
                    #append qa flag if standard flag is decreed scientifically invalid
                    if qa_flags_science[ii] == 'I':
                        qa_str = qa_str+standardisation_modules.get_standard_qa_flag('Invalid Data Provider Flags - Scientifically Decreed')
                    #append qa flag if standard flag is network decreed invalid
                    if qa_flags_network[ii] == 'I':
                        qa_str = qa_str+standardisation_modules.get_standard_qa_flag('Invalid Data Provider Flags - Network Decreed')
                    qa.append(qa_str)
        
                #get timedeltas between start and end of measurement window
                temporal_resolutions = []
                for start, end in zip(start_dt,end_dt):
                    dt_window = end - start
                    #convert hours to minutes
                    dt_window_minutes = dt_window.total_seconds()/60.
    
                    try:
                        temporal_resolution_code = standard_resolution_dict[dt_window_minutes]
                    except KeyError:
                        temporal_resolution_code = str(int(dt_window_minutes))
                    temporal_resolutions.append(temporal_resolution_code)
        
                #parse unique reported measurement unit (take care to convert microgram symbol to u - for unit convert module)
                #can be instances where have multiple units in a file, in this case take the modal occurring unit
                unit_n = -1
                for unit_name in unit_dict.keys():
                    if len(unit_dict[unit_name]) > unit_n:
                        reported_units = unit_name                
                #convert raw reported units given to standard name
                conv_obj = converter.convert_units(reported_units, reported_units, 1, measured_species = parameter_details['chemical_formula'])
                conv_obj.do_conversion()
                site_metadata['measuring_instrument_reported_units']['value'].append(conv_obj.output_standard_units)

                #parse measurements
                measurements = []
                #measurement values may need converting (i.e. mass density to mixing ratio)
        
                #gas volume is not provided by EIONET and if units need converting, this conversion is done using assumed network standard pressure and temperatures - write qa data flag to inform of this assumption
                #measurement temperature and pressure values are never provided by EIONET so are always assumed to be the network standard pressure and temperatures
                #these are defined as for 293 K temperature, and 1013 hPa for pressure here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32008L0050&from=en and here: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32004L0107&from=EN 
                site_metadata['measuring_instrument_volume_standard_temperature']['value'].append('293.0')
                site_metadata['measuring_instrument_volume_standard_pressure']['value'].append('1013.0')
        
                #this flag below alerts for writing of qa flag later
                write_volume_assumption_flag = False
        
                #check if reported standard units, match required standard units
                #take care to deal with multiple units in file
                order_n = []
                for reported_units in unit_dict.keys():

                    #convert raw reported units given to standard name
                    conv_obj = converter.convert_units(reported_units, reported_units, 1, measured_species = parameter_details['chemical_formula'])
                    conv_obj.do_conversion()
                    standard_reported_units = conv_obj.output_standard_units
        
                    #cut concentration data to be that just associated with units
                    cut_concentrations = grouped_data['Concentration'][unit_dict[reported_units]]
            
                    if standard_reported_units != site_data[parameter_details['bsc_parameter_name']]['units']:
                        #mass density to mixing ratio?
                        if ('m-3' in standard_reported_units) & ('mol-1' in site_data[parameter_details['bsc_parameter_name']]['units']):
            
                            #set flag to instruct writing of flag that am assuming gas volume in converting units 
                            write_volume_assumption_flag = True
                
                            #get conversion factor
                            input_units = {'temperature':'K', 'pressure':'hPa', 'molar_mass':'kg mol-1', 'mass_density':standard_reported_units}
                            input_vals = {'temperature':site_metadata['measuring_instrument_volume_standard_temperature']['value'][-1], 'pressure':site_metadata['measuring_instrument_volume_standard_pressure']['value'][-1], 'molar_mass':crossnetwork_processing_modules.get_molecular_mass(parameter_details['chemical_formula']), 'mass_density':1.0}
                            conv_obj = converter.convert_units(input_units,site_data[parameter_details['bsc_parameter_name']]['units'],input_vals,measured_species = parameter_details['chemical_formula'],conversion_input_quantity = 'mass_density')
            
                        #otherwise simple unit conversion, i.e. pptv to ppbv or mass density to mass density
                        else:
                            conv_obj = converter.convert_units(standard_reported_units,site_data[parameter_details['bsc_parameter_name']]['units'],1.0, measured_species = parameter_details['chemical_formula'])
    
                        #do conversion
                        conv_obj.do_conversion()
        
                        #process through each measurement and convert value using derived conversion factor
                        for n, meas in enumerate(cut_concentrations):
                            actual_n = unit_dict[reported_units][n]
                            order_n.append(actual_n)
                            #only do conversion if measurement value is above 0 and measurement is not missing (determined using standard flag)
                            if (float(meas) > 0.0) & ('M' not in flags[actual_n]):
                                measurements.append(float(meas)*conv_obj.conversion_factor)
                            else:
                                measurements.append(meas)
        
                    #otherwise do nothing to measured values
                    else:
                        for n, meas in enumerate(cut_concentrations):
                            order_n.append(unit_dict[reported_units][n])
                            measurements.append(meas)
        
                #resort measurements by original order
                measurements = [x for _,x in sorted(zip(order_n,measurements))]
        
                #filter frac_time, temporal_resolutions, flags and qa by order_n (in case there are some measurements with unknown units in file
                order_n = sorted(order_n)
                list_of_vars = [frac_time, temporal_resolutions, flags, qa]
                #for var in list_of_vars:
                frac_time = [frac_time[x] for x in order_n]
                temporal_resolutions = [temporal_resolutions[x] for x in order_n]
                flags = [flags[x] for x in order_n]
                qa = [qa[x] for x in order_n]
        
                #------------------------------------------------------------------------#
                #PARSE METADATA
                #------------------------------------------------------------------------#

                #GHOST version
                site_metadata['GHOST_version']['value'].append(run_details['ghost_version'])

                #add station_reference
                site_metadata['station_reference']['value'].append(eoicode)
        
                #get length of current metadata lists written 
                n_meta = len(site_metadata['station_reference']['value'])
        
                #write known metadata  
                site_metadata['network']['value'].append('EIONET')
                site_metadata['data_level']['value'].append('E1a')

                #define list of variable names that require no processing that can be simply added to site_metadata
                metadata_to_add = ['latitude','longitude','altitude','sampling_height','distance_to_kerb','distance_to_building',
                                  'distance_to_junction','distance_to_source','street_type','street_width','daytime_traffic_speed','projection',
                                  'daily_passing_vehicles','station_name','city','representative_radius','principal_investigator_institution',
                                  'principal_investigator_email_address','contact_institution','contact_email_address']

                #iterate through all metadata variable names and put values into standard dictionary
                for variable_name in metadata_to_add:
                    #check variable is in unified metadata dict
                    if variable_name in cut_unified_metadata.keys():
                        site_metadata[variable_name]['value'].append(cut_unified_metadata[variable_name].replace('"',''))
    
                #parse associated networks 
                #first add AIRBASE with EoICode
                associated_network_string = 'AIRBASE:%s'%(eoicode)
                #second add EBAS, if have 'emep_station' variable and is 'yes'
                if 'emep_station' in cut_unified_metadata.keys():
                    if cut_unified_metadata['emep_station'] == 'yes':
                        #add EBAS with EoICode
                        ebas_string = 'EBAS:%s'%(eoicode)
                        associated_network_string = '%s;%s'%(associated_network_string, ebas_string)
                #finally add local network with local code
                if ('local_network_code' in cut_unified_metadata.keys()) & ('local_reference' in cut_unified_metadata.keys()):
                    local_string = '%s:%s'%(cut_unified_metadata['local_network_code'],cut_unified_metadata['local_reference'])
                    associated_network_string = '%s;%s'%(associated_network_string, local_string)
                site_metadata['associated_networks']['value'].append(associated_network_string)
    
                #parse geodatic datum - take from projection information (certain projections use specific datums - i.e. 'EPSG:4979' uses 'WSG84' datum)
                if 'projection' in cut_unified_metadata.keys():
                    if cut_unified_metadata['projection'] == 'EPSG:4979':
                       site_metadata['geodetic_datum']['value'].append('WSG84') 
                    else:
                        print 'unknown projection'
                        print cut_unified_metadata['projection']
                        1+'a'
                
                #parse population (need to check to remove non-numeric characters)
                if 'population' in cut_unified_metadata.keys():
                    population = cut_unified_metadata['population']
                    population = re.sub("[^0-9]", "", population)
                    site_metadata['population']['value'].append(population) 
        
                #parse 'principal_investigator_name' (concatenate title, first name and last name, if given)
                principal_investigator_name = ''
                if 'principal_investigator_title' in cut_unified_metadata.keys():
                    principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_title']
                if 'principal_investigator_first_name' in cut_unified_metadata.keys():
                    if principal_investigator_name == '':
                        principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_first_name']
                    else:
                        principal_investigator_name = '%s %s'%(principal_investigator_name, cut_unified_metadata['principal_investigator_first_name'])
                if 'principal_investigator_last_name' in cut_unified_metadata.keys():
                    if principal_investigator_name == '':
                        principal_investigator_name = principal_investigator_name + cut_unified_metadata['principal_investigator_last_name']
                    else:
                        principal_investigator_name = '%s %s'%(principal_investigator_name, cut_unified_metadata['principal_investigator_last_name'])
                if principal_investigator_name != '':
                    site_metadata['principal_investigator_name']['value'].append(principal_investigator_name) 
         
                #parse 'contact_name' (concatenate title, first name and last name, if given)
                contact_name = ''
                if 'contact_title' in cut_unified_metadata.keys():
                    contact_name = contact_name + cut_unified_metadata['contact_title']
                if 'contact_first_name' in cut_unified_metadata.keys():
                    if contact_name == '':
                        contact_name = contact_name + cut_unified_metadata['contact_first_name']
                    else:
                        contact_name = '%s %s'%(contact_name, cut_unified_metadata['contact_first_name'])
                if 'contact_last_name' in cut_unified_metadata.keys():
                    if contact_name == '':
                        contact_name = contact_name + cut_unified_metadata['contact_last_name']
                    else:
                        contact_name = '%s %s'%(contact_name, cut_unified_metadata['contact_last_name'])
                if contact_name != '':
                    site_metadata['contact_name']['value'].append(contact_name) 
        
                #parse data timezone (in hours from UTC)
                data_timezone = (dateutil.parser.parse(grouped_data['DatetimeBegin'][0]).utcoffset().seconds)/60.0/60.0
                site_metadata['data_timezone']['value'].append(data_timezone) 
        
                #parse miscellaneous details
                #deal with miscellaneous sampling details
                if 'sample_location_miscellaneous' in cut_unified_metadata.keys():
                    miscellaneous_sampling = 'Sampling location: %s'%(cut_unified_metadata['sample_location_miscellaneous'])
                    site_metadata['network_sampling_details']['value'].append(miscellaneous_sampling) 
        
                #deal with miscellaneous sampling inlet information
                if 'length_sampling_line' in cut_unified_metadata.keys():
                    inlet_info = 'Length: %s'%(cut_unified_metadata['length_sampling_line'])
                    site_metadata['measuring_instrument_inlet_information']['value'].append(inlet_info)
                        
                #deal with miscellaneous maintenance details
                miscellaneous_maintenance = ''
                if 'calibration_method' in cut_unified_metadata.keys():
                    string_to_add = 'Calibration methodology: %s'%(cut_unified_metadata['calibration_method'])
                    miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
                if 'calibration_frequency' in cut_unified_metadata.keys():
                    string_to_add = 'Calibration frequency: %s'%(cut_unified_metadata['calibration_frequency'])
                    if miscellaneous_maintenance == '':
                        miscellaneous_maintenance = miscellaneous_maintenance + string_to_add
                    else:
                        miscellaneous_maintenance = '%s;%s'%(miscellaneous_maintenance, string_to_add)            
                if miscellaneous_maintenance != '':
                    site_metadata['network_maintenance_details']['value'].append(miscellaneous_maintenance)
        
                #deal with other miscellaneous details
                if 'station_description' in cut_unified_metadata.keys():
                    station_description = 'Station description: %s'%(cut_unified_metadata['station_description'])
                    site_metadata['network_miscellaneous_details']['value'].append(station_description)
        
                #parse country code
                country_code = grouped_data['Countrycode'][0]
                site_metadata['country']['value'].append(crossnetwork_processing_modules.country_ISO3166_to_name(country_code)) 

                #parse measurement information (from external metadata) - use all possible variables of: 'sampling_equipment', 'measurement_equipment', 'analytical_technique', 'measurement_method', 'measurement_type', 'measurement_automatic'
                if 'sampling_equipment' in cut_unified_metadata.keys():
                    sampling_equipment = cut_unified_metadata['sampling_equipment']
                else:
                    sampling_equipment = ''
                if 'measurement_equipment' in cut_unified_metadata.keys():
                    #remove 'unnessary long string from variable
                    measurement_equipment = cut_unified_metadata['measurement_equipment'].split('measurementequipment/')[-1]
                else:
                    measurement_equipment = ''
                if 'analytical_technique' in cut_unified_metadata.keys():
                    analytical_technique = cut_unified_metadata['analytical_technique']
                else:
                    analytical_technique = ''
                if 'measurement_method' in cut_unified_metadata.keys():
                    #remove 'unnessary long string from variable
                    measurement_method = cut_unified_metadata['measurement_method'].split('measurementmethod/')[-1]
                else:
                    measurement_method = ''
                if 'measurement_type' in cut_unified_metadata.keys():
                    measurement_type = cut_unified_metadata['measurement_type']
                else:
                    measurement_type = ''
                #only parse 'measurement_automatic' if don't have 'measurement_type'
                if ('measurement_automatic' in cut_unified_metadata.keys()) & (measurement_type == ''):
                    measurement_automatic = cut_unified_metadata['measurement_automatic']
                    if measurement_automatic == 'yes':
                        measurement_type = 'automatic'
                    elif measurement_automatic == 'no':
                        measurement_type = 'not-automatic'
                else:
                    measurement_automatic = ''
                #concatenate measurement information together
                measurement_information = '%s %s %s %s %s'%(sampling_equipment, measurement_equipment, analytical_technique, measurement_method, measurement_type)
                #strip leading/trailing white spaces, and change multiple white spaces to be singular
                measurement_information = measurement_information.strip()
                measurement_information = ' '.join(measurement_information.split())
                measurment_standard_dict, measurement_method_qa_flags = instrument_processing_modules.network_instrument_mapping(measurement_information,site_metadata['meta_update_stamp']['value'][-1])
                #if have correctly mapped standardised measurement information, write it out 
                if measurement_method_qa_flags['un_mapped_method_flag'] == False:
                    site_metadata['primary_sampling_instrument_name']['value'].append(measurment_standard_dict['primary_sampling_instrument'])
                    site_metadata['primary_sampling_type']['value'].append(measurment_standard_dict['primary_sampling_type'])
                    site_metadata['primary_sampling_instrument_documented_flow_rate']['value'].append(measurment_standard_dict['primary_sampling_instrument_documented_flow_rate'])
                    site_metadata['primary_sampling_instrument_reported_flow_rate']['value'].append(measurment_standard_dict['primary_sampling_instrument_reported_flow_rate'])
                    site_metadata['primary_sampling_process_details']['value'].append(measurment_standard_dict['primary_sampling_process_details'])
                    site_metadata['primary_sampling_instrument_manual']['value'].append(measurment_standard_dict['primary_sampling_instrument_manual_name'])
                    site_metadata['primary_sampling_further_details']['value'].append(measurment_standard_dict['primary_sampling_instrument_further_details'])
                    site_metadata['sample_preparation_methodology_names']['value'].append(measurment_standard_dict['sample_preparation'])
                    site_metadata['sample_preparation_methodology_types']['value'].append(measurment_standard_dict['preparation_type'])
                    site_metadata['sample_preparation_process_details']['value'].append(measurment_standard_dict['preparation_process_details'])
                    site_metadata['sample_preparation_further_details']['value'].append(measurment_standard_dict['preparation_further_details'])
                    site_metadata['measuring_instrument_name']['value'].append(measurment_standard_dict['measuring_instrument'])
                    site_metadata['measurement_methodology']['value'].append(measurment_standard_dict['methodology'])
                    site_metadata['measuring_instrument_sampling_type']['value'].append(measurment_standard_dict['measuring_instrument_sampling_type'])
                    site_metadata['measuring_instrument_documented_flow_rate']['value'].append(measurment_standard_dict['measuring_instrument_documented_flow_rate'])
                    site_metadata['measuring_instrument_reported_flow_rate']['value'].append(measurment_standard_dict['measuring_instrument_reported_flow_rate'])
                    site_metadata['measuring_instrument_process_details']['value'].append(measurment_standard_dict['measuring_instrument_process_details'])
                    site_metadata['measuring_instrument_manual_name']['value'].append(measurment_standard_dict['measuring_instrument_manual_name'])
                    site_metadata['measuring_instrument_further_details']['value'].append(measurment_standard_dict['measuring_instrument_further_details'])
                    site_metadata['measuring_instrument_reported_lower_limit_of_detection']['value'].append(measurment_standard_dict['reported_lower_limit_of_detection'])
                    site_metadata['measuring_instrument_documented_lower_limit_of_detection']['value'].append(measurment_standard_dict['documented_lower_limit_of_detection'])
                    site_metadata['measuring_instrument_reported_upper_limit_of_detection']['value'].append(measurment_standard_dict['reported_upper_limit_of_detection'])
                    site_metadata['measuring_instrument_documented_upper_limit_of_detection']['value'].append(measurment_standard_dict['documented_upper_limit_of_detection'])
                    site_metadata['measuring_instrument_reported_uncertainty']['value'].append(measurment_standard_dict['reported_uncertainty'])
                    site_metadata['measuring_instrument_documented_uncertainty']['value'].append(measurment_standard_dict['documented_uncertainty'])
                    site_metadata['measuring_instrument_reported_accuracy']['value'].append(measurment_standard_dict['reported_accuracy'])
                    site_metadata['measuring_instrument_documented_accuracy']['value'].append(measurment_standard_dict['documented_accuracy'])
                    site_metadata['measuring_instrument_reported_precision']['value'].append(measurment_standard_dict['reported_precision'])
                    site_metadata['measuring_instrument_documented_precision']['value'].append(measurment_standard_dict['documented_precision'])
                    site_metadata['measuring_instrument_reported_zero_drift']['value'].append(measurment_standard_dict['reported_zero_drift'])
                    site_metadata['measuring_instrument_documented_zero_drift']['value'].append(measurment_standard_dict['documented_zero_drift'])
                    site_metadata['measuring_instrument_reported_span_drift']['value'].append(measurment_standard_dict['reported_span_drift'])
                    site_metadata['measuring_instrument_documented_span_drift']['value'].append(measurment_standard_dict['documented_span_drift'])
                    site_metadata['measuring_instrument_reported_measurement_resolution']['value'].append(measurment_standard_dict['reported_resolution'])
                    site_metadata['measuring_instrument_documented_measurement_resolution']['value'].append(measurment_standard_dict['documented_resolution'])
                    site_metadata['measuring_instrument_reported_absorption_cross_section']['value'].append(measurment_standard_dict['reported_absorption_cross_section'])
                    site_metadata['measuring_instrument_documented_absorption_cross_section']['value'].append(measurment_standard_dict['documented_absorption_cross_section'])
    
    
                #parse station classifications (from external metadata) - use all possible variables of: 'area_classification', 'station_classification', 'main_emission_source', 'land_use', 'area_classification_rural_subdivision', 'representative_radius'
                if 'area_classification' in cut_unified_metadata.keys():
                    area_classification = cut_unified_metadata['area_classification']
                else:
                    area_classification = ''
                if 'station_classification' in cut_unified_metadata.keys():
                    station_classification = cut_unified_metadata['station_classification']
                else:
                    station_classification = ''
                if 'main_emission_source' in cut_unified_metadata.keys():
                    main_emission_source = cut_unified_metadata['main_emission_source']
                else:
                    main_emission_source = ''
                if 'land_use' in cut_unified_metadata.keys():
                    land_use = cut_unified_metadata['land_use']
                else:
                    land_use = ''
                if 'area_classification_rural_subdivision' in cut_unified_metadata.keys():
                    area_classification_rural_subdivision = cut_unified_metadata['area_classification_rural_subdivision']
                else:
                    area_classification_rural_subdivision = ''   
                if 'representative_radius' in cut_unified_metadata.keys():
                    representative_radius = cut_unified_metadata['representative_radius']
                else:
                    representative_radius = ''   
                native_classifications = {'area_classification':area_classification, 'station_classification':station_classification, 'main_emission_source':main_emission_source, 'land_use':land_use, 'area_classification_rural_subdivision':area_classification_rural_subdivision, 'representative_radius':representative_radius}
                area_classification,station_classification,main_emission_source,land_use,terrain,measurement_scale = standardisation_modules.mapping_network_classifications(native_classifications)
                site_metadata['area_classification']['value'].append(area_classification)
                site_metadata['station_classification']['value'].append(station_classification)
                site_metadata['main_emission_source']['value'].append(main_emission_source)
                site_metadata['land_use']['value'].append(land_use)
                site_metadata['terrain']['value'].append(terrain)
                site_metadata['measurement_scale']['value'].append(measurement_scale)
        
                #save data_download_stamp --> rounded to nearest minute
                data_download_datetime = datetime.datetime.strptime(time.ctime(os.path.getmtime(data_file)), "%a %b %d %H:%M:%S %Y")
                data_download_datetime = crossnetwork_processing_modules.round_to_minute(data_download_datetime)
                site_metadata['data_download_stamp']['value'].append(crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(data_download_datetime))
    
                #iterate through metadata variables, if any columns have not been appended to add a NaN
                #check number of refs in list
                for column_header in site_metadata.keys():
                    n_column = len(site_metadata[column_header]['value'])
                    #if length of column is less than number of references, then append NaN
                    if n_column < n_meta:
                        site_metadata[column_header]['value'].append(np.NaN)
                
                
                #get station_timezone (at reference 2000-01-01 00:00 UTC) & wmo_region (only if have valid latitude and longitude)
                if (pd.isnull(site_metadata['latitude']['value'][-1]) == False) & (pd.isnull(site_metadata['longitude']['value'][-1]) == False):
                    dt_ref = datetime.datetime(2000,01,01,00,00)
                    #call to get timezone is costly, therefore take shortcut, if have already processed this for the station, and lat and lon have not changed, copy from previous iteration, otherwise process it
                    if len(site_metadata['station_timezone']['value']) != 1:
                        if (site_metadata['latitude']['value'][-1] == site_metadata['latitude']['value'][-2]) & (site_metadata['longitude']['value'][-1] == site_metadata['longitude']['value'][-2]):
                            site_metadata['station_timezone']['value'][-1] = site_metadata['station_timezone']['value'][-2]
                        else:
                            pytz_str, site_metadata['station_timezone']['value'][-1] = crossnetwork_processing_modules.local_offset_from_utc(dt_ref,longitude=site_metadata['longitude']['value'][-1],latitude=site_metadata['latitude']['value'][-1])
                    else: 
                        pytz_str, site_metadata['station_timezone']['value'][-1] = crossnetwork_processing_modules.local_offset_from_utc(dt_ref,longitude=site_metadata['longitude']['value'][-1],latitude=site_metadata['latitude']['value'][-1])
                    site_metadata['wmo_region']['value'][-1] = standardisation_modules.get_wmo_region(longitude=site_metadata['longitude']['value'][-1],latitude=site_metadata['latitude']['value'][-1])
    
                #----------------------------------------------------------------------------#    
                #QA
                #----------------------------------------------------------------------------#
                #write instrument/methodology qa flags, using standard qa flags
                #dictionary maps variable names returned from methodology processing to standardised QA codes
                qa_mapping_dict = {'un_mapped_method_flag':'Methodology Not Mapped',
                                   'primary_sampling_assumption_flag':'Assumed Primary Sampling',
                                   'preparation_assumption_flag':'Assumed Sample Preparation',
                                   'measuring_assumption_flag':'Assumed Measurement Methodology',
                                   'unknown_primary_sampling_instrument_flag':'Unknown Primary Sampling Instrument',
                                   'unknown_measuring_instrument_flag':'Unknown Measuring Instrument',
                                   'erroneous_primary_sampling_flag':'Erroneous Primary Sampling',
                                   'erroneous_sample_preparation_flag':'Erroneous Sample Preparation',
                                   'erroneous_measurement_methodology_flag':'Erroneous Measurement Methodology',
                                   'not_qc_accepted_method_flag':'Invalid QA Measurement Methodology'}
                       
                #iterate through instrument/methodology flags and write standard qa flag if required
                for variable_name in qa_mapping_dict.keys():
                    if measurement_method_qa_flags[variable_name] == True:
                        for x, qa_str in enumerate(qa):
                            qa[x] = qa_str+standardisation_modules.get_standard_qa_flag(qa_mapping_dict[variable_name])
    
                #write assumed volume of gas qa flags (inform have assumed volume of gas for converting units)
                if write_volume_assumption_flag == True:
                    for x, qa_str in enumerate(qa):
                        qa[x] = qa_str+standardisation_modules.get_standard_qa_flag('Assumed Gas Volume')
                
                #----------------------------------------------------------------------------#
                #WRITE OUT DATA
                #----------------------------------------------------------------------------#

                #No LODs or measurement uncertainties per measurement - append list of NaNs    
                site_data['time']['value'].extend(frac_time)
                site_data[parameter_details['bsc_parameter_name']]['value'].extend(measurements)
                site_data['temporal_resolution']['value'].extend(temporal_resolutions)   
                site_data['reported_lower_limit_of_detection_per_measurement']['value'].extend([np.NaN]*len(frac_time))
                site_data['reported_upper_limit_of_detection_per_measurement']['value'].extend([np.NaN]*len(frac_time))
                site_data['reported_uncertainty_per_measurement']['value'].extend([np.NaN]*len(frac_time))
                site_data['derived_uncertainty_per_measurement']['value'].extend([np.NaN]*len(frac_time))
                site_data['flag']['value'].extend(flags)
                site_data['qa']['value'].extend(qa)
                
                #write information to dicts to inform of data/metadata extents of unique eoicodes and sampling processes
                if eoicode not in eoicode_data_extent.keys():
                    eoicode_data_extent[eoicode]     = range(len(site_data['time']['value'])-len(frac_time), len(frac_time))
                    eoicode_metadata_extent[eoicode] = [len(site_metadata['station_reference']['value'])-1]
                else:
                    eoicode_data_extent[eoicode]     = eoicode_data_extent[eoicode].extend(range(len(site_data['time']['value'])-len(frac_time), len(frac_time)))
                    eoicode_metadata_extent[eoicode] = eoicode_metadata_extent[eoicode].extend([len(site_metadata['station_reference']['value'])-1])
                    
                if data_sampling_process not in sampling_process_data_extent.keys():
                    sampling_process_data_extent[data_sampling_process]     = range(len(site_data['time']['value'])-len(frac_time), len(frac_time))
                    sampling_process_metadata_extent[data_sampling_process] = [len(site_metadata['station_reference']['value'])-1]
                else:
                    sampling_process_data_extent[data_sampling_process]     = sampling_process_data_extent[data_sampling_process].extend(range(len(site_data['time']['value'])-len(frac_time), len(frac_time)))
                    sampling_process_metadata_extent[data_sampling_process] = sampling_process_metadata_extent[data_sampling_process].extend([len(site_metadata['station_reference']['value'])-1])
    
    
    #only write out if have some data (make sure)
    if len(site_data['time']['value']) == 0:
        print '%s -- HAVE NO VALID DATA FOR STATION, NOT PROCEEDING TO WRITE TO PRIMARY .npz'%(ref)
    else:
        cut_for_eoicode = False
        cut_for_sampling_process = False
        
        #if there is more than 1 eoicode used across files, only keep data/metadata with eoicode modally used (i.e. number of different observations)
        if len(eoicode_data_extent.keys()) > 1:
            #find which nested list has the most data observations
            modal_eoicode, modal_eiocode_data_indices = max(eoicode_data_extent.items(), key = lambda x: len(set(x[1])))
            modal_eoicode_metadata_indices = eoicode_metadata_extent[modal_eoicode]
            cut_for_eoicode = True
            print '%s -- MULTIPLE EOICODES ACROSS DATA FILES. KEEPING MODALLY OCCURRING EOICODE: %s'%(ref, modal_eoicode)
            
        #if there is more than 1 sampling process used across files, only keep data/metadata with sampling_process modally used 
        if len(sampling_process_data_extent.keys()) > 1:
            #find which nested list has the most data observations
            modal_sampling_process, modal_sampling_process_data_indices = max(sampling_process_data_extent.items(), key = lambda x: len(set(x[1])))
            modal_sampling_process_metadata_indices = sampling_process_metadata_extent[modal_sampling_process]
            cut_for_sampling_process = True
            print '%s -- MULTIPLE SAMPLING PROCESSES ACROSS DATA FILES. KEEPING MODALLY OCCURRING SAMPLING PROCESS: %s'%(ref, modal_sampling_process)
    
        #if are cutting data/metadata for modal eoicode and sampling process, find intersection of relevant indices first
        if (cut_for_eoicode == True) & (cut_for_sampling_process == True):
            intersect_modal_data_indices = np.intersect1d(modal_eiocode_data_indices, modal_sampling_process_data_indices)
            intersect_modal_metadata_indices = np.intersect1d(modal_eiocode_metadata_indices, modal_sampling_process_metadata_indices)
            site_data = [site_data[x] for x in intersect_modal_data_indices]
            site_metadata = [site_metadata[x] for x in intersect_modal_metadata_indices]
        #just cutting for modal eoicode    
        elif cut_for_eoicode == True:
            site_data = [site_data[x] for x in modal_eiocode_data_indices]
            site_metadata = [site_metadata[x] for x in modal_eoicode_metadata_indices]
        #just cutting for modal sampling process   
        elif cut_for_sampling_process == True:
            site_data = [site_data[x] for x in modal_sampling_process_data_indices]
            site_metadata = [site_metadata[x] for x in modal_sampling_process_metadata_indices]
                
        #do post processing
        crossnetwork_processing_modules.post_process_station_data(site_metadata,site_data,ref)
         
    return 
                
            