import bisect
from collections import Counter
import copy
import datetime
import glob
import itertools
import time

import numpy as np
import pandas as pd
import scipy.stats as ss

from crossnetwork_processing_functions import create_GHOST_stage_logfile, cut_data_array, read_from_numpy, determine_key_changing_metadata, merge_lists_sharing_common_elements, redo_standardisations, normalise_by_max, weighted_mode, write_station_specific_numpy
from GHOST_standards import standard_measurement_methodologies, standard_QA_name_to_QA_code, metadata_modification_field_to_warning_name
import standardisation_mapping_functions
from time_functions import datetime_to_fractional_minutes_from_reference, fractional_minutes_from_reference_to_datetime


class StationConcatenate:

    '''Class to concatenate parsed network files per station.'''

    def __init__(self, station_reference, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- JOIN STATION SPECIFIC MEASUREMENTS\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #join station measurement files
        self.join_station_measurements()

        #do quick sanity check to see if time has been processed correctly.
        #that check is simply to see if the time array is ordered correctly, i.e. all in ascending order?
        #if not ordered correctly, return caught error logfile (2)
        #self.run_details['log_file_str'] += 'FIRST/LAST TIME: {}:{}\n'.format(self.existing_data['time'][0],self.existing_data['time'][-1]) 
        if not all(self.existing_data['time'] == np.sort(self.existing_data['time'])):
            self.run_details['log_file_str'] += 'TIME COLUMN IS NOT IN ASCENDING ORDER\n'
            vals = np.invert(self.existing_data['time'] == np.sort(self.existing_data['time']))
            self.run_details['log_file_str'] += '{} {}\n'.format(self.existing_data['time'][vals], np.sort(self.existing_data['time'])[vals])       
            self.run_details['log_file_str'] += '{}\n'.format(len(self.existing_data['time']))     

            create_GHOST_stage_logfile(2,self.run_details)

        #process manually compiled metadata for station (if exists)
        modified_standard_fields = self.process_manually_compiled_metadata()
        #update standardisations for modified metadata fields (if any)
        if len(modified_standard_fields) > 0:
            self.existing_metadata = redo_standardisations(self.existing_metadata, self.parameter_details, modified_standard_fields)

        #handle measurement position accuracy issues for longitude/latitude fields
        self.resolve_measurement_position_accuracy(measurement_position_field_names=['longitude','latitude'])

        #fill gapped longitude values through column (by dependent latitude)
        self.handle_gapped_key_metadata(chosen_groups={'longitude':['latitude']})
        #fill gapped latitude values through column (by dependent longitude)
        self.handle_gapped_key_metadata(chosen_groups={'latitude':['longitude']})
        #fill gapped longitude/latitude values through column (non-dependent)
        self.handle_gapped_key_metadata(chosen_groups={'longitude':[],'latitude':[]})

        #perform a check on the interdependency between the xy measurement position
        self.check_dependent_xy_position()

        #before doing any manipulation of altitude fields, if have all 3 altitude fields check they sum correctly
        #if not, reset sampling_height/measurement_altitude fields to be NaN
        self.set_altitude_field('altitude',check_validity='reset')

        #handle measurement position accuracy issues for altitude/sampling_height/measurement_altitude fields
        self.resolve_measurement_position_accuracy(measurement_position_field_names=['altitude','sampling_height','measurement_altitude'])

        #if measurement_altitude is not set, create by altitude + sampling height if possible.
        #if all 3 altitude fields are set, but do not sum correctly, recalculate the measurement_altitude field as altitude + sampling_height
        self.set_altitude_field('measurement_altitude',check_validity='recalculate')
        #if altitude is not set, create by measurement_altitude - sampling height if possible.
        self.set_altitude_field('altitude')
        #if sampling_height is not set, create by measurement_altitude - altitude if possible.
        self.set_altitude_field('sampling_height')

        #fill gapped measurement_altitude values through column (by dependent longitude, latitude and altitude)
        self.handle_gapped_key_metadata(chosen_groups={'measurement_altitude':['longitude','latitude','altitude']})
        #fill gapped measurement_altitude values through column (by dependent longitude, latitude and sampling_height)
        self.handle_gapped_key_metadata(chosen_groups={'measurement_altitude':['longitude','latitude','sampling_height']})
        #if altitude is not set, create by measurement_altitude - sampling height if possible.
        self.set_altitude_field('altitude')
        #if sampling_height is not set, create by measurement_altitude - altitude if possible.
        self.set_altitude_field('sampling_height')

        #fill gapped altitude values through column (by dependent longitude, latitude and measurement_altitude)
        self.handle_gapped_key_metadata(chosen_groups={'altitude':['longitude','latitude','measurement_altitude']})
        #fill gapped altitude values through column (by dependent longitude, latitude and sampling_height)
        self.handle_gapped_key_metadata(chosen_groups={'altitude':['longitude','latitude','sampling_height']})
        #if measurement_altitude is not set, create by altitude + sampling height if possible.
        self.set_altitude_field('measurement_altitude')        
        #if sampling_height is not set, create by measurement_altitude - altitude if possible.
        self.set_altitude_field('sampling_height')

        #fill gapped sampling_height values through column (by dependent longitude, latitude and measurement_altitude)
        self.handle_gapped_key_metadata(chosen_groups={'sampling_height':['longitude','latitude','measurement_altitude']})
        #fill gapped sampling_height values through column (by dependent longitude, latitude and altitude)
        self.handle_gapped_key_metadata(chosen_groups={'sampling_height':['longitude','latitude','altitude']})
        #if measurement_altitude is not set, create by altitude + sampling height if possible.
        self.set_altitude_field('measurement_altitude')
        #if altitude is not set, create by measurement_altitude - sampling height if possible.
        self.set_altitude_field('altitude')

        #fill gapped measurement_altitude values through column (by dependent longitude, latitude)
        self.handle_gapped_key_metadata(chosen_groups={'measurement_altitude':['longitude','latitude']})
        #if altitude is not set, create by measurement_altitude - sampling height if possible.
        self.set_altitude_field('altitude')
        #if sampling_height is not set, create by measurement_altitude - altitude if possible.
        self.set_altitude_field('sampling_height')

        #fill gapped altitude values through column (by dependent longitude, latitude)
        self.handle_gapped_key_metadata(chosen_groups={'altitude':['longitude','latitude']})
        #if measurement_altitude is not set, create by altitude + sampling height if possible.
        self.set_altitude_field('measurement_altitude')        
        #if sampling_height is not set, create by measurement_altitude - altitude if possible.
        self.set_altitude_field('sampling_height')

        #fill gapped sampling_height values through column (by dependent longitude, latitude)
        self.handle_gapped_key_metadata(chosen_groups={'sampling_height':['longitude','latitude']})
        #if measurement_altitude is not set, create by altitude + sampling height if possible.
        self.set_altitude_field('measurement_altitude')
        #if altitude is not set, create by measurement_altitude - sampling height if possible.
        self.set_altitude_field('altitude')

        #fill gapped measurement_altitude values through column (non-dependent)
        self.handle_gapped_key_metadata(chosen_groups={'measurement_altitude':[]})
        #if altitude is not set, create by measurement_altitude - sampling height if possible.
        self.set_altitude_field('altitude')
        #if sampling_height is not set, create by measurement_altitude - altitude if possible.
        self.set_altitude_field('sampling_height')

        #fill gapped altitude values through column (non-dependent)
        self.handle_gapped_key_metadata(chosen_groups={'altitude':[]})
        #if measurement_altitude is not set, create by altitude + sampling height if possible.
        self.set_altitude_field('measurement_altitude')        
        #if sampling_height is not set, create by measurement_altitude - altitude if possible.
        self.set_altitude_field('sampling_height')

        #fill gapped sampling_height values through column (non-dependent)
        self.handle_gapped_key_metadata(chosen_groups={'sampling_height':[]})
        #if measurement_altitude is not set, create by altitude + sampling height if possible.
        self.set_altitude_field('measurement_altitude')
        #if altitude is not set, create by measurement_altitude - sampling height if possible.
        self.set_altitude_field('altitude')        

        #if after all filling for the altitude fields has been done, there are still NaN values in the altitude/measurement altitude fields, 
        #then if one of the altitude/measurement_altitude field pairs is non Nan, set the other field to be equivalent and set sampling height to 0.
        for meta_ii, current_altitude in enumerate(self.existing_metadata['altitude']):
            current_measurement_altitude = self.existing_metadata['measurement_altitude'][meta_ii]
            #altitude == np.NaN?
            if pd.isnull(current_altitude):
                #have valid measurement_altitude?
                #if so set altitude to be copy of measurement_altitude, and sampling height to be 0.
                if not pd.isnull(current_measurement_altitude):
                    self.existing_metadata['altitude'][meta_ii] = copy.deepcopy(current_measurement_altitude)   
                    self.existing_metadata['sampling_height'][meta_ii] = 0.0 
            #measurement_altitude == np.NaN?
            elif pd.isnull(current_measurement_altitude):
                #have valid altitude?
                #if so set measurement_altitude to be copy of altitude, and sampling height to be 0.
                if not pd.isnull(current_altitude):
                    self.existing_metadata['measurement_altitude'][meta_ii] = copy.deepcopy(current_altitude)   
                    self.existing_metadata['sampling_height'][meta_ii] = 0.0 

        #split stations by significantly changing locational information
        self.changing_measurement_position_new_stations()

        #return valid process logfile (0)
        self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
        create_GHOST_stage_logfile(0,self.run_details,previous_stage_data_files=self.data_files)


    def join_station_measurements(self):

        '''function for handling of all processed reported station specific data files to create continuous time series (joining measurements using the same methodology).
           this function specifically handles instances of temporally overlapping reported files, in a logical manner, creating consistently time series.

           first, all files are grouped by raw station_reference:measurement_method

           if only 1 file is available for a station_reference:measurement_method, then data file is simply taken as is

           if more than 1 file is available for a station_reference:measurement_method but there is no overlap between any of them, then the files are simply appended together

           if more than 1 file is available for a station_reference:measurement_method and there is overlap between them, then a stepped process is undertaken to determine which files should be kept on the occasions of data overlap.
           The entire potential time series is split into segments, which correspond to unique periods of different reporting files.
           i.e. for 3 months, it is possible 3 files are reporting data for the period, and after that for a month just 1 file, and after that 2 files. In this case 3 segments would be created to represent the variability of files reporting data.
           In segments where just 1 file is reporting data, data is simply taken from the 1 unique file for that period.
           In cases where there are multiple files reporting data in a segment then firstly only files with the highest number of measurements associated with the 'Corrected Parameter' QA flag are retained.
           If multiple files still exist, then only files associated with highest average data level hierarchy ranking are retained.
           If multiple files then still exist, then only files associated with the latest average data revision timestamp are retained.
           If there are still multiple files remaining, then a ranking algorithm is used to determine which data file should be chosen for the period from the remaining files.
           The factors considered in the ranking algorithm are as follows:
             -- average measurement resolution in segment (higher preferred)
             -- number of valid measurement points in segment (higher preferred)
             -- consistency with weighted metadata in segment across entire time record, for each of latitude, longitude, altitude, measurement methodology and standard instrument name fields
             -- sampling_height/measurement altitude (lower preferred)
           For each file, each of the previous listed weighting factors contribute an individual score between 1 and 2, which are summed to give total score rankings for the files.
           The file which garners the highest ranking is kept for the segment.
           With specific respect to the accounting for evaluating the metadata consistency, data from all the different station_reference:measurement_method specific files is first read and counts are made of the unique values of the per metadata field (excluding NaNs) across the time series.
           Metadata fields with values equivalent to the most utilised fields across the time record garner the highest ranking scores, and vice versa (i.e. consistency is preferred.) 
           If there is a tiebreak, this is first attempted to be broken by ranking the number of valid measurement points across the whole files (i.e. not just valid values for the segment)
           If there is still a tiebreak, this is then attempted top be broken by counting the amount of non-NaN metadata variables provided.
           If there is still a tiebreak, the first index is preferentially taken.
        '''

        #get list of data files specific for station_reference:method
        self.data_files = sorted(glob.glob(self.run_details['data_root']+'{}/{}/native/{}/temporary_S1/{}_*'.format(self.run_details['network'],self.run_details['ghost_version'],self.parameter_details['bsc_parameter_name'],self.station_reference)))

        #------------------------------------------------------------------------------------------------------------------------#
        #------------------------------------------------------------------------------------------------------------------------#

        #if have just one data file, nothing to process, just read file into memory
        if len(self.data_files) == 1:
            self.run_details['log_file_str'] += 'ONLY 1 FILE. NOTHING TO PROCESS.\n'
            loaded_variables = read_from_numpy(self.data_files[0], ['data', 'metadata'])
            self.existing_data = loaded_variables['data']
            self.existing_metadata = loaded_variables['metadata']

        #------------------------------------------------------------------------------------------------------------------------#
        #------------------------------------------------------------------------------------------------------------------------#

        #if have more than 1 file, determine if have data overlaps between files
        #this is done by simply looking at the ordered files (by start minute and then end minute),
        #if the start of the next file begins before the end of the previous file, then overlaps exist.
        #additionally if there is the string ":dup" in filename then there are overlapping files,
        #also if both the start minute and end minute are equal, are equal with the previous end time, overlaps also exist
        elif len(self.data_files) > 1:

            relevant_file_dictionary = {}

            have_overlaps = False

            for data_i, data_file in enumerate(self.data_files):
                split = data_file.split('/')[-1].split('_')
                file_start = int(split[-2])
                file_end = int(split[-1].split('.npz')[0].split(':')[0])
                if ':dup' in split[-1]:
                    have_overlaps = True
                if data_i > 0:
                    if file_start < previous_end:
                        have_overlaps = True
                    if (file_start == file_end) & (file_start == previous_end):
                        have_overlaps = True 

                previous_end = copy.deepcopy(file_end)

                #load data/metadata for file, and store in dictionary for later access (preventing re-loading)
                loaded_variables = read_from_numpy(data_file, ['data', 'metadata'])
                data = loaded_variables['data']
                metadata = loaded_variables['metadata']
                relevant_file_dictionary[data_file] = {'data':data,'metadata':metadata,'file_start':file_start,'file_end':file_end,'non_checked_indices':range(len(data))}

            #--------------------------------------------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------------------------------------------#

            #if there are no overlapping files, simply append them together
            if not have_overlaps:
                self.run_details['log_file_str'] += 'MULTIPLE FILES, BUT NO OVERLAPS. APPENDING FILES.\n'
                for data_ii, data_file in enumerate(self.data_files):
                    if data_ii == 0:
                        #if first file to put data in time series, create variables for continuous time series
                        self.existing_data = copy.deepcopy(relevant_file_dictionary[data_file]['data'])
                        self.existing_metadata = copy.deepcopy(relevant_file_dictionary[data_file]['metadata'])
                    else:
                        #append current metadata/data to existing metadata/data
                        data = copy.deepcopy(relevant_file_dictionary[data_file]['data'])
                        metadata = copy.deepcopy(relevant_file_dictionary[data_file]['metadata'])
                        existing_data_n_rows = self.existing_data.shape[0]
                        data_n_rows = data.shape[0]
                        self.existing_data.resize(existing_data_n_rows+data_n_rows)
                        self.existing_data[-data_n_rows:] = data
                        #check if each current metadata column is same as last key appended metadata column
                        #if different, a new metadata block is appended from column onwards
                        for metadata_ii, metadata_column in enumerate(metadata):
                            metadata_has_changed = determine_key_changing_metadata(self.existing_metadata[-1],metadata_column)
                            #append metadata
                            if metadata_has_changed:
                                metadata_block = metadata[metadata_ii:]
                                existing_metadata_n_rows = self.existing_metadata.shape[0]
                                metadata_n_rows = metadata_block.shape[0]
                                self.existing_metadata.resize(existing_metadata_n_rows+metadata_n_rows)
                                self.existing_metadata[-metadata_n_rows:] = metadata_block
                                break
            #--------------------------------------------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------------------------------------------#

            #if have overlapping files, split the entire potential time series into series of segments where different combinations of files provide data
            #determine which parts of time series can be taken from which file. If only have 1 file for a certain portion, take data from that file.
            #in periods where have duplicate data from different files, use ranking algorithm to determine which file to take data from
            else:
                self.run_details['log_file_str'] += 'MULTIPLE FILES, WITH OVERLAPS. CHOOSING BEST FILE FOR EACH OVERLAPPING PERIOD USING ALGORITHM.\n'
                #----------------------------------------------------------------------------------------------------------------#
                #----------------------------------------------------------------------------------------------------------------#

                #before iteration of overlap instances, get counts (in terms of minutes represented) of unique values in certain key metadata fields across entire time series (from all possible files)
                #do not allow NaNs to be counted
                metadata_vars = ['latitude','longitude','altitude','sampling_height','measurement_altitude','measurement_methodology','measuring_instrument_name']
                metadata_count_dicts = {metadata_var:{} for metadata_var in metadata_vars}
                #iterate through data files
                for data_i, data_file in enumerate(self.data_files):
                    metadata = relevant_file_dictionary[data_file]['metadata']
                    #iterate through all metadata columns
                    for metadata_ii, metadata_column in enumerate(metadata):
                        #get minutes that metadata column represents
                        if (metadata_ii+1) == len(metadata):
                            meta_representativity = relevant_file_dictionary[data_file]['file_end'] - metadata_column['meta_update_stamp'] 
                        else:
                            meta_representativity = metadata[metadata_ii+1]['meta_update_stamp'] - metadata_column['meta_update_stamp']
                        #iterate through all metadata fields
                        for metadata_var in metadata_vars:
                            metadata_value = metadata_column[metadata_var]
                            if not pd.isnull(metadata_value):
                                #if metadata value is already registered in count dictionary, add the minutes represented of metadata fields
                                if metadata_value in metadata_count_dicts[metadata_var]:
                                    metadata_count_dicts[metadata_var][metadata_value] = metadata_count_dicts[metadata_var][metadata_value] + meta_representativity
                                #otherwise add metadata value to count dictionary
                                else:
                                    metadata_count_dicts[metadata_var][metadata_value] = meta_representativity

                #normalise (between 1 and 2) metadata counts dict values by max count   
                for metadata_var in metadata_vars:
                    normalised_counts = normalise_by_max(list(metadata_count_dicts[metadata_var].values()),offset=1.0)
                    metadata_count_dicts[metadata_var].update(zip(metadata_count_dicts[metadata_var], normalised_counts))

                #normalise (between 1 and 2) unique sampling heights/measurement altitudes by minimum values (lower heights = higher ranking)
                altitude_norm_dicts = {}
                altitude_norm_dicts['sampling_height'] = {key:val for key,val in zip(list(metadata_count_dicts['sampling_height'].keys()), normalise_by_max(list(metadata_count_dicts['sampling_height'].keys()), offset=1.0, reverse=True))}  
                altitude_norm_dicts['measurement_altitude'] = {key:val for key,val in zip(list(metadata_count_dicts['measurement_altitude'].keys()), normalise_by_max(list(metadata_count_dicts['measurement_altitude'].keys()), offset=1.0, reverse=True))}

                #----------------------------------------------------------------------------------------------------------------#
                #----------------------------------------------------------------------------------------------------------------#

                #split time series into segments of periods where different combinations of files provide data
                #this is done by iterating through all measurement windows, of all data files and checking for overlaps with measurement windows of all other data files
                #all periods where there are overlaps across multiple data files are counted as the same segments until there are no more overlaps and the contributing files change.
                #where there are overlaps between files, the edges of the relevant measurement windows are taken as the borders of the overlapping setion.

                #create timeseries dict for storing unique segment start/ends associated with files providing data in the segments
                timeseries_compilation_dict = {}

                #first do quick and dirty check to group together files which overlap based on file start/end times

                #list to store grouped lists of associated overlapping files
                overlap_groups = []

                #initialise list wrote_lists to store all the files you have associated with a list
                wrote_files = []

                #iterate through all files
                for base_data_ii, base_data_file in enumerate(self.data_files):

                    #if have already associated base file, continue to next file
                    if base_data_file in wrote_files:
                        continue

                    #set current start of overlap group as beginning/end of iterated base file
                    current_group_start = relevant_file_dictionary[base_data_file]['file_start']
                    current_group_end = relevant_file_dictionary[base_data_file]['file_end']

                    #initialise current group files as current base file
                    current_group_files = [base_data_file]
                    #add base file to wrote files
                    wrote_files.append(base_data_file)

                    #set flag to keep iterating over files until current group start/end boundaries are unmodified
                    keep_iterating = True
                    while keep_iterating:
                        #set keep iterating flag to False, i.e. will stop iterating over list of next files, if current group start/end is not modified on this pass
                        keep_iterating = False

                        #iterate through all files after current base file
                        for next_data_file in self.data_files[base_data_ii+1:]:

                            #if have already iterated associated file, continue to next file
                            if next_data_file in wrote_files:
                                continue

                            #get start/end times of iterated file
                            next_file_start = relevant_file_dictionary[next_data_file]['file_start']
                            next_file_end = relevant_file_dictionary[next_data_file]['file_end']

                            #determine if iterated file has overlap with current_group
                            have_overlap = (current_group_start <= next_file_start < current_group_end)

                            #if have overlap, then append iterated file to current group files and modify current end of group overlap
                            #also write filename to list of files you have already associated
                            if have_overlap:
                                current_group_files.append(next_data_file)
                                wrote_files.append(next_data_file)

                                #determine if group start/end need modifying
                                #if either do, reset keep_iterating to true
                                if next_file_start < current_group_start:
                                    current_group_start = copy.deepcopy(next_file_start)
                                    keep_iterating = True
                                if next_file_end > current_group_end:
                                    current_group_end = copy.deepcopy(next_file_end)
                                    keep_iterating = True

                    #append associated overlapping data files together as list to overlap groups list
                    overlap_groups.append(current_group_files)

                #iterate through overlapping group data files and compare measurement windows with all other files (the files in this iteration which be called base files)
                for overlap_group_data_files in overlap_groups:
                    #if only have 1 file in group, no overlaps with that file therefore simply write file start/end as segment start/end
                    if len(overlap_group_data_files) == 1:
                        timeseries_compilation_dict['{}-{}'.format(relevant_file_dictionary[overlap_group_data_files[0]]['file_start'],relevant_file_dictionary[overlap_group_data_files[0]]['file_end'])] = overlap_group_data_files

                    #else if have overlapping files, resolve overlaps window by window
                    else:

                        for base_data_ii, base_data_file in enumerate(overlap_group_data_files):
                            #get data for base file
                            data = relevant_file_dictionary[base_data_file]['data']

                            #get base file start/end times
                            base_file_start = relevant_file_dictionary[base_data_file]['file_start']
                            base_file_end = relevant_file_dictionary[base_data_file]['file_end']

                            #get indices of measurement windows in base file that have not been cross-checked with other file windows
                            base_non_checked_indices = relevant_file_dictionary[base_data_file]['non_checked_indices']

                            #get base data file non-checked window start times and measurement resolutions
                            base_file_times = data['time'][base_non_checked_indices]
                            base_file_temporal_resolutions = data['temporal_resolution'][base_non_checked_indices]

                            if len(base_file_times) > 0:

                                #initialise previous iteration variables
                                previous_iteration_end = 0
                                previous_iteration_files = []

                                #initialise previous/current segment variables
                                current_segment_start = base_file_times[0]
                                current_segment_end = current_segment_start + base_file_temporal_resolutions[0]
                                current_segment_files = []

                                #initialise segment iteration variables
                                segment_iteration_starts = []
                                segment_iteration_ends = []

                                #initialise list for handling duplicate files
                                duplicates_to_add_to_iteration_files = []

                                #iterate over base file measurement windows
                                for base_window_ii, (base_window_start, base_temporal_resolution) in enumerate(zip(base_file_times,base_file_temporal_resolutions)):

                                    #get end of base measurement window
                                    base_window_end = base_window_start + base_temporal_resolution

                                    #initialise current iteration variables
                                    current_iteration_start = copy.deepcopy(base_window_start)
                                    current_iteration_end = copy.deepcopy(base_window_end)
                                    if len(duplicates_to_add_to_iteration_files) > 0:
                                        current_iteration_files = [base_data_file] + duplicates_to_add_to_iteration_files
                                    else:
                                        current_iteration_files = [base_data_file]

                                    #update indices of non-checked windows for base file (drop current index)
                                    relevant_file_dictionary[base_data_file]['non_checked_indices'] = relevant_file_dictionary[base_data_file]['non_checked_indices'][1:]

                                    #iterate through all other files and through all file measurement windows. Check overlap initially with current base file measurement window, and then iteratively with all other file windows
                                    #set flag to indicate if you want to keep iterating  (this will be set to False when have no more overlaps)
                                    keep_iterating = True

                                    while keep_iterating:

                                        #set flag for keep_iterating to be False -- this will be modified if find any overlaps are found with current segment through pass of all other files
                                        keep_iterating = False

                                        #set flag for forcing write on this iteration if window in void between current segment end and current iteration start
                                        #this reverts back to False each time current iteration segment is modified
                                        force_write = False

                                        #check overlap of current segment with all measurement windows in other files
                                        for next_data_file in overlap_group_data_files[base_data_ii+1:]:
                                            #if iterated file has been decreed as a duplicate, continue to next file immediately
                                            if next_data_file in duplicates_to_add_to_iteration_files:
                                                continue

                                            #get iterated file start/end times
                                            next_file_start = relevant_file_dictionary[next_data_file]['file_start']
                                            next_file_end = relevant_file_dictionary[next_data_file]['file_end']

                                            #get iterated file data
                                            next_data = relevant_file_dictionary[next_data_file]['data']

                                            #get non-checked indices of measurement windows in iterated file that have not been cross-checked with other file windows
                                            next_non_checked_indices = relevant_file_dictionary[next_data_file]['non_checked_indices']

                                            #get iterated data file non-checked window start times and measurement resolutions
                                            next_file_times = next_data['time'][next_non_checked_indices]
                                            next_file_temporal_resolutions = next_data['temporal_resolution'][next_non_checked_indices]

                                            #if both iterated file times and iterated file temporal resolutions are exactly same as the base file, this is because they are exact duplicates
                                            #set flag to add duplicate file to base file on each iteration for rest of iteration through base file windows
                                            #also set all indices to be checked, and add duplicated file to current_iteration_files
                                            if (base_file_start == next_file_start) & (base_file_end == next_file_end):
                                                if (np.array_equal(base_file_times, next_file_times, equal_nan=True)) & (np.array_equal(base_file_temporal_resolutions, next_file_temporal_resolutions, equal_nan=True)):
                                                #if np.all((base_file_times == next_file_times) & (base_file_temporal_resolutions == next_file_temporal_resolutions)):
                                                    duplicates_to_add_to_iteration_files.append(next_data_file)
                                                    relevant_file_dictionary[next_data_file]['non_checked_indices'] = []
                                                    current_iteration_files.append(next_data_file)
                                                    continue

                                            #set list for appending checked indices to drop in iterated file
                                            next_file_indices_to_drop = []

                                            #iterate over file windows
                                            for next_window_ii, (next_window_start, next_temporal_resolution) in enumerate(zip(next_file_times,next_file_temporal_resolutions)):

                                                #get end of iterated file measurement window
                                                next_window_end = next_window_start + next_temporal_resolution

                                                #determine if there are any windows in void between current_segment_end and current_iteration_start
                                                #if there are, force write on this iteration (if current iteration window is not subsequently modified - in which case this will be reevaluated)
                                                if (current_segment_end <= next_window_start < current_iteration_start) & ((current_segment_end < next_window_end <= current_iteration_start)):
                                                    force_write = True
                                                    continue

                                                #if next_window_end <= current_iteration_start, then continue to next file window
                                                if next_window_end <= current_iteration_start:
                                                    continue

                                                #if next_window_start >= current_iteration_end, break to next file
                                                if next_window_start >= current_iteration_end:
                                                    break

                                                #else, have overlap with current iteration segment!
                                                #add iterated file to list of current iteration files (if not already there)
                                                if next_data_file not in current_iteration_files:
                                                    current_iteration_files.append(next_data_file)
                                                #if next_window_start < current_iteration_start, update current_iteration_start
                                                #also set keep_iterating flag to True, as have modified current iteration segment boundaries
                                                if next_window_start < current_iteration_start:
                                                    current_iteration_start = copy.deepcopy(next_window_start)
                                                    keep_iterating = True
                                                #if next_window_end > current_iteration_end, update current_iteration_end
                                                #also set keep_iterating flag to True, as have modified current iteration segment boundaries
                                                if next_window_end > current_iteration_end:
                                                    current_iteration_end = copy.deepcopy(next_window_end)
                                                    keep_iterating = True

                                                #append checked iterated file index to list of indices to drop (after have gone through all relevant file windows on this pass)
                                                next_file_indices_to_drop.append(next_non_checked_indices[next_window_ii])

                                            #update indices of non-checked windows for iterated file (drop current index)
                                            relevant_file_dictionary[next_data_file]['non_checked_indices'] = [x for x in relevant_file_dictionary[next_data_file]['non_checked_indices'] if x not in next_file_indices_to_drop]

                                    #now have found all overlapping windows with one base file window in all other files
                                    #now determine if will write out segment on this iteration

                                    #1. determine if you have overlap between current iteration start and current segment end
                                        #if have overlap do not write out on this iteration
                                    #AND
                                    #2. #determine if current segment files are different from current iteration files
                                        #if are not different do not write

                                    #OR
                                    #1. force_write flag == True

                                    if ((current_segment_end <= current_iteration_start) & (sorted(current_segment_files) != sorted(current_iteration_files))) or (force_write):

                                        #WRITE OUT SEGMENT!

                                        #test intersections between current segment start/end points with already written segments through time record. If have any intersection, split current segment files into chunks split by intersecting written segment start points.
                                        #this is for dealing with cases where you have segments in files (which are not the initial base file) entirely disconnected from all other segments

                                        intersect_break_points = []
                                        for written_segment in list(timeseries_compilation_dict.keys()):
                                            written_segment_split = written_segment.split('-')
                                            written_segment_start, written_segment_end = int(written_segment_split[0]), int(written_segment_split[1])
                                            have_intersect = (current_segment_start <= written_segment_start < current_segment_end) or (written_segment_start <= current_segment_start < written_segment_end)
                                            if have_intersect:
                                                intersect_break_points.append(written_segment_start)
                                        if len(intersect_break_points) > 0:
                                            #iterate through break points
                                            for break_point in sorted(intersect_break_points):
                                                #chunk segment_iteration_starts and segment_iteration_ends into separate groups split by breakpoints
                                                chunked_group_inds = [i for i in range(len(segment_iteration_starts)) if segment_iteration_starts[i] < break_point]
                                                #check have some indices to chunk, if not continue to next break point
                                                if len(chunked_group_inds) > 0:
                                                    #filter segment_iteration_starts and segment_iteration_ends by valid inds
                                                    chunked_group_starts = [segment_iteration_starts[i] for i in chunked_group_inds]
                                                    chunked_group_ends = [segment_iteration_ends[i] for i in chunked_group_inds]
                                                    #cut segment_iteration_starts and segment_iteration_starts lists
                                                    segment_iteration_starts = [segment_iteration_starts[i] for i in range(len(segment_iteration_starts)) if i not in chunked_group_inds]
                                                    segment_iteration_ends = [segment_iteration_ends[i] for i in range(len(segment_iteration_ends)) if i not in chunked_group_inds]

                                                    #write out chunked groups (take first chunked group start and last chunked group end as boundaries)
                                                    timeseries_compilation_dict['{}-{}'.format(chunked_group_starts[0],chunked_group_ends[-1])] = copy.deepcopy(current_segment_files)
                                                    #self.run_details['log_file_str'] += '------b {}-{} {}\n'.format(chunked_group_starts[0],chunked_group_ends[-1], current_segment_files)

                                            #write out final chunked group after last breakpoint - if have data remaining (take first remaining group start and last group end as boundaries)
                                            if len(segment_iteration_starts) > 0:
                                                timeseries_compilation_dict['{}-{}'.format(segment_iteration_starts[0],segment_iteration_ends[-1])] = copy.deepcopy(current_segment_files)
                                                #self.run_details['log_file_str'] += '------c {}-{} {}\n'.format(segment_iteration_starts[0],segment_iteration_ends[-1],current_segment_files)

                                        else:
                                            timeseries_compilation_dict['{}-{}'.format(current_segment_start,current_segment_end)] = copy.deepcopy(current_segment_files)
                                            #self.run_details['log_file_str'] += '------a {}-{} {}\n'.format(current_segment_start,current_segment_end,current_segment_files)

                                        #set current segment files to be current iteration files
                                        current_segment_files = copy.deepcopy(current_iteration_files)

                                        #reset current segment start/end to current iteration start/end
                                        current_segment_start = copy.deepcopy(current_iteration_start)
                                        current_segment_end = copy.deepcopy(current_iteration_end)

                                        #reset segment_iteration_starts with current_iteration_start
                                        segment_iteration_starts = [current_iteration_start]

                                        #reset segment_iteration_ends with current_iteration_end
                                        segment_iteration_ends = [current_iteration_end]


                                    #if not writing out, update current segment variables
                                    else:
                                        #if current iteration start < current segment start, update it
                                        if current_iteration_start < current_segment_start:
                                            current_segment_start = copy.deepcopy(current_iteration_start)

                                        #if current iteration end > current segment end, update it
                                        if current_iteration_end > current_segment_end:
                                            current_segment_end = copy.deepcopy(current_iteration_end)

                                        #update current segment files (with all new files that are not in current group files)
                                        for iteration_file in current_iteration_files:
                                            if iteration_file not in current_segment_files:
                                                current_segment_files.append(iteration_file)

                                        #append segment_iteration_starts with current_iteration_start
                                        segment_iteration_starts.append(current_iteration_start)

                                        #append to segment_iteration_ends with current_iteration_end
                                        segment_iteration_ends.append(current_iteration_end)


                                    #set previous iteration end as current_iteration_end
                                    previous_iteration_end = copy.deepcopy(current_iteration_end)


                                #now have cross-checked all windows of one file with all other file windows, write out what remains in current_segment files which has start data before last written segment start

                                #WRITE OUT SEGMENT!

                                #test intersections between current segment start/end points with already written segments through time record. If have any intersection, split current segment files into chunks split by intersecting written segment start points.
                                #this is for dealing with cases where you have segments in files (which are not the initial base file) entirely disconnected from all other segments

                                intersect_break_points = []
                                for written_segment in list(timeseries_compilation_dict.keys()):
                                    written_segment_split = written_segment.split('-')
                                    written_segment_start, written_segment_end = int(written_segment_split[0]), int(written_segment_split[1])
                                    have_intersect = (current_segment_start <= written_segment_start < current_segment_end) or (written_segment_start <= current_segment_start < written_segment_end)
                                    if have_intersect:
                                        intersect_break_points.append(written_segment_start)
                                if len(intersect_break_points) > 0:
                                    #iterate through break points
                                    for break_point in sorted(intersect_break_points):
                                        #chunk segment_iteration_starts and segment_iteration_ends into separate groups split by breakpoints
                                        chunked_group_inds = [i for i in range(len(segment_iteration_starts)) if segment_iteration_starts[i] < break_point]
                                        #check have some indices to chunk, if not continue to next break point
                                        if len(chunked_group_inds) > 0:
                                            #filter segment_iteration_starts and segment_iteration_ends by valid inds
                                            chunked_group_starts = [segment_iteration_starts[i] for i in chunked_group_inds]
                                            chunked_group_ends = [segment_iteration_ends[i] for i in chunked_group_inds]
                                            #cut segment_iteration_starts and segment_iteration_starts lists
                                            segment_iteration_starts = [segment_iteration_starts[i] for i in range(len(segment_iteration_starts)) if i not in chunked_group_inds]
                                            segment_iteration_ends = [segment_iteration_ends[i] for i in range(len(segment_iteration_ends)) if i not in chunked_group_inds]

                                            #write out chunked groups (take first chunked group start and last chunked group end as boundaries)
                                            timeseries_compilation_dict['{}-{}'.format(chunked_group_starts[0],chunked_group_ends[-1])] = copy.deepcopy(current_segment_files)
                                            #self.run_details['log_file_str'] += '------e {}-{} {}\n'.format(chunked_group_starts[0],chunked_group_ends[-1],current_segment_files)

                                    #write out final chunked group after last breakpoint - if have data remaining (take first remaining group start and last group end as boundaries)
                                    if len(segment_iteration_starts) > 0:
                                        timeseries_compilation_dict['{}-{}'.format(segment_iteration_starts[0],segment_iteration_ends[-1])] = copy.deepcopy(current_segment_files)
                                        #self.run_details['log_file_str'] += '------f {}-{} {}\n'.format(segment_iteration_starts[0],segment_iteration_ends[-1],current_segment_files)

                                else:
                                    timeseries_compilation_dict['{}-{}'.format(current_segment_start,current_segment_end)] = copy.deepcopy(current_segment_files)
                                    #self.run_details['log_file_str'] += '------d {}-{} {}\n'.format(current_segment_start,current_segment_end,current_segment_files)

                #-----------------------------------------------------#

                #now have finished all cross-checking of files
                #for key in sorted(list(timeseries_compilation_dict.keys())):
                #    split_key = key.split('-')
                #    self.run_details['log_file_str'] += '{} {} {} {} {}\n'.format(fractional_minutes_from_reference_to_datetime(float(split_key[0])), fractional_minutes_from_reference_to_datetime(float(split_key[1])), split_key[0], split_key[1], timeseries_compilation_dict[key])
                #    self.run_details['log_file_str'] += '{}\n'.format([np.unique(relevant_file_dictionary[period_file]['data']['temporal_resolution']) for period_file in timeseries_compilation_dict[key]])

                #iterate through segmented periods
                for period in sorted(timeseries_compilation_dict):
                    period_split = period.split('-')
                    current_period_start = int(period_split[0])
                    current_period_end = int(period_split[1])
                    current_period_files = timeseries_compilation_dict[period]

                    self.run_details['log_file_str'] += '{} {} {} {} {}\n'.format(fractional_minutes_from_reference_to_datetime(float(period_split[0])), fractional_minutes_from_reference_to_datetime(float(period_split[1])), period_split[0], period_split[1], timeseries_compilation_dict[period])
                    #self.run_details['log_file_str'] += '{}\n'.format([np.unique(relevant_file_dictionary[period_file]['data']['temporal_resolution']) for period_file in timeseries_compilation_dict[period]])

                    #------------------------------------------------------------------------------------------------------------#
                    #------------------------------------------------------------------------------------------------------------#

                    #if only 1 file provides data in period, simply append relevant data from it to current timeseries (if already exists)
                    if len(current_period_files) == 1:
                        chosen_file = current_period_files[0]

                    #------------------------------------------------------------------------------------------------------------#
                    #------------------------------------------------------------------------------------------------------------#

                    #if have file overlap for period, resolve by using ranking algorithm
                    else:

                        #self.run_details['log_file_str'] += 'PERIOD:{}\n'.format(period)

                        #set some variables for storing file data level ranks, and for calculation of ranking algorithm
                        file_data_level_ranks = np.array([],dtype=object)
                        file_data_revision_stamps = np.array([],dtype=object)
                        algorithm_ranks = np.array([0.0]*len(current_period_files), dtype=np.float32)
                        average_temporal_resolutions = np.array([],dtype=np.float32)
                        valid_data_N_relevant = np.array([],dtype=np.float32)
                        valid_data_N_full = np.array([],dtype=np.float32)
                        valid_data_corrected = np.array([],dtype=np.float32)
                        valid_metadata_vars_N = np.array([],dtype=np.float32)
                        chosen_file_index = -99

                        #iterate through all overlapping files in current period
                        for period_i, period_file in enumerate(current_period_files):

                            #self.run_details['log_file_str'] += 'PERIOD FILE:{}\n'.format(period_file)

                            full_data = copy.deepcopy(relevant_file_dictionary[period_file]['data'])
                            full_metadata = copy.deepcopy(relevant_file_dictionary[period_file]['metadata'])
                            #check if want all of data from file or it needs cutting
                            if (current_period_start != relevant_file_dictionary[period_file]['file_start']) or (current_period_end != relevant_file_dictionary[period_file]['file_end']):
                                relevant_data = cut_data_array(full_data, 'data', current_period_start, current_period_end)
                                relevant_metadata = cut_data_array(full_metadata, 'metadata', current_period_start, current_period_end)
                            else:
                                relevant_data = copy.deepcopy(full_data)
                                relevant_metadata = copy.deepcopy(full_metadata)

                            #---------------------------------------------------------#
                            #calculate algorithm scores

                            #process modal occurring metadata scores

                            #iterate through all metadata fields
                            evaluated_altitude = False
                            for metadata_var in metadata_vars:

                                all_algorithm_ranks = [] 
                                all_algorithm_weights = []                                
                                    
                                #iterate through metadata columns
                                for metadata_column_ii, metadata_column in enumerate(relevant_metadata):
                                    #get current metadata value
                                    current_meta_value = metadata_column[metadata_var]

                                    #get minutes that metadata column represents (for use as weights)
                                    if (metadata_column_ii+1) == len(relevant_metadata):
                                        meta_representativity = current_period_end - metadata_column['meta_update_stamp']
                                    else:
                                        meta_representativity = relevant_metadata[metadata_column_ii+1]['meta_update_stamp'] - metadata_column['meta_update_stamp']

                                    #process modally occurring metadata score (only process if have current metadata is not np.NaN)
                                    if not pd.isnull(current_meta_value):
                                        #variable is sampling_height/measurement_altitude?
                                        if metadata_var in list(altitude_norm_dicts.keys()):
                                            if not evaluated_altitude:  
                                                all_algorithm_ranks.append(np.average([metadata_count_dicts[metadata_var][current_meta_value],altitude_norm_dicts[metadata_var][current_meta_value]]))  
                                                all_algorithm_weights.append(meta_representativity)
                                        else:
                                            all_algorithm_ranks.append(metadata_count_dicts[metadata_var][current_meta_value])
                                            all_algorithm_weights.append(meta_representativity)
                                        
                                #append weighted average algorithm ranks across metadata columns (if have values for metadata variable)
                                if len(all_algorithm_ranks) > 0:
                                    algorithm_ranks[period_i] += np.average(all_algorithm_ranks, weights=all_algorithm_weights)
                                    if metadata_var in list(altitude_norm_dicts.keys()):
                                        evaluated_altitude = True 

                            #calculate average temporal resolution in relevant segment of file, and append to list
                            average_temporal_resolutions = np.append(average_temporal_resolutions, np.average(relevant_data['temporal_resolution']))

                            #calculate number of valid data in relevant segment of file (after removing data by flags - GHOST decreed)
                            invalid_qa_flag_code = standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed']
                            valid_data_N_relevant = np.append(valid_data_N_relevant, len([0 for qa_codes in relevant_data['qa'] if invalid_qa_flag_code not in qa_codes]))

                            #calculate number of valid data across all of file (after removing data by flags - GHOST decreed) --> used as primary tiebreaker if file ranks are equal
                            valid_data_N_full = np.append(valid_data_N_full, len([0 for qa_codes in full_data['qa'] if invalid_qa_flag_code not in qa_codes]))

                            #calculate number of measurements associated with 'Corrected Parameter' QA flag
                            corrected_code = standard_QA_name_to_QA_code['Corrected Parameter']
                            valid_data_corrected = np.append(valid_data_corrected, len([0 for qa_codes in relevant_data['qa'] if corrected_code in qa_codes]))

                            #calculate average data level hierarchy ranking of overlapping file (higher being better)
                            all_data_level_ranks = []
                            #iterate through metadata columns
                            for metadata_column in relevant_metadata:
                                all_data_level_ranks.append(self.network_details['data_level_hierarchy'][metadata_column['data_level']])
                            #append average data level rank across metadata columns
                            file_data_level_ranks = np.append(file_data_level_ranks,np.average(all_data_level_ranks))

                            #self.run_details['log_file_str'] += 'FILE DATA LEVEL RANKS:{}\n'.format(file_data_level_ranks)

                            #calculate average data revision timestamp of overlapping file (later is better)
                            all_data_revision_stamps = []
                            #iterate through metadata columns
                            for metadata_column in relevant_metadata:
                                all_data_revision_stamps.append(metadata_column['data_revision_stamp'])
                            #append average data level rank across metadata columns
                            file_data_revision_stamps = np.append(file_data_revision_stamps,np.nanmean(all_data_revision_stamps))

                            #calculate number of valid metadata fields provided (i.e. not np.NaN) --> used as secondary tiebreaker if file ranks are equal
                            valid_metadata_vars = 0
                            for metadata_column_ii, metadata_column in enumerate(relevant_metadata):
                                for meta_name in metadata_column.dtype.names:
                                    if not pd.isnull(metadata_column[meta_name]):
                                        valid_metadata_vars += 1.0
                            #append number of average valid metadata fields across data columns 
                            valid_metadata_vars_N = np.append(valid_metadata_vars_N, valid_metadata_vars/len(relevant_metadata))

                        #get normalised ranks (between 1 and 2) of average temporal resolutions of files (higher resolution (i.e. smaller number) == higher rank)
                        #first convert temporal resolutions to be negative to ensure that the smallest numbers are given the biggest ranks
                        #update algorithm ranks
                        algorithm_ranks += normalise_by_max(average_temporal_resolutions, offset=1.0, reverse=True)

                        #get normalised ranks (between 1 and 2) of number of valid data in relevant sections of files (more data == higher rank)
                        #update algorithm ranks 
                        algorithm_ranks += normalise_by_max(valid_data_N_relevant, offset=1.0)

                        #get normalised ranks (between 1 and 2) of the average data level hierarchy ranking (higher data level == higher rank)
                        #algorithm_ranks += normalise_by_max(file_data_level_ranks, offset=1.0)

                        #get normalised ranks (between 1 and 2) of the average data revision timestamp (later stamp == higher rank)
                        #if the revison timestamp is not given (i.e. NaN) for a file then the rank for the file is set as 0. 
                        #no ranks are added if there are no data revison timestamps provided
                        #no_data_revision_stamp = pd.isnull(file_data_revision_stamps)
                        #have some non-NaN stamps?
                        #if not all(no_data_revision_stamp): 
                        #    #set NaN values as non-NaN minimum values 
                        #    file_data_revision_stamps[no_data_revision_stamp] = np.nanmin(file_data_revision_stamps)
                        #    #get normalised ranks
                        #    data_revision_ranks = normalise_by_max(file_data_revision_stamps, offset=1.0)
                        #    #set NaN stamps as 0
                        #    data_revision_ranks[no_data_revision_stamp] = 0
                        #    algorithm_ranks += data_revision_ranks

                        #----------------------------------------#
                        #now have iterated through all overlapping files and and have processed ranking algorithm
                        #first attempt to choose between duplicate files by keeping file with greatest number of measurements associated with 'Corrected Parameter' QA flag
                        max_corrected = np.max(valid_data_corrected)
                        #get indices of overlapping files which have N measurements associated with the max corrected number
                        max_corrected_indices = np.where(valid_data_corrected == max_corrected)[0]

                        #if have multiple files with the same max N corrected number, drop any files which have do not have the max number from consideration
                        if (len(max_corrected_indices) > 1) & (len(max_corrected_indices) != len(valid_data_corrected)):
                            #drop files which do not have max N corrected number from consideration when resolving overlap using ranking algorithm
                            current_period_files = np.array(current_period_files)[max_corrected_indices]
                            algorithm_ranks = algorithm_ranks[max_corrected_indices]
                            valid_data_N_full = valid_data_N_full[max_corrected_indices]
                            valid_metadata_vars_N = valid_metadata_vars_N[max_corrected_indices]
                            file_data_revision_stamps = file_data_revision_stamps[max_corrected_indices]
                            file_data_level_ranks = file_data_level_ranks[max_corrected_indices]
                        #else if have only 1 file with max N corrected number, use this file preferentially for current period and do not use ranking algorithm
                        elif len(max_corrected_indices) == 1:
                            chosen_file_index = max_corrected_indices[0]
                            self.run_details['log_file_str'] += 'OVERLAP RESOLVED BY "CORRECTED PARAMETER" CHECK\n'

                        #----------------------------------------#
                        #if still have overlapping files, now attempt to choose between duplicate files by keeping file with the highest priority data level.
                        
                        if chosen_file_index < 0:

                            #get max ranking of data levels across the overlapping files
                            max_rank = np.max(file_data_level_ranks)
                            #get indices of overlapping files which have a ranking == max rank
                            max_rank_indices = np.where(file_data_level_ranks == max_rank)[0]

                            #if have multiple files with the same max rank, drop any files which have do not have the max rank from consideration
                            if (len(max_rank_indices) > 1) & (len(max_rank_indices) != len(file_data_level_ranks)):
                                #drop files which do not have max data level rank from consideration when resolving overlap using ranking algorithm
                                current_period_files = np.array(current_period_files)[max_rank_indices]
                                algorithm_ranks = algorithm_ranks[max_rank_indices]
                                valid_data_N_full = valid_data_N_full[max_rank_indices]
                                valid_metadata_vars_N = valid_metadata_vars_N[max_rank_indices]
                                file_data_revision_stamps = file_data_revision_stamps[max_rank_indices]
                            #else if have only 1 file with max rank, use this file preferentially for current period and do not use ranking algorithm
                            elif len(max_rank_indices) == 1:
                                chosen_file_index = max_rank_indices[0]
                                self.run_details['log_file_str'] += 'OVERLAP RESOLVED BY "HIGHEST PRIORITY DATA LEVEL" CHECK\n'
                            
                            #----------------------------------------#
                            #if still have overlapping files, now attempt to revise by keeping file with the latest data revison timestamp  
                            #get latest data revision timestamp

                            if chosen_file_index < 0:

                                latest_file_data_revision_stamp = np.max(file_data_revision_stamps)                
                                #get indices of overlapping files which have a data revision timestamp == latest timestamp
                                latest_indices = np.where(file_data_revision_stamps == latest_file_data_revision_stamp)[0]

                                #if have multiple files with the same latest data revision timestamp, drop any files which have do not have the latest data revision timestamp from consideration, and proceed to resolve overlap using ranking algorithm
                                if (len(latest_indices) > 1) & (len(latest_indices) != len(file_data_revision_stamps)):
                                    #drop files which do not have max data level rank from consideration when resolving overlap using ranking algorithm
                                    current_period_files = np.array(current_period_files)[latest_indices]
                                    algorithm_ranks = algorithm_ranks[latest_indices]
                                    valid_data_N_full = valid_data_N_full[latest_indices]
                                    valid_metadata_vars_N = valid_metadata_vars_N[latest_indices]
                                #else if have only 1 file with latest data revision timestamp, use this file preferentially for current period and do not use ranking algorithm
                                elif len(latest_indices) == 1:
                                    chosen_file_index = latest_indices[0]
                                    self.run_details['log_file_str'] += 'OVERLAP RESOLVED BY "LATEST DATA REVISION TIMESTAMP" CHECK\n'

                                #----------------------------------------#
                                #if still have overlapping files, now resolve using ranking algorithm
                        
                                if chosen_file_index < 0:

                                    #now have iterated through all overlapping files and have processed ranking algorithm

                                    #get max algorithm ranking across the overlapping files
                                    max_rank = np.max(algorithm_ranks)
                                    #get indices of overlapping files which have a ranking == max rank
                                    max_rank_indices = np.where(algorithm_ranks == max_rank)[0]

                                    #if have multiple files with the same rank, tiebreak by N valid data in whole data file, and then N metadata values provided (that are not np.NaN)
                                    if len(max_rank_indices) > 1:
                                        #reduce variables for valid files
                                        current_period_files = np.array(current_period_files)[max_rank_indices]
                                        valid_data_N_full = valid_data_N_full[max_rank_indices]
                                        valid_metadata_vars_N = valid_metadata_vars_N[max_rank_indices]

                                        #tiebreak preferentially by N valid data in whole data file
                                        #get ranks of number of valid data in across all of files (more data == higher rank)
                                        valid_data_N_full_ranks = ss.rankdata(valid_data_N_full)
                                        max_rank_indices = np.where(valid_data_N_full_ranks == np.max(valid_data_N_full_ranks))[0]

                                        #if have broken tiebreak, set preferential file index here
                                        if len(max_rank_indices) == 1:
                                            chosen_file_index = max_rank_indices[0]
                                            self.run_details['log_file_str'] += 'OVERLAP RESOLVED BY "N VALID DATA" TIEBREAK\n'
                                        else:
                                            #tiebreak secondly by N metadata values provided (that are not np.NaN)
                                            #reduce variables for valid files
                                            current_period_files = np.array(current_period_files)[max_rank_indices]
                                            valid_metadata_vars_N = valid_metadata_vars_N[max_rank_indices]

                                            #get ranks of number of metadata fields values are provided for (i.e. not NaN) in relevant section of file (more fields == higher rank)
                                            valid_metadata_vars_N_ranks = ss.rankdata(valid_metadata_vars_N)
                                            #finally, if N metadata values are equal also, then simply first index is taken
                                            chosen_file_index = np.argmax(valid_metadata_vars_N_ranks)
                                            self.run_details['log_file_str'] += 'OVERLAP RESOLVED BY "N VALID METADATA VALUES" TIEBREAK\n'

                                    else:
                                        chosen_file_index = max_rank_indices[0]
                                        self.run_details['log_file_str'] += 'OVERLAP RESOLVED BY "RANKING ALGORITHM" CHECK\n'

                        #----------------------------------------#
                        #define chosen file to keep for data period
                        chosen_file = current_period_files[chosen_file_index]
                        self.run_details['log_file_str'] += 'CHOSEN FILE: {}\n'.format(chosen_file)

                    #------------------------------------------------------------------------------------------------------------#
                    #------------------------------------------------------------------------------------------------------------#

                    #add chosen file data/metadata for period to existing time series (if already exists)
                    data = copy.deepcopy(relevant_file_dictionary[chosen_file]['data'])
                    metadata = copy.deepcopy(relevant_file_dictionary[chosen_file]['metadata'])

                    #check if want all of data from file or it needs cutting
                    if (current_period_start != relevant_file_dictionary[chosen_file]['file_start']) or (current_period_end != relevant_file_dictionary[chosen_file]['file_end']):
                        data = cut_data_array(data, 'data', current_period_start, current_period_end)
                        metadata = cut_data_array(metadata, 'metadata', current_period_start, current_period_end)

                    try:
                        #check if have some continuous time series data stored already - if so, append to these data/metadata variables
                        self.existing_data

                        #append current metadata/data to existing metadata/data
                        #self.run_details['log_file_str'] += 'EXISTING DATA SHAPE:{}\n'.format(self.existing_data.shape)
                        #self.run_details['log_file_str'] += 'DATA SHAPE:{}\n'.format(data.shape)
                        #self.run_details['log_file_str'] += 'DATA FIRST/LAST:{},{}\n'.format(data['time'][0],data['time'][-1])
                        existing_data_n_rows = self.existing_data.shape[0]
                        data_n_rows = data.shape[0]
                        self.existing_data.resize(existing_data_n_rows+data_n_rows)
                        self.existing_data[-data_n_rows:] = data
                        #if the file is the same as previous, metadata continues to be the same, therefore do not need to append metadata
                        if previous_chosen_file != chosen_file:
                            #otherwise, check if each current metadata column is same as last key appended metadata column
                            #if different, a new metadata block is appended from column onwards
                            for metadata_ii, metadata_column in enumerate(metadata):
                                metadata_has_changed = determine_key_changing_metadata(self.existing_metadata[-1],metadata_column)
                                #append metadata
                                if metadata_has_changed:
                                    metadata_block = metadata[metadata_ii:]
                                    existing_metadata_n_rows = self.existing_metadata.shape[0]
                                    metadata_n_rows = metadata_block.shape[0]
                                    self.existing_metadata.resize(existing_metadata_n_rows+metadata_n_rows)
                                    self.existing_metadata[-metadata_n_rows:] = metadata_block
                                    break

                    except Exception as e:
                        self.run_details['log_file_str'] += '{}\n'.format(e)
                        #if first file to put data in time series, create variables for continuous time series
                        self.existing_data = copy.deepcopy(data)
                        self.existing_metadata = copy.deepcopy(metadata)

                    #update previous chosen file to be current chosen file
                    previous_chosen_file = copy.deepcopy(chosen_file)


    def process_manually_compiled_metadata(self):

        '''function for handling manually compiled metadata.
           Manually compiled metadata is used to set metadata fields for the entire time record when there is no previously processed valid metadata throughout the time record for a field.
           For some fields (i.e. classifications), multiple manually complied fields are processed together to return standard output.
        '''

        #determine if have manually compiled metadata for network (if there is the 'MANUAL' string in the raw external filename)
        have_external_metadata_file = any(['MANUAL' in f for f in self.network_details['external_metadata_files']])
        print(have_external_metadata_file)

        #list for keeping modified standard field names
        modified_standard_fields = []

        #parse external metadata file if exists
        if have_external_metadata_file:

            while True:
                try:
                    #read external metadata file into memory (specific for network)
                    self.external_metadata = np.load(self.run_details['data_root']+'{}/metadata/{}/processed/{}_META.npy'.format(self.run_details['network'], self.run_details['ghost_version'], self.run_details['network']), allow_pickle=True)
                except:
                    time.sleep(10.0)
                    continue
                break

            #get station reference without standard method abbreviation
            station_reference_split = self.station_reference.split('_')[:-1]
            station_reference_without_method = '_'.join(station_reference_split)

            #get external metadata for the specific station reference being processed (if none exists, then return handled error logile (2))
            self.join_field_column_name = self.network_details['metadata_data_join_variable_name']
            self.join_variable = station_reference_without_method
            external_metadata_station = self.get_join_variable_specific_external_metadata(need_variable=True)

            #load dictionary that stores mapping information for variable names between manually compiled metadata file and standard fields
            variable_mapping_dictionary = getattr(standardisation_mapping_functions, '{}_manual_external_meta_map'.format(self.run_details['network']))

            #check if any station classification information is stored within the external file, if so, package it together in a separate dictionary
            classification_standard_variables = ['area_classification', 'station_classification', 'main_emission_source', 'land_use', 'terrain', 'measurement_scale']         
            manual_native_classifications = {} 
            for k,v in list(variable_mapping_dictionary.items()):
                if v in classification_standard_variables:
                    manual_native_classifications[v] = external_metadata_station[k]
                    #remove all existing classification variables from variable_mapping_dictionary
                    del variable_mapping_dictionary[k]

            #if have some station classification information to process, then do do
            if len(manual_native_classifications) > 0:
                standard_classifications = standardisation_mapping_functions.map_network_classifications_to_standard_classifications(manual_native_classifications, self.existing_metadata, self.run_details, write_to_metadata_array=False)
                #iterate through standard classification metadata variables 
                for standard_classification_variable in list(standard_classifications.keys()):
                    #is standard classification value not NaN or empty string?
                    if (not pd.isnull(standard_classifications[standard_classification_variable])) & (standard_classifications[standard_classification_variable] != ''):
                        #if you do not have any values saved for current standard classification field, then set it through time record 
                        if all(pd.isnull(self.existing_metadata[standard_classification_variable])):
                            self.existing_metadata[standard_classification_variable] = [standard_classifications[standard_classification_variable]] * len(self.existing_metadata[standard_classification_variable])
                            #add standard variable name to modified_standard_fields list
                            modified_standard_fields.append(standard_classification_variable)
                            #if have modified any of standard station classification fields a warning should be written to inform of this
                            for inv in range(len(self.existing_metadata)):
                                self.existing_metadata['process_warnings'][inv] += 'No {} Metadata - Used Manually Compiled Metadata. '.format(metadata_modification_field_to_warning_name[standard_classification_variable])

            #iterate through rest of variables in manually compiled external metadata file
            for raw_var in list(variable_mapping_dictionary.keys()):

                #get standard variable name
                standard_var = variable_mapping_dictionary[raw_var]

                #is external metadata value not NaN or empty string?
                if (not pd.isnull(external_metadata_station[raw_var])) & (external_metadata_station[raw_var] != ''):
                    #if do not have any values saved for current standard classification field, then set through time record 
                    if all(pd.isnull(self.existing_metadata[standard_var])):
                        self.existing_metadata[standard_var] = [external_metadata_station[raw_var]] * len(self.existing_metadata[standard_var])
                        #add standard variable name to modified_standard_fields list
                        modified_standard_fields.append(standard_var)
                        #if have modified a metadata modification field, a warning should be written to inform of this
                        if standard_var in list(metadata_modification_field_to_warning_name.keys()):
                            for inv in range(len(self.existing_metadata)):
                                self.existing_metadata['process_warnings'][inv] += 'No {} Metadata - Used Manually Compiled Metadata. '.format(metadata_modification_field_to_warning_name[standard_var])

        return modified_standard_fields


    def get_join_variable_specific_external_metadata(self, need_variable=False):

        '''get external metadata specific for a certain join variable.
           if do not have data for a join variable and "need_variable" is set to "True" then stop process and return handled error logfile (2)
           if do not have data for a join variable and "need_variable" is set to "False" then return empty list.
        '''

        #first, check if you have join variable in join field column
        if self.join_variable not in self.external_metadata[self.join_field_column_name]:
            #if do not have join variable in join field column , and "need_variable" is set to "True", write handled error logfile: 2
            if need_variable:
                self.run_details['log_file_str'] += 'JOIN VARIABLE: {} DOES NOT EXIST IN EXTERNAL STATION METADATA JOIN FIELD COLUMN.'.format(self.join_variable)
                create_GHOST_stage_logfile(2,self.run_details)
            #if do not have join variable in join field column , and "need_variable" is set to "False", return empty list
            else:
                return []            
        #if have join variable in join field column, get join variable specific metadata
        else:
            join_variable_index = np.where(self.external_metadata[self.join_field_column_name] == self.join_variable)[0][0]
            return self.external_metadata[join_variable_index]


    def handle_gapped_key_metadata(self, chosen_groups=None, metadata=None):

        '''function for resolving gaps in chosen metadata fields across time record.
           if one of the metadata fields is NaN at any point in time modify them to be the most recent valid value, or if not possible the next valid value.
           for locational metadata, this stops stations being separated out due to small inconsistencies in the metadata
           it is therefore assumed that the most recent valid metadata value, or next valid value is applicable for the missing period.
           some fields are only filled if dependent fields are equivalent in the column being attempted to copy metadata from, and non NaN
           if one of the fields is deemed a 'key metadata' field, appropriate process warnings are written to inform of instances where these assumptions are made.

           if all of one of the latitude or longitude fields are NaNs, do not run function, and do not proceed with GHOST pipeline for the specific station further.

           no assumptions are also made for any fields that are highly sensitive with time (daytime_traffic_speed, daytime_passing_vehicles, population)
           fields which are always written out/handled explicitly in network parsing script are not processed (GHOST_version, station_reference, station_timezone, data_level, network, network_provided_volume_standard_temperature, network_provided_volume_standard_pressure, meta_update_stamp, data_download_stamp, data_revision_stamp, data_licence, retrieval_algorithm)
        '''

        #make copy of current metadata (if no metadata array is passed to function)
        if metadata is None:
            metadata = copy.deepcopy(self.existing_metadata)
            return_meta = False
        else:
            return_meta = True

        #get metadata update timestamps
        update_metadata_stamp = metadata['meta_update_stamp']

        #set meta groupings (and nested fields) to do check upon
        meta_groupings = {'longitude':                            ['longitude'],
                          'latitude':                             ['latitude'],
                          'altitude':                             ['altitude'],
                          'sampling_height':                      ['sampling_height'],
                          'measurement_altitude':                 ['measurement_altitude'],
                          'position_dependent':                   ['area_classification', 'station_classification', 'main_emission_source', 'land_use', 'terrain', 'measurement_scale', 'representative_radius', 
                                                                   'distance_to_building', 'distance_to_kerb', 'distance_to_junction', 'distance_to_source',
                                                                   'street_width', 'street_type', 'ellipsoid', 'horizontal_datum', 'vertical_datum', 'projection', 'climatology', 'station_name', 'city', 'country', 'associated_networks'], 
                          'primary_sampling_type':                ['primary_sampling_type','primary_sampling_process_details','primary_sampling_further_details'],
                          'primary_sampling_type_dependent':      ['primary_sampling_instrument_name'],
                          'primary_sampling_instrument_dependent':['primary_sampling_instrument_documented_flow_rate', 'primary_sampling_instrument_reported_flow_rate', 'primary_sampling_instrument_manual_name'],
                          'sample_preparation_types':             ['sample_preparation_types', 'sample_preparation_process_details', 'sample_preparation_further_details'],
                          'sample_preparation_type_dependent':    ['sample_preparation_techniques'],
                          'measurement_method':                   ['measurement_methodology', 'measuring_instrument_sampling_type', 'measuring_instrument_process_details', 'measuring_instrument_further_details', 'measuring_instrument_calibration_scale'],
                          'measurement_method_dependent':         ['measuring_instrument_name'],
                          'measuring_instrument_dependent':       ['measuring_instrument_documented_flow_rate', 'measuring_instrument_reported_flow_rate',
                                                                   'measuring_instrument_manual_name', 'measuring_instrument_reported_units',
                                                                   'measuring_instrument_reported_lower_limit_of_detection', 'measuring_instrument_documented_lower_limit_of_detection', 
                                                                   'measuring_instrument_reported_upper_limit_of_detection', 'measuring_instrument_documented_upper_limit_of_detection', 
                                                                   'measuring_instrument_reported_uncertainty', 'measuring_instrument_documented_uncertainty',
                                                                   'measuring_instrument_reported_accuracy', 'measuring_instrument_documented_accuracy',
                                                                   'measuring_instrument_reported_precision', 'measuring_instrument_documented_precision',
                                                                   'measuring_instrument_reported_zero_drift', 'measuring_instrument_documented_zero_drift',
                                                                   'measuring_instrument_reported_span_drift', 'measuring_instrument_documented_span_drift',
                                                                   'measuring_instrument_reported_zonal_drift', 'measuring_instrument_documented_zonal_drift',
                                                                   'measuring_instrument_reported_measurement_resolution', 'measuring_instrument_documented_measurement_resolution',
                                                                   'measuring_instrument_reported_absorption_cross_section', 'measuring_instrument_documented_absorption_cross_section'],                                               
                          'other':                                ['principal_investigator_name', 'principal_investigator_institution', 'principal_investigator_email_address', 
                                                                   'contact_name', 'contact_institution', 'contact_email_address',
                                                                   'measuring_instrument_inlet_information',
                                                                   'network_sampling_details', 'network_uncertainty_details', 'network_maintenance_details', 'network_qa_details', 'network_miscellaneous_details']
                          }

        meta_grouping_dependencies_default = {'position_dependent':                   ['latitude','longitude'],
                                              'primary_sampling_type_dependent':      ['primary_sampling_type'],
                                              'primary_sampling_instrument_dependent':['primary_sampling_instrument_name'],
                                              'sample_preparation_type_dependent':    ['sample_preparation_types'],
                                              'measurement_method_dependent':         ['measurement_methodology'], 
                                              'measuring_instrument_dependent':       ['measuring_instrument_name']
                                             }
                                   
        #have defined meta groups to process?
        if chosen_groups is not None:
            chosen_group_names = list(chosen_groups.keys())
        #if not process all meta groups that are non-positional (defined here)
        else:            
            chosen_group_names = ['position_dependent', 'primary_sampling_type', 'primary_sampling_type_dependent', 'primary_sampling_instrument_dependent', 
                                  'sample_preparation_types', 'sample_preparation_type_dependent', 'measurement_method', 'measurement_method_dependent', 'measuring_instrument_dependent',
                                  'other']
        #iterate through defined meta groups
        for meta_group in chosen_group_names:

            #have dependencies defined for meta group?
            if chosen_groups is not None:
                group_dependencies = chosen_groups[meta_group]
            #otherwise, get default dependencies of curent meta grouping
            else:
                if meta_group in list(meta_grouping_dependencies_default.keys()):
                    group_dependencies = meta_grouping_dependencies_default[meta_group]
                else:
                    group_dependencies = []

            #iterate through meta fields
            for meta_field in meta_groupings[meta_group]:

                #get metadata values for field
                meta_values = metadata[meta_field]

                #if all of one of the latitude or longitude fields are all NaN, then do not proceed further with function for this station.
                #return logfile to inform that station is to be removed from further processing in pipeline (1)
                if (meta_field == 'latitude') or (meta_field == 'longitude'):
                    if all(pd.isnull(meta_values)):
                        self.run_details['log_file_str'] += 'ALL OF {} VALUES ARE NaN. STATION REMOVED FROM FURTHER PROCESSING IN PIPELINE'.format(meta_field.upper())
                        create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)

                #if any one of the key metadata field values are NaN values, modify them to be the most recent valid value, or if not possible the next valid value
                #for locational metadata, this stops stations being separated out due to small inconsistencies in the metadata
                #it is therefore assumed that the most recent valid metadata value, or next valid value is applicable for the missing period.
                #some fields are only filled if dependent fields are equivalent in the column being attempted to copy metadata from, and non NaN
                #for each instance that a key metadata field value is modified, write a warning to inform of this assumption

                #continue to next key metadata field if all values are NaN, or all values non-NaN
                if (all(pd.isnull(meta_values))) or (all(~pd.isnull(meta_values))):
                    continue

                #if modifying measurement information, set relevant qa fields information for codes to copy appropriately
                if meta_field == 'primary_sampling_type':
                    relevant_qa_fields=['Assumed Primary Sampling','Unknown Primary Sampling Type','Erroneous Primary Sampling',]
                elif meta_field == 'primary_sampling_instrument_name':
                    relevant_qa_fields=['Assumed Primary Sampling','Unknown Primary Sampling Type','Unknown Primary Sampling Instrument','Erroneous Primary Sampling']
                elif meta_field == 'sample_preparation_types':
                    relevant_qa_fields=['Assumed Sample Preparation','Unknown Sample Preparation Type','Erroneous Sample Preparation']
                elif meta_field == 'sample_preparation_techniques':
                    relevant_qa_fields=['Assumed Sample Preparation','Unknown Sample Preparation Type','Unknown Sample Preparation Technique','Erroneous Sample Preparation']
                elif meta_field == 'measurement_methodology':
                    relevant_qa_fields=['Assumed Measurement Methodology','Unknown Measurement Method','Erroneous Measurement Methodology','Invalid QA Measurement Methodology']
                elif meta_field == 'measuring_instrument_name':
                     relevant_qa_fields=['Assumed Measurement Methodology','Unknown Measurement Method','Unknown Measuring Instrument','Erroneous Measurement Methodology','Invalid QA Measurement Methodology']
                else:
                    relevant_qa_fields = []             

                #set arrays for writing new metadata, and keeping metadata indices of values modified
                new_meta = copy.deepcopy(meta_values)
                invalid_inds_backward = []
                invalid_inds_forward = []

                #iterate through all values for field
                for ii,val in enumerate(meta_values):
                    #if current value is NaN, first check for most recent valid value (going backwards in time), don't check form index 0 as no values before
                    if pd.isnull(val):
                        #if not == index 0
                        if ii != 0:
                            for jj in np.arange(ii-1,-1,-1):
                                write_meta = True
                                check_valid_var_back = meta_values[jj]
                                #if have found valid value overwrite it temporally to the current time window for purposes of this function (if group dependencies are met)
                                if not pd.isnull(check_valid_var_back):
                                    #are meta grouping dependencies met (if not do not copy value)?
                                    for dependency in group_dependencies:
                                        if metadata[dependency][ii] != metadata[dependency][jj]:
                                            #if one of dependencies not met do not copy meta value
                                            write_meta = False
                                    if write_meta:
                                        #copy meta value
                                        new_meta[ii] = copy.deepcopy(check_valid_var_back)
                                        #Save relevant metadata index to write warning informing that are assuming some key metadata
                                        invalid_inds_backward.append(ii)
                                        #if modifying measurement information, copy relevant qa codes appropriately
                                        if pd.isnull(metadata['measurement_qa_flags'][ii]):
                                            current_qa_flags = []
                                        else:
                                            current_qa_flags = metadata['measurement_qa_flags'][ii].split(',')
                                        if pd.isnull(metadata['measurement_qa_flags'][jj]):
                                            previous_qa_flags = []
                                        else:
                                            previous_qa_flags = metadata['measurement_qa_flags'][jj].split(',')
                                        #iterate through relevant_qa_fields
                                        for relevant_qa_field in relevant_qa_fields:
                                            #get qa code for standard qa name
                                            qa_code = str(standard_QA_name_to_QA_code[relevant_qa_field])
                                            #is qa_code not already set in current_qa_flags, AND, qa code in previous_qa_flags?
                                            if (qa_code not in current_qa_flags) & (qa_code in previous_qa_flags):
                                                #if so, write to current field
                                                current_qa_flags.append(qa_code)
                                            #is qa_code already set in current_qa_flags, AND, qa code not in previous_qa_flags?
                                            if (qa_code in current_qa_flags) & (qa_code not in previous_qa_flags):
                                                #if so, delete qa code from current field
                                                current_qa_flags.remove(qa_code)
                                        if len(current_qa_flags) == 0:
                                            metadata['measurement_qa_flags'][ii] = np.NaN
                                        else:
                                            metadata['measurement_qa_flags'][ii] = ','.join(current_qa_flags)
                                        break

                    #if have not found valid value going backwards in time, check forwards in time
                    if pd.isnull(new_meta[ii]):
                        for jj in np.arange(ii+1,len(meta_values)):
                            write_meta = True
                            check_valid_var_forward = meta_values[jj]
                            #if have found valid value overwrite it temporally to the current time window for purposes of this function
                            if not pd.isnull(check_valid_var_forward):
                                #are meta grouping dependencies met (if not do not copy value)?
                                for dependency in group_dependencies:
                                    if metadata[dependency][ii] != metadata[dependency][jj]:
                                        #if one of dependencies not met do not copy meta value
                                        write_meta = False
                                if write_meta:
                                    #copy meta value
                                    new_meta[ii] = copy.deepcopy(check_valid_var_forward)
                                    #Save relevant metadata index to write warning informing that are assuming some key metadata
                                    invalid_inds_forward.append(ii)
                                    #if modifying measurement information, copy relevant qa codes appropriately
                                    if pd.isnull(metadata['measurement_qa_flags'][ii]):
                                        current_qa_flags = []
                                    else:
                                        current_qa_flags = metadata['measurement_qa_flags'][ii].split(',')
                                    if pd.isnull(metadata['measurement_qa_flags'][jj]): 
                                        next_qa_flags = []
                                    else:
                                        next_qa_flags = metadata['measurement_qa_flags'][jj].split(',')
                                    #iterate through relevant_qa_fields
                                    for relevant_qa_field in relevant_qa_fields:
                                        #get qa code for standard qa name
                                        qa_code = str(standard_QA_name_to_QA_code[relevant_qa_field])
                                        #is qa_code not already set in current_qa_flags, AND, qa code in next_qa_flags?
                                        if (qa_code not in current_qa_flags) & (qa_code in next_qa_flags):
                                            #if so, write to current field
                                            current_qa_flags.append(qa_code)
                                        #is qa_code already set in current_qa_flags, AND, qa code not in next_qa_flags?
                                        if (qa_code in current_qa_flags) & (qa_code not in next_qa_flags):
                                            #if so, delete qa code from current field
                                            current_qa_flags.remove(qa_code)
                                    if len(current_qa_flags) == 0:
                                        metadata['measurement_qa_flags'][ii] = np.NaN
                                    else:
                                        metadata['measurement_qa_flags'][ii] = ','.join(current_qa_flags)
                                    break

                #update metadata field, with modified data
                metadata[meta_field] = copy.deepcopy(new_meta)

                #write process warnings out (if have made modifications to key meta fields)
                if meta_field in list(metadata_modification_field_to_warning_name.keys()):
                    if len(invalid_inds_backward) > 0:
                        for inv in invalid_inds_backward:
                            metadata['process_warnings'][inv] += 'No {} Metadata - Took Most Applicable Recent Value. '.format(metadata_modification_field_to_warning_name[meta_field])

                    if len(invalid_inds_forward) > 0:
                        for inv in invalid_inds_forward:
                            metadata['process_warnings'][inv] += 'No {} Metadata - Took Most Applicable Next Value. '.format(metadata_modification_field_to_warning_name[meta_field])

        #return meta array if desired
        if return_meta:
            return metadata
        #otherwise overwrite class metadata 
        else:
            self.existing_metadata = copy.deepcopy(metadata)


    def resolve_measurement_position_accuracy(self, measurement_position_field_names=None):

        '''function for resolving simple inconsistencies in the reported measurement position metadata.
           In some cases the level of accuracy that metadata is provided by the data reporter can vary over time.
           This can be critical in the case of the measurement position, where a decimal place or two can make a significant difference in the subsequent evaluation of data from the station.
           Most of these instances are simply a case of the number of reported decimal places changing over time.
       
           To attempt to rectify the majority of these cases a 2-step procedure is undertaken to resolve all measurement position fields (latitude, longitude, altitude, sampling_height, measurement_altitude):

           1. Group all values for a measurement position field together within a certain tolerance (0.0001 degrees =~11m for longitude/latitude, 11m for altitude/sampling height/measurement altitude)
              All positions within each group will be within the tolerance of at least 1 other position e.g. (10m, 17m, 21m) would be in the same group, but (10m, 21m) would be in separate groups
              Take the weighted modal measurement position (temporal representativity as weight) in each group to be the set value for the group and overwrite the measurement position group field values throughout the time record
              Write process_warnings to inform of any assumptions made

           2. Group all values which are sub-strings of any of the other positions across the time record: e.g. 0.01 is a sub-string of 0.012322
              For each group then make the assumption the sub-string refers to the most detailed version of the position across the time record (i.e. longest str)
              If there are 2 values with the same level of detail, then take the value with largest temporal representativity. 
              Take the chosen longest str to be the set value for the group and overwrite the measurement position group field values throughout the time record
              Write process_warnings to inform of any assumptions made
        '''  
        #define measurement position field names if undefined
        if measurement_position_field_names is None:
            measurement_position_field_names = ['longitude','latitude','altitude','sampling_height','measurement_altitude']

        #iterate through all measurement position fields
        for measurement_position_field_name in measurement_position_field_names:
            #get measurement postion field values
            measurement_position_field_values = self.existing_metadata[measurement_position_field_name]
            #if all the measurement_position_field_values are NaN then continue to the next field
            if all(pd.isnull(measurement_position_field_values)):
                continue

            #Convert all NaNs to -9999 (as np.unique will otherwise class them as each as unique values)
            measurement_position_field_values[pd.isnull(measurement_position_field_values)] = -9999

            #get all unique measurement positions field values across time record 
            unique_measurement_position_field_values, unique_measurement_position_field_value_indices = np.unique(measurement_position_field_values, return_index=True)
            unique_measurement_position_field_values = [x for _,x in sorted(zip(unique_measurement_position_field_value_indices,unique_measurement_position_field_values))]

            #more than 1 unique measurement position field values across time record?
            if len(unique_measurement_position_field_values) > 1:
                #group all measurement position field values together within a certain tolerance (0.0001 degrees =~11m for longitude/latitude, 11m for altitude/sampling height)
                grouped_unique_measurement_position_field_values = []
                #set tolerance for creating grouped values
                if measurement_position_field_name in ['longitude','latitude']:
                    tolerance = 0.0001
                elif measurement_position_field_name in ['altitude','sampling_height','measurement_altitude']:
                    tolerance = 11.0
                #iterate through unique_measurement_position_field_values
                for ii, unique_measurement_position_field_value in enumerate(unique_measurement_position_field_values):  
                    #create new group
                    current_group = [unique_measurement_position_field_value]
                    #iterate through rest of unique_measurement_position_field_values
                    for jj in range(ii+1,(ii+1)+len(unique_measurement_position_field_values[ii+1:])):
                        #get next unique_measurement_position_field_value
                        next_unique_measurement_position_field_value = unique_measurement_position_field_values[jj]
                        #difference between unique_measurement_position_field_values <= tolerance?
                        if np.abs(unique_measurement_position_field_value - next_unique_measurement_position_field_value) <= tolerance:
                            #append resolved group of indices
                            current_group.append(next_unique_measurement_position_field_value)
                    #append current group values to grouped list
                    grouped_unique_measurement_position_field_values.append(current_group)

                #merge groups with common elements
                grouped_unique_measurement_position_field_values = sorted(merge_lists_sharing_common_elements(grouped_unique_measurement_position_field_values))

                #iterate through grouped unique measurement position field values and get indices of all values in full metadata array 
                unique_measurement_position_field_metadata_indices = []
                for unique_measurement_position_field_group in grouped_unique_measurement_position_field_values:
                    current_group_inds = []
                    for ii, unique_measurement_position_field in enumerate(unique_measurement_position_field_group):
                        for jj, measurement_position_field in enumerate(measurement_position_field_values):
                            if unique_measurement_position_field == measurement_position_field:
                                current_group_inds.append(jj)
                    unique_measurement_position_field_metadata_indices.append(current_group_inds)

                #get the weighted modal measurement position in each group and set that value for all the relevant metadata indices throughout the time record
                #take the length of the temporal representativity of each metadata value as a weight 
                for group_indices in unique_measurement_position_field_metadata_indices:
                    group_indices = np.array(group_indices)
                    weights = []
                    for stamp_ii in group_indices:
                        stamp_start = self.existing_metadata['meta_update_stamp'][stamp_ii]
                        if stamp_ii == (len(self.existing_metadata['meta_update_stamp'])-1):
                            stamp_end = self.existing_data['time'][-1]+int(self.existing_data['temporal_resolution'][-1])
                        else:
                            stamp_end = self.existing_metadata['meta_update_stamp'][stamp_ii+1]
                        weights.append(stamp_end-stamp_start)
                    modal_value = weighted_mode(measurement_position_field_values[group_indices],weights)
                    #set -9999 values back to NaN
                    if modal_value == -9999:
                        self.existing_metadata[measurement_position_field_name][group_indices] = np.NaN
                    else:    
                        self.existing_metadata[measurement_position_field_name][group_indices] = modal_value
                        if len(group_indices) > 1:
                            self.existing_metadata['process_warnings'][group_indices] += 'Original {} value within a certain tolerance ({}) of other value/s. Set {} to be the modal value in the group. '.format(measurement_position_field_name, tolerance, measurement_position_field_name)

                #----------------------------------------------------------------------------------#

                #Group all values which are sub-strings of any of the other positions across the time record: e.g. 0.01 is a sub-string of 0.012322
                #For each group then make the assumption the sub-string refers to the most detailed version of the position across the time record (i.e. longest str)
                #If there are 2 values with the same level of detail, then take the value with largest temporal representativity. 
                #Take the chosen longest str to be the set value for the group and overwrite the measurement position group field values throughout the time record
          
                #re-get measurement postion field values, 
                measurement_position_field_values = self.existing_metadata[measurement_position_field_name]

                #Convert all NaNs to -9999 (as np.unique will otherwise class them as each as unique values)
                measurement_position_field_values[pd.isnull(measurement_position_field_values)] = -9999

                #converting array to strings
                measurement_position_field_values = measurement_position_field_values.astype(np.str_)
            
                #get all unique measurement positions field values across time record 
                unique_measurement_position_field_values, unique_measurement_position_field_value_indices = np.unique(measurement_position_field_values, return_index=True)
                unique_measurement_position_field_values = [x for _,x in sorted(zip(unique_measurement_position_field_value_indices,unique_measurement_position_field_values))]

                #more than 1 unique measurement position field values across time record?
                if len(unique_measurement_position_field_values) > 1:

                    #group all measurement position field values that are sub-strings of one another
                    grouped_unique_measurement_position_field_values = []

                    #iterate through unique_measurement_position_field_values
                    for ii, unique_measurement_position_field_value in enumerate(unique_measurement_position_field_values):  
                        #create new group
                        current_group = [unique_measurement_position_field_value]
                        #iterate through rest of unique_measurement_position_field_values
                        for jj in range(ii+1,(ii+1)+len(unique_measurement_position_field_values[ii+1:])):
                            #get next unique_measurement_position_field_value
                            next_unique_measurement_position_field_value = unique_measurement_position_field_values[jj]
                            #unique_measurement_position_field_values substrings of one another?
                            if (unique_measurement_position_field_value in next_unique_measurement_position_field_value) or (next_unique_measurement_position_field_value in unique_measurement_position_field_value):
                                #append resolved group of indices
                                current_group.append(next_unique_measurement_position_field_value)
                        #append current group values to grouped list
                        grouped_unique_measurement_position_field_values.append(current_group)

                    #merge groups with common elements
                    grouped_unique_measurement_position_field_values = sorted(merge_lists_sharing_common_elements(grouped_unique_measurement_position_field_values))

                    #iterate through grouped unique measurement position field values and get indices of all values in full metadata array 
                    unique_measurement_position_field_metadata_indices = []
                    for unique_measurement_position_field_group in grouped_unique_measurement_position_field_values:
                        current_group_inds = []
                        for ii, unique_measurement_position_field in enumerate(unique_measurement_position_field_group):
                            for jj, measurement_position_field in enumerate(measurement_position_field_values):
                                if unique_measurement_position_field == measurement_position_field:
                                    current_group_inds.append(jj)
                        unique_measurement_position_field_metadata_indices.append(current_group_inds)

                    #get the most detailed measurement position in each group (longest str length) and set that value for all the relevant metadata indices throughout the time record
                    #if there are 2 values with the same level of detail, then take the value with largest temporal representativity 
                    for group_indices in unique_measurement_position_field_metadata_indices:
                        group_indices = np.array(group_indices)
                        field_str_len = np.array([len(value) for value in measurement_position_field_values[group_indices]])
                        max_str_inds = np.argwhere(field_str_len == np.amax(field_str_len)).flatten().tolist()
                        if len(max_str_inds) > 1:
                            weights =  []
                            for stamp_ii in max_str_inds:
                                stamp_start = self.existing_metadata['meta_update_stamp'][group_indices[stamp_ii]]
                                if group_indices[stamp_ii] == (len(self.existing_metadata['meta_update_stamp'])-1):
                                    stamp_end = self.existing_data['time'][-1]+int(self.existing_data['temporal_resolution'][-1])
                                else:
                                    stamp_end = self.existing_metadata['meta_update_stamp'][group_indices[stamp_ii]+1]
                                weights.append(stamp_end-stamp_start)
                            max_str_ind = max_str_inds[np.argmax(weights)]
                        else:
                            max_str_ind = max_str_inds[0]
                        most_accurate_value = measurement_position_field_values[group_indices[max_str_ind]].astype(self.existing_metadata[measurement_position_field_name].dtype)
                        #set -9999 values back to NaN
                        if most_accurate_value == -9999:
                            self.existing_metadata[measurement_position_field_name][group_indices] = np.NaN
                        else:    
                            self.existing_metadata[measurement_position_field_name][group_indices] = most_accurate_value
                            if len(group_indices) > 1:
                                self.existing_metadata['process_warnings'][group_indices] += 'Original {} value is sub-string of other value/s. Set {} to be the most detailed value in the group. '.format(measurement_position_field_name, measurement_position_field_name)

    def set_altitude_field(self,field_to_set,check_validity=None):
        '''function that ensures a defined altitude fields is set.
           for each field only set the field if the other 2 related fields are set
           for altitude, the sampling_height and measurement_altitude need to be set
           for measurement_altitude, the altitude and sampling_height need to be set
           for sampling_height, the altitude and measurement_altitude need to be set
          
           write a process_warning for any fields that are set.

           if the check_validity option is not None, and if the needed altitude fields are set and do not sum correctly, 
           then reset/recalculate the set field/associated fields dependent on passed option
        '''

        #define needed valid fields to set defined altitude field
        if field_to_set == 'altitude':
            needed_fields = ['sampling_height','measurement_altitude']
        elif field_to_set == 'sampling_height':
            needed_fields = ['altitude','measurement_altitude'] 
        elif field_to_set == 'measurement_altitude':
            needed_fields = ['altitude','sampling_height']

        #iterate through altitude field to set
        for meta_ii in range(len(self.existing_metadata[field_to_set])):

            #field_to_set == NaN?
            if pd.isnull(self.existing_metadata[field_to_set][meta_ii]):
                nan_field_to_set = True
            else:
                nan_field_to_set = False 

            #if check validity is not None, then check if have all 3 valid altitude fields
            if check_validity is not None:
                check_validity_active = True
                for field in ['altitude','sampling_height','measurement_altitude']:
                    if pd.isnull(self.existing_metadata[field][meta_ii]):
                        check_validity_active = False
            else:
                check_validity_active = False

            #don't have a current_value or check_validity active?
            if (nan_field_to_set) or (check_validity_active):
                #iterate through needed fields and determine if have valid values
                have_needed_fields = True
                for needed_field in needed_fields:
                    if pd.isnull(self.existing_metadata[needed_field][meta_ii]):
                        have_needed_fields = False
                #have needed fields? then set defined altitude field
                #also set process warning
                #if check_validity is not None then check if needed fields sum correctly,
                #if not, then reset/recalculate the set field/associated fields dependent on passed option
                if have_needed_fields:
                    
                    if field_to_set == 'altitude':
                        if check_validity is not None:
                            sum_validity = self.existing_metadata[field_to_set][meta_ii] == (self.existing_metadata['measurement_altitude'][meta_ii] - self.existing_metadata['sampling_height'][meta_ii]) 
                            if sum_validity == False:
                                if check_validity == 'reset':
                                    if nan_field_to_set == False:
                                        self.existing_metadata['sampling_height'][meta_ii] = np.NaN
                                        self.existing_metadata['measurement_altitude'][meta_ii] = np.NaN
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Altitude fields do not sum correctly. Redefining Sampling Height and Measurement Altitude fields to be NaN. '
                                elif check_validity == 'recalculate':        
                                    self.existing_metadata[field_to_set][meta_ii] = self.existing_metadata['measurement_altitude'][meta_ii] - self.existing_metadata['sampling_height'][meta_ii]
                                    if nan_field_to_set:
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Altitude assumed to be Measurement Altitude - Sampling Height. '    
                                    else:
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Altitude fields do not sum correctly. Redefining Altitude field to be Measurement Altitude - Sampling Height. '    
                        else:
                            self.existing_metadata[field_to_set][meta_ii] = self.existing_metadata['measurement_altitude'][meta_ii] - self.existing_metadata['sampling_height'][meta_ii]
                            self.existing_metadata['process_warnings'][meta_ii] += 'Altitude assumed to be Measurement Altitude - Sampling Height. '
                    
                    elif field_to_set == 'sampling_height':
                        if check_validity is not None:
                            sum_validity = self.existing_metadata[field_to_set][meta_ii] == (self.existing_metadata['measurement_altitude'][meta_ii] - self.existing_metadata['altitude'][meta_ii]) 
                            if sum_validity == False:
                                if check_validity == 'reset':
                                    if nan_field_to_set == False:
                                        self.existing_metadata['altitude'][meta_ii] = np.NaN
                                        self.existing_metadata['measurement_altitude'][meta_ii] = np.NaN
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Altitude fields do not sum correctly. Redefining Altitude and Measurement Altitude fields to be NaN. '
                                elif check_validity == 'recalculate':        
                                    self.existing_metadata[field_to_set][meta_ii] = self.existing_metadata['measurement_altitude'][meta_ii] - self.existing_metadata['altitude'][meta_ii]
                                    if nan_field_to_set:
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Sampling Height assumed to be Measurement Altitude - Altitude. '
                                    else:
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Altitude fields do not sum correctly. Redefining Sampling Height field to be Measurement Altitude - Altitude. '    
                        else:
                            self.existing_metadata[field_to_set][meta_ii] = self.existing_metadata['measurement_altitude'][meta_ii] - self.existing_metadata['altitude'][meta_ii] 
                            self.existing_metadata['process_warnings'][meta_ii] += 'Sampling Height assumed to be Measurement Altitude - Altitude. '
                    
                    elif field_to_set == 'measurement_altitude':
                        if check_validity is not None:
                            sum_validity = self.existing_metadata[field_to_set][meta_ii] == (self.existing_metadata['altitude'][meta_ii] + self.existing_metadata['sampling_height'][meta_ii]) 
                            if sum_validity == False:
                                if check_validity == 'reset':
                                    if nan_field_to_set == False:
                                        self.existing_metadata['altitude'][meta_ii] = np.NaN
                                        self.existing_metadata['sampling_height'][meta_ii] = np.NaN
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Altitude fields do not sum correctly. Redefining Altitude and Sampling Height fields to be NaN. '
                                elif check_validity == 'recalculate':        
                                    self.existing_metadata[field_to_set][meta_ii] = self.existing_metadata['altitude'][meta_ii] + self.existing_metadata['sampling_height'][meta_ii]
                                    if nan_field_to_set:
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Measurement Altitude assumed to be Altitude + Sampling Height. '
                                    else:
                                        self.existing_metadata['process_warnings'][meta_ii] += 'Altitude fields do not sum correctly. Redefining Measurement Altitude field to be Altitude + Sampling Height. '    
                        else:
                            self.existing_metadata[field_to_set][meta_ii] = self.existing_metadata['altitude'][meta_ii] + self.existing_metadata['sampling_height'][meta_ii] 
                            self.existing_metadata['process_warnings'][meta_ii] += 'Measurement Altitude assumed to be Altitude + Sampling Height. '



    def check_dependent_xy_position(self):
    
        '''function that performs a check on the interdependency between the xy measurement position after the resolution of the accuracy inconsistencies in each field.
           in some cases the longitude or latitude changes, but the other associated longitude / latitude does not.
           it is extremely rare that one of the xy coordinate moves, yet the other stays exactly the same.
           
           therefore wall instances are grouped where a longitude changes yet the latitude stays the same, and vice versa. 
           1 longitude, and 1 latitude, are selected for each grouping associated with the largest temporal representation through the time record, 
           overwriting the other grouped values through the metadata record.  

           write a process_warning for any fields that are overwritten.        
        '''

        #get required metadata
        lons = self.existing_metadata['longitude']
        lats = self.existing_metadata['latitude']

        #get all unique pairs of longitudes and latitudes across time record
        all_measurement_positions = np.array([[lon, lats[ii]] for ii,lon in enumerate(lons)])
        unique_measurement_positions, unique_measurement_positions_indices = np.unique(all_measurement_positions, axis=0, return_index=True)
        unique_measurement_positions = [x for _,x in sorted(zip(unique_measurement_positions_indices,unique_measurement_positions))]
        
        #if only 1 unique position, return
        if len(unique_measurement_positions) == 1:
            return
        
        #get independent longitudes/latitudes in unique lon/lat pairs
        unique_measurement_position_lons = [x[0] for x in unique_measurement_positions]
        unique_measurement_position_lats = [x[1] for x in unique_measurement_positions]

        #count how many times each lon/lat occurs in unique lon/lat_pairs
        count_lons = Counter(unique_measurement_position_lons)
        count_lats = Counter(unique_measurement_position_lats)

        #log all longitude/latitudes that occur more than once in lon/lat pairs 
        repeating_lons = [lon for lon, lon_count in count_lons.items() if lon_count > 1]                
        repeating_lats = [lat for lat, lat_count in count_lats.items() if lat_count > 1]  

        #if no repeating lons or lats, then return
        if (len(repeating_lons) == 0) & (len(repeating_lats) == 0):
            return

        #iterate through repeating longitudes and get associated shifting latitudes  
        shifting_lat_groups = []
        for repeating_lon in repeating_lons:
            shifting_lat_groups.append([x[1] for x in unique_measurement_positions if x[0] == repeating_lon]) 

        #iterate through repeating latitudes and get associated shifting longitudes  
        shifting_lon_groups = []
        for repeating_lat in repeating_lats:
            shifting_lon_groups.append([x[0] for x in unique_measurement_positions if x[1] == repeating_lat]) 

        #iterate through each shifting longitude grouping and select longitude associated with greatest temporal representativity
        #overwrite non-selected longitudes with chosen longitude, writing process warning.
        for position_ii, shifting_lon_group in enumerate(shifting_lon_groups):
            lon_rep_count = {lon:0 for lon in shifting_lon_group}
            #get temporal representation count of all lons in group
            for shifting_lon in shifting_lon_group:                             
                for meta_ii in range(len(self.existing_metadata['meta_update_stamp'])):
                    if (repeating_lats[position_ii] == self.existing_metadata['latitude'][meta_ii]) & (shifting_lon == self.existing_metadata['longitude'][meta_ii]):
                        stamp_start = self.existing_metadata['meta_update_stamp'][meta_ii]
                        if meta_ii == (len(self.existing_metadata['meta_update_stamp'])-1):
                            stamp_end = self.existing_data['time'][-1]+int(self.existing_data['temporal_resolution'][-1])
                        else:
                            stamp_end = self.existing_metadata['meta_update_stamp'][meta_ii+1]
                        lon_rep_count[shifting_lon] += stamp_end-stamp_start
            #select longitude with biggest temporal representation in group, and overwrite other longitudes in group
            selected_lon = max(lon_rep_count, key=lon_rep_count.get)
            for shifting_lon in shifting_lon_group:             
                if shifting_lon != selected_lon:                
                    for meta_ii in range(len(self.existing_metadata['meta_update_stamp'])): 
                        if (repeating_lats[position_ii] == self.existing_metadata['latitude'][meta_ii]) & (shifting_lon == self.existing_metadata['longitude'][meta_ii]):
                            self.existing_metadata['longitude'][meta_ii] = selected_lon   
                            self.existing_metadata['process_warnings'][meta_ii] += 'Original longitude changes through record when associated latitude does not. Set longitude to be that most represented through the time record (of the relevant longitudes). '

        #iterate through each shifting latitude grouping and select latitude associated with greatest temporal representativity
        #overwrite non-selected latitude with chosen latitude, writing process warning.
        for position_ii, shifting_lat_group in enumerate(shifting_lat_groups):
            lat_rep_count = {lat:0 for lat in shifting_lat_group}
            #get temporal representation count of all lats in group
            for shifting_lat in shifting_lat_group:                             
                for meta_ii in range(len(self.existing_metadata['meta_update_stamp'])):
                    if (repeating_lons[position_ii] == self.existing_metadata['longitude'][meta_ii]) & (shifting_lat == self.existing_metadata['latitude'][meta_ii]):
                        stamp_start = self.existing_metadata['meta_update_stamp'][meta_ii]
                        if meta_ii == (len(self.existing_metadata['meta_update_stamp'])-1):
                            stamp_end = self.existing_data['time'][-1]+int(self.existing_data['temporal_resolution'][-1])
                        else:
                            stamp_end = self.existing_metadata['meta_update_stamp'][meta_ii+1]
                        lat_rep_count[shifting_lat] += stamp_end-stamp_start
            #select latitude with biggest temporal representation in group, and overwrite other latitudes in group
            selected_lat = max(lat_rep_count, key=lat_rep_count.get)
            for shifting_lat in shifting_lat_group:             
                if shifting_lat != selected_lat:                
                    for meta_ii in range(len(self.existing_metadata['meta_update_stamp'])): 
                        if (repeating_lons[position_ii] == self.existing_metadata['longitude'][meta_ii]) & (shifting_lat == self.existing_metadata['latitude'][meta_ii]):
                            self.existing_metadata['latitude'][meta_ii] = selected_lat   
                            self.existing_metadata['process_warnings'][meta_ii] += 'Original latitude changes through record when associated longitude does not. Set latitude to be that most represented through the time record (of the relevant latitudes). '



    def resolve_duplicate_metadata(self, metadata=None):

        '''function that removes duplicate metadata columns
           this function checks column by column and if all key metadata metadata values are identical to the last column, the column is removed.
        '''

        #make copy of current metadata (if no metadata array is passed to function)
        if metadata is None:
            metadata = copy.deepcopy(self.existing_metadata)
            return_meta = False
        else:
            return_meta = True
        
        #if number of metadata column is already 1, do not need to run function
        if len(metadata) == 1:
            #return meta array if desired
            if return_meta:
                return metadata
            #otherwise overwrite class metadata 
            else:
                self.existing_metadata = copy.deepcopy(metadata)
        else:
            #initialise list to store metadata columns to remove
            metadata_column_indices_to_remove = np.array([], dtype=np.uint32)

            #iterate through metadata columns, check if one of key fields are different from previous column
            previous_metadata_column = copy.deepcopy(metadata[0])
            for metadata_column_ii in range(1, 1+len(metadata[1:])):
                metadata_column = metadata[metadata_column_ii]

                #has one of key metadata values changed between columns?
                has_column_changed = determine_key_changing_metadata(previous_metadata_column, metadata_column)
            
                #if not append column index to remove
                if has_column_changed == False:
                    metadata_column_indices_to_remove = np.append(metadata_column_indices_to_remove, metadata_column_ii)
                #otherwise take current column as previous column to test against
                else:
                    previous_metadata_column = copy.deepcopy(metadata_column)

            #if have metadata columns to remove, do so now
            if len(metadata_column_indices_to_remove) > 0:
                metadata = np.delete(metadata, metadata_column_indices_to_remove)

            #return meta array if desired
            if return_meta:
                return metadata
            #otherwise overwrite class metadata 
            else:
                self.existing_metadata = copy.deepcopy(metadata)


    def changing_measurement_position_new_stations(self):

        '''check if key metadata (Latitude, Longitude, Measurement Altitude (Altitude + Sampling Height) changes significantly through time record.
           Split station data if necessary into separate stations on the occasions of significant changes in key metadata.

           significant change in Latitude/Longitude coordinates is defined to be >= 0.0001 degrees (~11m)
           significant change in Measurement Altitude is defined to be >= 11m.

           check that Latitude, Longitude, Measurement Altitude does not change then revert back at later date.
           therefore group different collections of Latitudes, Longitudes & Measurement Altitudes together through entire time record - outputting these groupings as 1 unique station,
           this prevents writing of multiple new stations needlessly.
           all the individual latitudes/longitudes/altitudes values in each group will be within the defined tolerance of at least one other value in the group. 
           for example altitudes of 0.0, 6.0, 15.0, 22.0 would all be in the same group, but altitudes of 0.0 and 15.0 would be in separate groups.

           finally split out data/metadata into new stations if required - appending each new split station reference with a --S%N string

           unique station groupings with < 31 days in total data extent are dropped.
           if there are more than 5 valid unique station groupings, then no data at all is written out, as it is not considered trustworthy data. 

           if there are too many metadata columns per total days (<=90.0 days per column) for a station then it is not written out, as metadata is too variable to be trusted.

        '''

        self.run_details['log_file_str'] += 'EVALUATING VARIABILITY OF REPORTED MEASUREMENT POSITION.\n'

        #tolerance limit for changing significant lat/lon coordinate changes
        latlon_tolerance = 0.0001 #~11m
        #tolerance limit for changing significant measurement altitude changes
        measurement_alt_tolerance = 11.0 #11m

        #load required metadata
        lats = self.existing_metadata['latitude']
        lons = self.existing_metadata['longitude']
        measurement_alts = self.existing_metadata['measurement_altitude']
        update_metadata_stamp = self.existing_metadata['meta_update_stamp']

        #if all measurement altitudes are NaN, set them all to be -99999 for this function 
        if all(pd.isnull(measurement_alts)):
            measurement_alts = np.full(len(lats), -99999)

        #get all unique measurement positions across time record (i.e. longitude, latitude & measurement altitude)
        all_measurement_positions = np.array([[lon, lats[ii], measurement_alts[ii]] for ii,lon in enumerate(lons)])
        unique_measurement_positions, unique_measurement_positions_indices = np.unique(all_measurement_positions, axis=0, return_index=True)
        unique_measurement_positions = [x for _,x in sorted(zip(unique_measurement_positions_indices,unique_measurement_positions))]

        #if have just 1 unique measurement position, write current station data/metadata to new file
        if len(unique_measurement_positions) == 1:

            #get total data extent of data for unique measurement position, in days 
            data_extent = ((self.existing_data['time'][-1]+int(self.existing_data['temporal_resolution'][-1])) - self.existing_data['time'][0]) / 60.0 / 24.0 

            #if data extent is < 31 days then do not write file for station --> station does not continue in pipeline
            if data_extent < 31.0:
                self.run_details['log_file_str'] += 'NO VALID DATA TO WRITE OUT AFTER REMOVING UNIQUE STATION DATA WITH EXTENT < 31 DAYS.\nSTATION WILL NOT CONTINUE IN PIPELINE.'
                create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)
            else:
                #fill gapped values through column of all non-positional fields (default dependencies set inside function)
                self.handle_gapped_key_metadata()

                #remove duplicate metadata columns
                self.resolve_duplicate_metadata()
                
                #force 1 column of metadata to be written for each unique UTC year
                write_station = self.write_metadata_per_year()

                #if too many metadata columns per total days (<=90.0 days per column) then do not write out station as metadata is too variable to be trusted
                if write_station:
                    #write data/metadata to new file
                    write_station_specific_numpy(self.existing_metadata,self.existing_data,self.run_details,self.parameter_details)
                else:
                    self.run_details['log_file_str'] += 'TOO MANY METADATA COLUMNS PER DAYS (<= 90.0 DAYS PER COLUMN). THE METADATA FROM THIS STATION IS NOT CONSIDERED TRUSTWORTHY, AND THIS STATION DOES NOT CONTINUE IN THE PIPELINE.'
                    create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)

        #else, if have more than 1 unique measurement position, evaluate if they are significantly different
        else:

            #group all unique measurement positions that are not significantly different 
            #evaluate all unique positions with all over positions
            #if there are multiple groups with common positions, then merge these groups
            grouped_unique_measurement_positions = []
            for ii, unique_measurement_position in enumerate(unique_measurement_positions):
                #if ii not in grouped_measurement_positions:
                current_group = [tuple(unique_measurement_position)]
                for jj in range(ii+1, (ii+1)+len(unique_measurement_positions[ii+1:])):
                    next_unique_measurement_position = unique_measurement_positions[jj]
                    lon_diff = np.abs(next_unique_measurement_position[0] - unique_measurement_position[0])
                    lat_diff = np.abs(next_unique_measurement_position[1] - unique_measurement_position[1])
                    measurement_alt_diff = np.abs(next_unique_measurement_position[2] - unique_measurement_position[2])
                    if (lon_diff < latlon_tolerance) & (lat_diff < latlon_tolerance) & (measurement_alt_diff < measurement_alt_tolerance):
                        current_group.append(tuple(next_unique_measurement_position))
                grouped_unique_measurement_positions.append(current_group)

            #merge groups with common elements
            grouped_unique_measurement_positions = sorted(merge_lists_sharing_common_elements(grouped_unique_measurement_positions))
            #convert tuples back to groups inside np.array
            for group_ii, group in enumerate(grouped_unique_measurement_positions):
                new_group = []
                for measurement_position in group:
                    new_group.append(list(measurement_position))
                grouped_unique_measurement_positions[group_ii] = new_group
            grouped_unique_measurement_positions = np.array(grouped_unique_measurement_positions)

            #iterate through grouped unique measurement positions and get indices of all relevant metadata 
            unique_measurement_position_metadata_indices = []
            for unique_measurement_positions_group in grouped_unique_measurement_positions:
                current_group_inds = []
                for ii, unique_measurement_position in enumerate(unique_measurement_positions_group):
                    for jj, measurement_position in enumerate(all_measurement_positions):
                        if np.array_equal(unique_measurement_position, measurement_position):
                            current_group_inds.append(jj)
                unique_measurement_position_metadata_indices.append(current_group_inds)
               
            #if have no significantly different measurement positions, write current station data/metadata to new file
            if len(unique_measurement_position_metadata_indices) == 1:        

                #get total data extent of data for unique measurement position, in days 
                data_extent = ((self.existing_data['time'][-1]+int(self.existing_data['temporal_resolution'][-1])) - self.existing_data['time'][0]) / 60.0 / 24.0 

                #if data extent is < 31 days then do not write file for station --> station does not continue in pipeline
                if data_extent < 31.0:
                    self.run_details['log_file_str'] += 'NO VALID DATA TO WRITE OUT AFTER REMOVING UNIQUE STATION DATA WITH EXTENT < 31 DAYS.\nSTATION WILL NOT CONTINUE IN PIPELINE.'
                    create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)
                else:
                    #fill gapped values through column of all non-positional fields (default dependencies set inside function)
                    self.handle_gapped_key_metadata()

                    #remove duplicate metadata columns
                    self.resolve_duplicate_metadata()

                    #force 1 column of metadata to be written for each unique UTC year
                    write_station = self.write_metadata_per_year()

                    #if too many metadata columns per total days (<=90.0 days per column) then do not write out station as metadata is too variable to be trusted
                    if write_station:
                        #write data/metadata to new file
                        write_station_specific_numpy(self.existing_metadata,self.existing_data,self.run_details,self.parameter_details)
                    else:
                        self.run_details['log_file_str'] += 'TOO MANY METADATA COLUMNS PER DAYS (<= 90.0 DAYS PER COLUMN). THE METADATA FROM THIS STATION IS NOT CONSIDERED TRUSTWORTHY, AND THIS STATION DOES NOT CONTINUE IN THE PIPELINE.'
                        create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)
  
            #else, if have significantly different measurement positions, then split out metadata/data into different stations
            #the first written station will retain the reference of the original station
            elif len(unique_measurement_position_metadata_indices) > 1:
 
                self.run_details['log_file_str'] += 'SPLITTING STATION INTO DIFFERENT STATIONS.\n'
 
                #create dictionary to store data/metadata indices for valid new stations to write
                valid_station_indices = {}

                #iterate through unique measurement positions, and evaluate if for each measurement position the data extent is sufficient to write out (>= 31 days) 
                for metadata_indices in sorted(unique_measurement_position_metadata_indices):
                
                    #get count of current written N split stations
                    N_stations_written = len(list(valid_station_indices.keys()))

                    #get relevant data indices associated with unique measurement position grouping
                    #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period 
                    data_indices = []
                    for metadata_index in metadata_indices: 
                        start_data_ind = bisect.bisect_left(self.existing_data['time'], update_metadata_stamp[metadata_index])
                        if (metadata_index + 1) == len(update_metadata_stamp):
                            end_data_ind = len(self.existing_data['time'])
                        else: 
                            end_data_ind = bisect.bisect_left(self.existing_data['time'], update_metadata_stamp[metadata_index+1])
                        if (start_data_ind == end_data_ind) & (start_data_ind == len(self.existing_data['time'])):
                            pass
                        elif start_data_ind == end_data_ind:
                            data_indices.extend([start_data_ind])
                        else:
                            data_indices.extend(range(start_data_ind, end_data_ind))
                    data_indices = sorted(data_indices)

                    #get total data extent of data for unique measurement position, in days 
                    data_extent = ((self.existing_data['time'][data_indices[-1]]+int(self.existing_data['temporal_resolution'][data_indices[-1]])) - self.existing_data['time'][data_indices[0]]) / 60.0 / 24.0  

                    #if data extent is >= 31 days then add data/metadata indices to valid_station_indices dict
                    if data_extent >= 31.0:
                        #if there more than 5 unique measurement positions to split data across, do not continue processing for station as the station is either a mobile station, or the data is simply not trusted from a station whose measurement position changes so much 
                        if N_stations_written == 5: 
                            self.run_details['log_file_str'] += 'MORE THAN 5 UNIQUE VALID MEASUREMENT POSITIONS TO SPLIT DATA ACROSS.\nTHE DATA FROM THIS STATION IS NOT CONSIDERED TRUSTWORTHY, AND THIS STATION DOES NOT CONTINUE IN THE PIPELINE.'
                            create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)
                        else:
                            valid_station_indices[str(N_stations_written)] = {'data_indices':data_indices, 'metadata_indices':metadata_indices}                               

                #have no valid data to write?
                if len(list(valid_station_indices.keys())) == 0:
                    self.run_details['log_file_str'] += 'NO VALID DATA TO WRITE OUT AFTER REMOVING UNIQUE STATION DATA WITH EXTENT < 31 DAYS.\nSTATION WILL NOT CONTINUE IN PIPELINE.'
                    create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)
                #have valid data to write?
                else:
                    station_ii = 0

                    #iterate through valid_station_indices dict
                    for unique_measurement_position_ii in sorted(list(valid_station_indices.keys())):

                        #cut new metadata/data from original numpy dataframe
                        cut_metadata = self.existing_metadata[valid_station_indices[unique_measurement_position_ii]['metadata_indices']]
                        cut_data = self.existing_data[valid_station_indices[unique_measurement_position_ii]['data_indices']]

                        #change station reference metadata (if not first station)
                        if station_ii > 0:
                            for qq in range(len(cut_metadata['station_reference'])):
                                cut_metadata['station_reference'][qq] = '{}--S{}'.format(self.station_reference,station_ii)

                        #fill gapped values through column of all non-positional fields (default dependencies set inside function)
                        cut_metadata = self.handle_gapped_key_metadata(metadata=cut_metadata)

                        #remove duplicate metadata columns
                        cut_metadata = self.resolve_duplicate_metadata(metadata=cut_metadata)

                        #force 1 column of metadata to be written for each unique UTC year
                        cut_metadata, write_station = self.write_metadata_per_year(metadata=cut_metadata, data=cut_data)
                        #if too many metadata columns per total days (<=90.0 days per column) then do not write out station as metadata is too variable to be trusted
                        if write_station:
                            #save cut data/metadata to new file
                            write_station_specific_numpy(cut_metadata,cut_data,self.run_details,self.parameter_details)
                            station_ii +=1

                    if station_ii == 0:
                        self.run_details['log_file_str'] += 'TOO MANY METADATA COLUMNS PER DAYS (<= 90.0 DAYS PER COLUMN). THE METADATA FROM THIS STATION IS NOT CONSIDERED TRUSTWORTHY, AND THIS STATION DOES NOT CONTINUE IN THE PIPELINE.'
                        create_GHOST_stage_logfile(1,self.run_details,previous_stage_data_files=self.data_files)


    def write_metadata_per_year(self, metadata=None, data=None):

        '''iterate through metadata columns and ensure there is a metadata column for each UTC year.
           this is done by iterating through each metadata column and breaking it across the UTC new year boundary.
           a new metadata chunk is created from the start of the UTC year
        ''' 

        #make copy of current metadata (if no metadata array is passed to function)
        if metadata is None:
            metadata = copy.deepcopy(self.existing_metadata)
            data = copy.deepcopy(self.existing_data)
            return_meta = False
        else:
            return_meta = True

        modified_metadata = copy.deepcopy(metadata[:1])

        #iterate through metadata column
        for meta_ii, metadata_column in enumerate(metadata):

            current_meta_start = metadata_column['meta_update_stamp']
            #last metadata field?
            if meta_ii == (len(metadata)-1):
                current_meta_end = data['time'][-1]+int(data['temporal_resolution'][-1])
            #not last metadata field?
            else:
                current_meta_end = metadata['meta_update_stamp'][meta_ii+1]

            #get start and end metadata periods as datetimes
            start_datetime = fractional_minutes_from_reference_to_datetime(float(current_meta_start))
            end_datetime = fractional_minutes_from_reference_to_datetime(float(current_meta_end))

            #first metadata column?
            if meta_ii == 0:
                first_datetime = copy.deepcopy(start_datetime)
            #last metadata column?
            if meta_ii == (len(metadata)-1):
                last_datetime = copy.deepcopy(end_datetime)

            #get all years that the metadata chunk covers
            if (end_datetime.month == 1) & (end_datetime.day == 1) & (end_datetime.hour == 0) & (end_datetime.minute == 0):
                relevant_years = np.arange(start_datetime.year,end_datetime.year) 
            else:
                relevant_years = np.arange(start_datetime.year,end_datetime.year+1) 

            #append current metadata column to modified metadata array (if not on first iteration)
            if meta_ii > 0:
                modified_metadata = np.resize(modified_metadata, len(modified_metadata)+1)
                modified_metadata[-1] = metadata_column

            #if there is more than 1 year covered, then split metadata across all years (after the first)
            if len(relevant_years) > 1:
                years_to_chunk_for = relevant_years[1:]

                #iterate through years to chunk out for
                for year in years_to_chunk_for:
    
                    #get start of year datetime in minutes from reference
                    start_of_year_stamp = datetime_to_fractional_minutes_from_reference(datetime.datetime(year, 1, 1, 0, 0))
                    #duplicate current metadata column and append it starting from start of UTC year
                    modified_metadata = np.resize(modified_metadata, len(modified_metadata)+1)
                    modified_metadata[-1] = metadata_column
                    modified_metadata['meta_update_stamp'][-1] = start_of_year_stamp

        #check n_minutes to number of metadata columns
        metadata_total_secs = (last_datetime - first_datetime).total_seconds()
        metadata_total_days = metadata_total_secs / 60.0 / 60.0 / 24.0
        #divide total days in metadata columns by number of metadata columns
        n_days_per_column = metadata_total_days / len(metadata)
        #if n_days_per_column is <= 90.0 days, and total days > 90 days, then return instruction to NOT write out station as metadata is too variable
        if (n_days_per_column <= 90.0) & (metadata_total_days > 90.0):
            write_station = False
        else:
            write_station = True

        #return modified meta array if desired
        if return_meta:
            return modified_metadata, write_station
        #otherwise overwrite class metadata 
        else:
            self.existing_metadata = copy.deepcopy(modified_metadata)
            return write_station