import bisect
from collections import Counter
import copy
import datetime
import itertools
import re

import numpy as np
import numpy.lib.recfunctions as rf
import pandas as pd
import scipy.stats as st
import statsmodels.stats.stattools as sm

from crossnetwork_processing_functions import normalise_by_defined_max, return_PDF_intersection, filter_by_qa, kde_fft
from GHOST_standards import standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code
from time_functions import datetime_to_fractional_minutes_from_reference, fractional_minutes_from_reference_to_datetime


class QualityControl:

    '''Class which applies data quality checks to station data.
       These checks generate quality assurance codes per measurement.
    '''

    def __init__(self, process_instance):

        #add passed instance variables to self
        self.station_reference = process_instance.station_reference
        self.station_numpy_filename = process_instance.station_numpy_filename
        self.metadata = process_instance.metadata
        self.data = process_instance.data
        self.run_details = process_instance.run_details
        self.parameter_details = process_instance.parameter_details
        self.network_details = process_instance.network_details

        self.run_details['log_file_str'] += 'QUALITY CONTROL\n'


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #get reported/derived uncertainty per measurement
        self.process_reported_and_derived_uncertainty_per_measurement()

        #missing measurements check
        self.check_missing_measurement()
    
        #negative measurement check 
        self.check_negative_measurement()
    
        #zero values check 
        self.check_zero_values()
    
        #infinite values check
        self.check_infinite_values()

        #station position check
        self.check_station_position()

        #measurement qa check
        self.check_measurement_qa()

        #data level check
        self.check_data_level()

        #preliminary data check
        self.check_preliminary_data()

        #local precipitation check
        self.check_local_precipitation()

        #local extreme weather check
        self.check_local_extreme_weather()

        #local atmospheric obscuration check
        self.check_local_atmospheric_obscuration()
    
        #local contamination check 
        self.check_local_contamination()

        #local exceptional event check
        self.check_local_exceptional_event()
    
        #LOD checks 
        self.check_lod()
    
        #data outlier - scientifically non-feasible check
        self.check_data_outlier_scientifically_non_feasible()
    
        #data outlier - network decreed check
        self.check_data_outlier_network()

        #data outlier - manually decreed check
        self.check_data_outlier_manual()
    
        #measurement resolution check - by reported metadata
        self.check_measurement_resolution_by_metadata()

        #recurring value check
        self.check_recurring_values()
    
        #run qa checks which require grouping by temporal_resolution/temporal window
        #the qa checks called are: 
        #non-integer local timezone check
        #empirical measurement resolution check
        #monthly adjusted boxplot check
        #data outlier - scientifically non-feasible monthly median check
        #monthly distribution consistency check
        self.grouped_measurement_checks()


    def process_reported_and_derived_uncertainty_per_measurement(self):

        '''function processes reported uncertainty per measurement from metadata reported (or if not available, documented) uncertainties, if not already given per measurement by network.

           function also derives +/- uncertainty on each measurement from the reported (or if not available, documented) accuracy and precision metadata.
           precision and accuracy are added in quadrature to generate uncertainties per measurement.
           calculation is only done if have both valid precision and accuracy information
        '''

        #get key data
        reported_uncertainty_per_measurement = self.data['reported_uncertainty_per_measurement']

        #get key metadata
        reported_uncertainty = self.metadata['measuring_instrument_reported_uncertainty']
        documented_uncertainty = self.metadata['measuring_instrument_documented_uncertainty']
        reported_precision = self.metadata['measuring_instrument_reported_precision']
        documented_precision = self.metadata['measuring_instrument_documented_precision']
        reported_accuracy = self.metadata['measuring_instrument_reported_accuracy']
        documented_accuracy = self.metadata['measuring_instrument_documented_accuracy']
        update_metadata_stamp = self.metadata['meta_update_stamp']

        #------------------------------------------------------------#
        #process reported uncertainties

        #iterate through each metadata parameter
        for ii in range(len(reported_uncertainty)):

            #do you have valid reported metadata uncertainty?
            if not pd.isnull(reported_uncertainty[ii]):
                uncertainty = reported_uncertainty[ii]
            #do you have valid documented metadata uncertainty?
            elif not pd.isnull(documented_uncertainty[ii]):
                uncertainty = documented_uncertainty[ii]
            #no valid uncertainty value, continue to next metadata index
            else:
                continue

            #get relevant range of inds to update reported_uncertainty_per_measurement data field
            #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period
            start_ind = bisect.bisect_left(self.data['time'], update_metadata_stamp[ii])
            if ii == (len(reported_precision)-1):
                end_ind = len(self.data['time'])
            else:
                end_ind = bisect.bisect_left(self.data['time'], update_metadata_stamp[ii+1])
            if (start_ind == end_ind) & (start_ind == len(self.data['time'])):
                continue
            elif start_ind == end_ind:
                relevant_inds = np.array([start_ind])
            else:
                relevant_inds = np.arange(start_ind, end_ind)

            #update relevant inds to be only for non-NaN measurement values
            relevant_inds = relevant_inds[np.isfinite(self.data[self.parameter_details['bsc_parameter_name']][relevant_inds])]

            #if have 0 relevant_inds, then continue to next metadata parameter
            if len(relevant_inds) == 0:
                continue

            #if uncertainty is in percentage terms, convert to absolute terms (for all values across relevant range)
            #determine type of way uncertainty is reported, and deal with processing accordingly

            #get all string characters (no numbers)
            uncertainty_char = ''.join([i for i in uncertainty if not i.isdigit()]).replace('.','')

            #put all separate numerical values in string as elements in new list
            uncertainty_val = np.array(re.findall(r"[+-]?\d+(?:\.\d+)?", uncertainty)).astype(np.float32)

            #1. absolute value
            if uncertainty_char == '':
                uncertainty_absolute = uncertainty_val[0] * np.ones(len(relevant_inds))

            #2. percentage value
            elif uncertainty_char == '%':
                fraction = uncertainty_val[0]/100.0
                uncertainty_absolute = self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]*fraction

            #3. the greater of either an absolute value or percentage
            elif (uncertainty_char == '/%') or (uncertainty_char == '%/'):

                if quantity_char == '/%':
                    absolute = uncertainty_val[0]
                    fraction = uncertainty_val[1]/100.0

                elif quantity_char == '%/':
                    absolute = uncertainty_val[1]
                    fraction = uncertainty_val[0]/100.0

                uncertainty_absolute = []
                for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                    #percentage (as absolute) greater
                    if (fraction*val) > absolute:
                        uncertainty_absolute=np.append(uncertainty_absolute,fraction*val)
                    #absolute greater
                    else:
                        uncertainty_absolute=np.append(uncertainty_absolute,absolute)

            #4. percentage value after a fixed limit (minimum uncertainty taken from this limit)
            elif uncertainty_char == '%>=':
                limit = uncertainty_val[1]
                fraction = uncertainty_val[0]/100.0

                uncertainty_absolute = []
                for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                    #percentage (as absolute) higher than limit
                    if (fraction*val) > limit:
                        uncertainty_absolute=np.append(uncertainty_absolute,fraction*val)
                    #not greater than limit
                    else:
                        uncertainty_absolute=np.append(uncertainty_absolute,fraction*limit)

            #5. choice of 2 absolute values under/over fixed limit
            elif uncertainty_char == '< >=':
                absolute_under =  uncertainty_val[0]
                limit =     uncertainty_val[1]
                absolute_over =  uncertainty_val[2]

                uncertainty_absolute = []
                for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                    #absolute value higher than limit
                    if val >= limit:
                        uncertainty_absolute=np.append(uncertainty_absolute,absolute_over)
                    #absolute value under limit
                    elif val < limit:
                        uncertainty_absolute=np.append(uncertainty_absolute,absolute_under)

            #6. choice of 2 percentage values under/over fixed limit
            elif uncertainty_char == '%< %>=':
                fraction_under =  uncertainty_val[0]/100.0
                limit =           uncertainty_val[1]
                fraction_over =   uncertainty_val[2]/100.0

                uncertainty_absolute = []
                for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                    #percentage (as absolute) >= limit
                    if (fraction_over*val) >= limit:
                        uncertainty_absolute=np.append(uncertainty_absolute,fraction_over*val)
                    #< limit
                    else:
                        uncertainty_absolute=np.append(uncertainty_absolute,fraction_under*val)


            #overwrite 'reported_uncertainty_per_measurement' field if == NaN
            for rr, relevant_ind in enumerate(relevant_inds):
                if pd.isnull(self.data['reported_uncertainty_per_measurement'][relevant_ind]):
                    self.data['reported_uncertainty_per_measurement'][relevant_ind] = uncertainty_absolute[rr]

        #------------------------------------------------------------#
        #process derived uncertainties

        #iterate through each metadata parameter
        for ii in range(len(reported_precision)):

            #do you have valid reported precision?
            if not pd.isnull(reported_precision[ii]):
                precision = reported_precision[ii]
            #do you have valid documented precision?
            elif not pd.isnull(documented_precision[ii]):
                precision = documented_precision[ii]
            #no valid precision value, continue to next metadata index
            else:
                continue

            #do you have valid reported accuracy?
            if not pd.isnull(reported_accuracy[ii]):
                accuracy = reported_accuracy[ii]
            #do you have valid documented accuracy?
            elif not pd.isnull(documented_accuracy[ii]):
                accuracy = documented_accuracy[ii]
            #no valid accuracy value, continue to next metadata index
            else:
                continue

            #get relevant range of inds to update derived_uncertainty_per_measurement data field
            #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period
            start_ind = bisect.bisect_left(self.data['time'], update_metadata_stamp[ii])
            if ii == (len(reported_precision)-1):
                end_ind = len(self.data['time'])
            else:
                end_ind = bisect.bisect_left(self.data['time'], update_metadata_stamp[ii+1])
            if (start_ind == end_ind) & (start_ind == len(self.data['time'])):
                continue
            elif start_ind == end_ind:
                relevant_inds = np.array([start_ind])
            else:
                relevant_inds = np.arange(start_ind, end_ind)

            #update relevant inds to be only for non-NaN/INF measurement values
            relevant_inds = relevant_inds[np.isfinite(self.data[self.parameter_details['bsc_parameter_name']][relevant_inds])]

            #if have 0 relevant_inds, then continue to next metadata parameter
            if len(relevant_inds) == 0:
                continue

            #if precision or accuracy are in percentage terms, convert to absolute terms (for all values across relevant range)
            #determine type of way precision/accuracy are reported, and deal with processing accordingly

            quantities = [precision,accuracy]
            quantities_absolute = []

            for quantity in quantities:

                #get all string characters (no numbers)
                quantity_char = ''.join([i for i in quantity if not i.isdigit()]).replace('.','')

                #put all separate numerical values in string as elements in new list
                quantity_val = np.array(re.findall(r"[+-]?\d+(?:\.\d+)?", quantity)).astype(np.float32)

                #1. absolute value
                if quantity_char == '':
                    quantity_absolute = quantity_val[0] * np.ones(len(relevant_inds))

                #2. percentage value
                elif quantity_char == '%':
                    fraction = quantity_val[0]/100.0
                    quantity_absolute = self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]*fraction

                #3. the greater of either an absolute value or percentage
                elif (quantity_char == '/%') or (quantity_char == '%/'):

                    if quantity_char == '/%':
                        absolute = quantity_val[0]
                        fraction = quantity_val[1]/100.0

                    elif quantity_char == '%/':
                        absolute = quantity_val[1]
                        fraction = quantity_val[0]/100.0

                    quantity_absolute = []
                    for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                        #percentage (as absolute) greater
                        if (fraction*val) > absolute:
                            quantity_absolute=np.append(quantity_absolute,fraction*val)
                        #absolute greater
                        else:
                            quantity_absolute=np.append(quantity_absolute,absolute)

                #4. percentage value after greater or equal to a fixed limit (minimum uncertainty taken from this limit)
                elif quantity_char == '%>=':
                    limit = quantity_val[1]
                    fraction = quantity_val[0]/100.0

                    quantity_absolute = []
                    for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                        #percentage (as absolute) higher than limit
                        if (fraction*val) >= limit:
                            quantity_absolute=np.append(quantity_absolute,fraction*val)
                        #not greater than limit
                        else:
                            quantity_absolute=np.append(quantity_absolute,fraction*limit)

                #5. choice of 2 absolute values under/over fixed limit
                elif quantity_char == '< >=':
                    absolute_under =  quantity_val[0]
                    limit =     quantity_val[1]
                    absolute_over =  quantity_val[2]

                    quantity_absolute = []
                    for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                        #absolute value higher than limit
                        if val >= limit:
                            quantity_absolute=np.append(quantity_absolute,absolute_over)
                        #absolute value under limit
                        elif val < limit:
                            quantity_absolute=np.append(quantity_absolute,absolute_under)

                #6. choice of 2 percentage values under/over fixed limit
                elif quantity_char == '%< %>=':
                    fraction_under =  quantity_val[0]/100.0
                    limit =           quantity_val[1]
                    fraction_over =   quantity_val[2]/100.0

                    quantity_absolute = []
                    for val in self.data[self.parameter_details['bsc_parameter_name']][relevant_inds]:
                        #percentage (as absolute) >= limit
                        if (fraction_over*val) >= limit:
                            quantity_absolute=np.append(quantity_absolute,fraction_over*val)
                        #< limit
                        else:
                            quantity_absolute=np.append(quantity_absolute,fraction_under*val)

                quantities_absolute.append(quantity_absolute)

            #calculate derived uncertainty -- sqrt((precision**2) + (accuracy**2))
            uncertainty = np.sqrt((quantities_absolute[0]**2)+(quantities_absolute[1]**2))

            #update field
            self.data['derived_uncertainty_per_measurement'][relevant_inds] = uncertainty


    def check_missing_measurement(self):
    
        '''Check if have missing measurements (i.e NaN), or have network flag stating missing measurement (possible in cases where have data actually reported, but network states it as being missing using a data flag)'''

        #get data flag code for indicating missing values
        missing_flag_code = standard_data_flag_name_to_data_flag_code['Missing Data']

        #check for missing values by evaluating if each measurement value is np.NaN
        invalid_inds_empirical = list(np.arange(len(self.data[self.parameter_details['bsc_parameter_name']]))[pd.isnull(self.data[self.parameter_details['bsc_parameter_name']])])
        #check missing values by evaluating missing data flags
        invalid_inds_missing_flags = [ii for ii,flag_codes in enumerate(self.data['flag']) if missing_flag_code in flag_codes] 

        #get unique indices from the multiple lists
        invalid_inds = list(set(invalid_inds_empirical + invalid_inds_missing_flags))

        #update relevant inds with qa codes 
        if len(invalid_inds) > 0:
            #get relevant qa flag code
            qa_flag_code = standard_QA_name_to_QA_code['Missing Measurement']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)
            

    def check_negative_measurement(self):
    
        '''Check if have negative measurements (i.e < 0.0), or have network flags stating negative measurements'''
    
        #get data flag codes for indicating negative values (untouched or modified (i.e. estimated) )
        negative_value_flag_code = standard_data_flag_name_to_data_flag_code['Negative Value']
        negative_value_estimated_flag_code = standard_data_flag_name_to_data_flag_code['Estimated Data - Measured Negative Value']

        #check negative values by evaluating if each measurement value is < 0.0
        invalid_inds_empirical = list(np.where(self.data[self.parameter_details['bsc_parameter_name']] < 0.0)[0])
        #check negative values by evaluating negative value data flags
        invalid_inds_negative_flags = [ii for ii,flag_codes in enumerate(self.data['flag']) if negative_value_flag_code in flag_codes]
        invalid_inds_negative_estimated_flags = [ii for ii,flag_codes in enumerate(self.data['flag']) if negative_value_estimated_flag_code in flag_codes]
    
        #get unique indices from the multiple lists
        invalid_inds = list(set(invalid_inds_empirical + invalid_inds_negative_flags + invalid_inds_negative_estimated_flags))
    
        #update relevant inds with qa codes 
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write
            qa_flag_code = standard_QA_name_to_QA_code['Negative Measurement']

            #set list of qa flags to check have not written first before writing qa flag
            check_do_not_have_flag_names = ['Missing Measurement']
            check_do_not_have_flag_codes = [standard_QA_name_to_QA_code[flag_name] for flag_name in check_do_not_have_flag_names]

            for inv in invalid_inds:
            
                #check have not written certain other qa flags before writing
                if set(self.data['qa'][inv]).isdisjoint(check_do_not_have_flag_codes):
                    self.data['qa'][inv].append(qa_flag_code)
            

    def check_zero_values(self):

        '''Check if have values == 0, or have network flags stating zero measurements (i.e. no values detected)'''

        #get data flag codes for indicating zero values (untouched or modified (i.e. estimated) )
        zero_value_flag_code = standard_data_flag_name_to_data_flag_code['No Value Detected']
        zero_value_estimated_flag_code = standard_data_flag_name_to_data_flag_code['Estimated Data - No Value Detected']

        #check zero values by evaluating if each measurement value is == 0.0
        invalid_inds_empirical = list(np.where(self.data[self.parameter_details['bsc_parameter_name']] == 0.0)[0])
        #check zero values by evaluating zero value data flags
        invalid_inds_zero_flags = [ii for ii,flag_codes in enumerate(self.data['flag']) if zero_value_flag_code in flag_codes]
        invalid_inds_zero_estimated_flags = [ii for ii,flag_codes in enumerate(self.data['flag']) if zero_value_estimated_flag_code in flag_codes]
    
        #get unique indices from the multiple lists
        invalid_inds = list(set(invalid_inds_empirical + invalid_inds_zero_flags + invalid_inds_zero_estimated_flags))
    
        #update relevant inds with qa codes 
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Zero Measurement']

            #set list of qa flags to check have not written first before writing qa flag
            check_do_not_have_flag_names = ['Missing Measurement','Negative Measurement']
            check_do_not_have_flag_codes = [standard_QA_name_to_QA_code[flag_name] for flag_name in check_do_not_have_flag_names]

            for inv in invalid_inds:
            
                #check have not written certain other qa flags before writing
                if set(self.data['qa'][inv]).isdisjoint(check_do_not_have_flag_codes):
                    self.data['qa'][inv].append(qa_flag_code)
            

    def check_infinite_values(self):

        '''Check if have infinite values (i.e. np.inf).
           This happens when values are outside of the range that the float32 data type can handle (-3.4E+38 to +3.4E+38)
        '''

        invalid_inds = np.arange(len(self.data[self.parameter_details['bsc_parameter_name']]))[np.isinf(self.data[self.parameter_details['bsc_parameter_name']])]
    
        #update relevant invalid inds with qa codes 
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write
            qa_flag_code = standard_QA_name_to_QA_code['Infinite Value']

            #set list of qa flags to check have not written first before writing qa flag
            check_do_not_have_flag_names = ['Missing Measurement','Negative Measurement','Zero Measurement']
            check_do_not_have_flag_codes = [standard_QA_name_to_QA_code[flag_name] for flag_name in check_do_not_have_flag_names]

            for inv in invalid_inds:
            
                #check have not written certain other qa flags before writing
                if set(self.data['qa'][inv]).isdisjoint(check_do_not_have_flag_codes):
                    self.data['qa'][inv].append(qa_flag_code)
             
 
    def check_station_position(self):
    
        '''Check station position through cross referencing reported station altitude with ASTER v3 DEM altitude (30m resolution).
           If the absolute difference is >= 50 metres, then flag the station position as suspicious.
        '''

        #get key metadata
        altitudes = self.metadata['altitude']
        ASTER_altitudes = self.metadata['ASTER_v3_altitude']
        update_metadata_stamps = self.metadata['meta_update_stamp']
    
        #create list for saving invalid data inds 
        invalid_inds = []

        #iterate through all altitudes
        for ii, altitude in enumerate(altitudes):
            #get reciprocal ASTER altitude
            ASTER_altitude = ASTER_altitudes[ii]

            #valid ASTER altitude?
            if not pd.isnull(ASTER_altitude):

                #absolute difference between altitudes >= 50m
                if np.abs(ASTER_altitude - altitude) >= 50:

                    #get data indices for applicable period of altitude
                    #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period 
                    start_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii])
                    if ii == (len(altitudes)-1):
                        end_ind = len(self.data['time'])
                    else:
                        end_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii+1])
                    if (start_ind == end_ind) & (start_ind == len(self.data['time'])): 
                        pass
                    elif start_ind == end_ind:  
                        invalid_inds.extend([start_ind])
                    else:
                        invalid_inds.extend(range(start_ind, end_ind))  
            
        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Station Position Doubt - DEM Decreed']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_measurement_qa(self):
        '''Add saved standardised measurement qa information as qa flags'''
        
        #get key metadata
        measurement_qa_flags = self.metadata['measurement_qa_flags']
        update_metadata_stamps = self.metadata['meta_update_stamp']

        #iterate through all measurement_qa_flags strings
        for ii, measurement_qa_flag_str in enumerate(measurement_qa_flags):
           #have some measurement qa flags set? 
           if not pd.isnull(measurement_qa_flag_str):
                #split measurement_qa_flag_str into separate qa codes, and convert it to int type
                split_measurement_qa_flags = np.array(measurement_qa_flag_str.split(','), dtype=np.int)

                #get data indices for applicable period of measurement qa
                #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period 
                start_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii])
                if ii == (len(measurement_qa_flags)-1):
                    end_ind = len(self.data['time'])
                else:
                    end_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii+1])
                if (start_ind == end_ind) & (start_ind == len(self.data['time'])): 
                    pass
                elif start_ind == end_ind:  
                    relevant_inds = [start_ind]
                else:
                    relevant_inds = range(start_ind, end_ind)
                
                #add measurement qa flags to relevant indices
                for ind in relevant_inds:
                    self.data['qa'][ind].extend(split_measurement_qa_flags)
                    
        #drop measurement_qa_flags field from metadata
        self.metadata = rf.drop_fields(self.metadata, ['measurement_qa_flags'])


    def check_data_level(self):

        '''Check all data which is not associated with the highest data level from the providing data network.
           Flag all of this data with its own QA flag.
        '''

        #get key metadata
        data_levels = self.metadata['data_level']
        update_metadata_stamps = self.metadata['meta_update_stamp']

        #get maximum possible data quality level for reporting network
        maximum_possible_data_level = max(self.network_details['data_level_hierarchy'], key=self.network_details['data_level_hierarchy'].get)

        #create list for saving invalid data inds 
        invalid_inds = []

        #iterate through all data levels
        for ii, data_level in enumerate(data_levels):

            #check if data_level is not maximum possible data quality level 
            if data_level != maximum_possible_data_level:

                #get data indices for applicable period of data_level
                #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period 
                start_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii])
                if ii == (len(data_levels)-1):
                    end_ind = len(self.data['time'])
                else:
                    end_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii+1])
                if (start_ind == end_ind) & (start_ind == len(self.data['time'])): 
                    pass
                elif start_ind == end_ind:  
                    invalid_inds.extend([start_ind])
                else:
                    invalid_inds.extend(range(start_ind, end_ind))               

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Not Maximum Data Quality Level']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_preliminary_data(self):

        '''Check all data which is flagged in the data provider flags as preliminary.
           Flag all of this data with its own QA flag.
        '''

        #get data flag code for "Preliminary Data"
        preliminary_data_flag_code = standard_data_flag_name_to_data_flag_code['Preliminary Data']

        #get all instances where data flags contain code for 'Preliminary Data'
        invalid_inds = [ii for ii,flag_codes in enumerate(self.data['flag']) if preliminary_data_flag_code in flag_codes]

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Preliminary Data']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_local_precipitation(self):

        '''Check all data which is flagged in the data provider flags as having some form of precipitation present at the measurement location at the time of measurement.
           Flag all of this data with its own QA flag.
        '''

        #get data flag codes for all forms of precipitation
        precipitation_flag_names = ['Precipitation - Unspecified Intensity', 'Precipitation - Light', 'Precipitation - Moderate', 'Precipitation - Heavy', 'Drizzle - Unspecified Intensity', 'Drizzle - Light', 'Drizzle - Moderate', 'Drizzle - Heavy', 'Freezing Drizzle - Unspecified Intensity', 'Freezing Drizzle - Light', 'Freezing Drizzle - Moderate', 'Freezing Drizzle - Heavy', 'Rain - Unspecified Intensity', 'Rain - Light', 'Rain - Moderate', 'Rain - Heavy', 'Rain Shower/s - Unspecified Intensity', 'Rain Shower/s - Light', 'Rain Shower/s - Moderate', 'Rain Shower/s - Heavy', 'Freezing Rain - Unspecified Intensity', 'Freezing Rain - Light', 'Freezing Rain - Moderate', 'Freezing Rain - Heavy', 'Freezing Rain Shower/s - Unspecified Intensity', 'Freezing Rain Shower/s - Light', 'Freezing Rain Shower/s - Moderate', 'Freezing Rain Shower/s - Heavy', 'Snow - Unspecified Intensity', 'Snow - Light', 'Snow - Moderate', 'Snow - Heavy', 'Snow Shower/s - Unspecified Intensity', 'Snow Shower/s - Light', 'Snow Shower/s - Moderate', 'Snow Shower/s - Heavy', 'Hail - Unspecified Intensity', 'Hail - Light', 'Hail - Moderate', 'Hail - Heavy', 'Hail Shower/s - Unspecified Intensity', 'Hail Shower/s - Light', 'Hail Shower/s - Moderate', 'Hail Shower/s - Heavy', 'Ice Pellets - Unspecified Intensity', 'Ice Pellets - Light', 'Ice Pellets - Moderate', 'Ice Pellets - Heavy', 'Ice Pellets Shower/s - Unspecified Intensity', 'Ice Pellets Shower/s - Light', 'Ice Pellets Shower/s - Moderate', 'Ice Pellets Shower/s - Heavy', 'Snow Pellets - Unspecified Intensity', 'Snow Pellets - Light', 'Snow Pellets - Moderate', 'Snow Pellets - Heavy', 'Snow Pellets Shower/s - Unspecified Intensity', 'Snow Pellets Shower/s - Light', 'Snow Pellets Shower/s - Moderate', 'Snow Pellets Shower/s - Heavy', 'Snow Grains - Unspecified Intensity', 'Snow Grains - Light', 'Snow Grains - Moderate', 'Snow Grains - Heavy', 'Diamond Dust - Unspecified Intensity', 'Diamond Dust - Light', 'Diamond Dust - Moderate', 'Diamond Dust - Heavy']
        precipitation_flag_codes = np.array([standard_data_flag_name_to_data_flag_code[precipitation_flag_name] for precipitation_flag_name in precipitation_flag_names], dtype=np.uint8)

        #get all instances where data flags contain codes for 'precipitation' of some kind
        invalid_inds = np.arange(len(self.data['flag'][:]), dtype=np.int)[np.array([any(x in flag_codes for x in precipitation_flag_codes) for flag_codes in self.data['flag'][:]])]

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Local Precipitation']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_local_extreme_weather(self):

        '''Check all data which is flagged in the data provider flags as having some form of extreme weather present at the measurement location at the time of measurement.
           Flag all of this data with its own QA flag.
        '''

        #get data flag codes for all forms of 'extreme weather' (extreme internal instrument/laboratory conditions, heavy precipitation, funnel clouds, squalls, tropical cyclones, duststorm, sandstorm, dust/sand whirls, high winds)
        extreme_weather_flag_names = ['Extreme Internal Instrument Meteorological Conditions', 'Extreme Ambient Laboratory Meteorological Conditions', 'Extreme External Meteorological Conditions', 'Precipitation - Heavy', 'Drizzle - Heavy', 'Freezing Drizzle - Heavy', 'Rain - Heavy', 'Rain Shower/s - Heavy', 'Freezing Rain - Heavy', 'Freezing Rain Shower/s - Heavy', 'Snow - Heavy', 'Snow Shower/s - Heavy', 'Hail - Heavy', 'Hail Shower/s - Heavy', 'Ice Pellets - Heavy', 'Ice Pellets Shower/s - Heavy', 'Snow Pellets - Heavy', 'Snow Pellets Shower/s - Heavy', 'Snow Grains - Heavy', 'Diamond Dust - Heavy', 'Thunderstorm', 'Funnel Cloud/s', 'Squalls', 'Tropical Cyclone (Cyclone/Hurricane/Typhoon)', 'Duststorm', 'Sandstorm', 'Dust/Sand Whirls', 'High Winds']
        extreme_weather_flag_codes = np.array([standard_data_flag_name_to_data_flag_code[extreme_weather_flag_name] for extreme_weather_flag_name in extreme_weather_flag_names], dtype=np.uint8)

        #get all instances where data flags contain codes for 'extreme weather' of some kind 
        invalid_inds = np.arange(len(self.data['flag'][:]), dtype=np.int)[np.array([any(x in flag_codes for x in extreme_weather_flag_codes) for flag_codes in self.data['flag'][:]])]

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Local Extreme Weather']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_local_atmospheric_obscuration(self):

        '''Check all data which is flagged in the data provider flags as having some form of atmospheric obscuration present at the measurement location at the time of measurement.
           Flag all of this data with its own QA flag.
        '''

        #get data flag codes for all forms of 'atmospheric obscuration' 
        atmospheric_obscuration_flag_names = ['Atmospheric Obscuration - Unknown', 'Dust', 'Blowing Dust', 'Drifting Dust', 'Sand', 'Blowing Sand', 'Drifting Sand', 'Blowing Snow', 'Drifting Snow', 'Fog', 'Freezing Fog', 'Ground Fog', 'Ice Fog', 'Haze', 'Mist', 'Sea Spray', 'Smoke', 'Volcanic Ash']
        atmospheric_obscuration_flag_codes = np.array([standard_data_flag_name_to_data_flag_code[atmospheric_obscuration_flag_name] for atmospheric_obscuration_flag_name in atmospheric_obscuration_flag_names], dtype=np.uint8)

        #get all instances where data flags contain codes for 'atmospheric obscuration' of some kind 
        invalid_inds = np.arange(len(self.data['flag'][:]), dtype=np.int)[np.array([any(x in flag_codes for x in atmospheric_obscuration_flag_codes) for flag_codes in self.data['flag'][:]])]

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Local Atmospheric Obscuration']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_local_contamination(self):

        '''Check all data which is flagged in the data provider flags as having some form of local contamination present at the measurement location at the time of measurement.
           Flag all of this data with its own QA flag.
        '''

        #get data flag codes for all forms of 'local contamination' 
        local_contamination_flag_names = ['Local Contamination - Unspecified', 'Agricultural Contamination', 'Bird-Dropping Contamination', 'Construction Contamination', 'Industrial Contamination', 'Insect Contamination', 'Internal Laboratory/Instrument Contamination', 'Pollen/Leaf Contamination', 'Traffic Contamination']
        local_contamination_flag_codes = np.array([standard_data_flag_name_to_data_flag_code[local_contamination_flag_name] for local_contamination_flag_name in local_contamination_flag_names], dtype=np.uint8)

        #get all instances where data flags contain codes for 'local contamination' of some kind 
        invalid_inds = np.arange(len(self.data['flag'][:]), dtype=np.int)[np.array([any(x in flag_codes for x in local_contamination_flag_codes) for flag_codes in self.data['flag'][:]])]

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Local Contamination']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_local_exceptional_event(self):

        '''Check all data which is flagged in the data provider flags as having some form of exceptional event (both natural and anthropogenically induced) active at the measurement location at the time of measurement.
           Flag all of this data with its own QA flag.
        '''

        #get data flag codes for all forms of 'exceptional events' 
        exceptional_event_flag_names = ['Exceptional Event - Unspecified', 'Seismic Activity', 'Stratospheric Ozone Intrusion', 'Volcanic Eruptions', 'Wildfire', 'Chemical Spill/Industrial Accident', 'Cleanup After a Major Disaster', 'Demolition', 'Fireworks', 'Infrequent Large Gathering', 'Terrorist Act']
        exceptional_event_flag_codes = np.array([standard_data_flag_name_to_data_flag_code[exceptional_event_flag_name] for exceptional_event_flag_name in exceptional_event_flag_names], dtype=np.uint8)

        #get all instances where data flags contain codes for 'exceptional event' of some kind 
        invalid_inds = np.arange(len(self.data['flag'][:]), dtype=np.int)[np.array([any(x in flag_codes for x in exceptional_event_flag_codes) for flag_codes in self.data['flag'][:]])]

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Local Exceptional Event']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_lod(self):

        '''iterate through measured data, and check if the values are <=/>= the reported/documented limits of detection (LODs)
           documented LODs are static and apply for the whole use time of measurement instrument
           the network reported limits of detection are either variable in time, or as a fixed value for an entire data range.
           additionally network providers often give flags stating if data is <=/>= detection limits
           these reported flags are first used in preference over testing if measurements are above/below reported static/variable detection limits
           variable reported LODs are then taken preferentially over reported static LODs where both are available.
           preferential checks are evaluated by first trying to evaluate measurements with reported LODs, or if not available, then with documented LODs 
           any measurements which have NaN values are not flagged with any flags by this function
        '''
    

        #get relevant qa flags
        below_lod_documented_qa_flag_code = standard_QA_name_to_QA_code['Below Documented Lower Limit of Detection']
        below_lod_reported_qa_flag_code = standard_QA_name_to_QA_code['Below Reported Lower Limit of Detection']
        below_lod_preferential_qa_flag_code = standard_QA_name_to_QA_code['Below Preferential Lower Limit of Detection']
        above_lod_documented_qa_flag_code = standard_QA_name_to_QA_code['Above Documented Upper Limit of Detection']
        above_lod_reported_qa_flag_code = standard_QA_name_to_QA_code['Above Reported Upper Limit of Detection']
        above_lod_preferential_qa_flag_code = standard_QA_name_to_QA_code['Above Preferential Upper Limit of Detection']

        #get needed data/metadata
        static_lower_lod_documented = self.metadata['measuring_instrument_documented_lower_limit_of_detection']
        static_lower_lod_reported = self.metadata['measuring_instrument_reported_lower_limit_of_detection']
        static_upper_lod_documented = self.metadata['measuring_instrument_documented_upper_limit_of_detection']
        static_upper_lod_reported = self.metadata['measuring_instrument_reported_upper_limit_of_detection']
        update_metadata_stamp = self.metadata['meta_update_stamp']

        #get relevant standard data flag codes
        below_lod_data_flag_code = standard_data_flag_name_to_data_flag_code['Value Below Detection Limit']
        estimated_below_lod_data_flag_code = standard_data_flag_name_to_data_flag_code['Estimated Data - Value Below Detection Limit']
        above_lod_data_flag_code = standard_data_flag_name_to_data_flag_code['Value Above Detection Limit']
        estimated_above_lod_data_flag_code = standard_data_flag_name_to_data_flag_code['Estimated Data - Value Above Detection Limit']

        #iterate through #iterate through all time points
        for ii in range(len(self.data['time'])):
        
            #get measurement value for timepoint
            current_data = self.data[self.parameter_details['bsc_parameter_name']][ii]
        
            #if current data is NaN, skip to next value
            if not np.isfinite(current_data):
                continue

            #set flags to state if have evaluated validity of measurements through comparison with reported/documented LODs 
            evaluated_lower_lod_reported = False
            evaluated_upper_lod_reported = False 
            evaluated_lower_lod_documented = False
            evaluated_upper_lod_documented = False 

            #set flags when have written reported/documented flags as measurements are below/above reported/documented LODs
            written_lower_lod_reported = False
            written_upper_lod_reported = False 
            written_lower_lod_documented = False
            written_upper_lod_documented = False 

            #preferentially check if have reported network data flags informing of hitting of LOD
            current_flag_codes = self.data['flag'][ii]
        
            #if have network flag stating measurement is <=/>= detection limits, write reported qa flag
            #have flag stating measurement is <= detection limit?
            if (below_lod_data_flag_code in current_flag_codes) or (estimated_below_lod_data_flag_code in current_flag_codes):
                self.data['qa'][ii].append(below_lod_reported_qa_flag_code)
                evaluated_lower_lod_reported = True
                written_lower_lod_reported = True
            #have flag stating measurement is >= detection limit?
            if (above_lod_data_flag_code in current_flag_codes) or (estimated_above_lod_data_flag_code in current_flag_codes):
                self.data['qa'][ii].append(above_lod_reported_qa_flag_code)
                evaluated_upper_lod_reported = True
                written_upper_lod_reported = True
                    
            #get utc time in minutes
            current_minutes = self.data['time'][ii]        
            #get index of current valid metadata for timepoint
            current_meta_index = bisect.bisect(update_metadata_stamp,current_minutes)-1
            #use metadata index to get static documented limit metadata
            current_lower_lod_documented = static_lower_lod_documented[current_meta_index]
            current_upper_lod_documented = static_upper_lod_documented[current_meta_index]
        
            #-----------------------------------------------------
            #test measurements by documented LODs

            #is documented lower lod available?
            if not pd.isnull(current_lower_lod_documented):
    
                #is measurement below or equal to documented lower limit of detection?
                if current_data <= current_lower_lod_documented:
                    self.data['qa'][ii].append(below_lod_documented_qa_flag_code)
                    written_lower_lod_documented = True

                #set flag that have evaluated documented lower LOD
                evaluated_lower_lod_documented = True

            #is documented upper lod available?
            if not pd.isnull(current_upper_lod_documented):            

                #is measurement above or equal to documented upper limit of detection?
                if current_data >= current_upper_lod_documented:
                    self.data['qa'][ii].append(above_lod_documented_qa_flag_code)
                    written_upper_lod_documented = True  

                #set flag that have evaluated documented upper LOD
                evaluated_upper_lod_documented = True

            #-----------------------------------------------------
        
            #test measurements by reported LODs (if not evaluated yet)
            #preferentially evaluate by variable reported LOD, and then static LOD

            #have not evaluated reported lower LOD yet?
            if not evaluated_lower_lod_reported:

                #get variable reported lower LOD
                current_variable_lower_lod_reported = self.data['reported_lower_limit_of_detection_per_measurement'][ii]
        
                #is variable reported lower lod available?
                if not pd.isnull(current_variable_lower_lod_reported):
 
                    #is measurement below or equal to reported lower limit of detection?
                    if current_data <= current_variable_lower_lod_reported:
                        self.data['qa'][ii].append(below_lod_reported_qa_flag_code)
                        written_lower_lod_reported = True

                    #set flag that have evaluated reported lower LOD
                    evaluated_lower_lod_reported = True

                #if variable reported LOD not available, evaluate using static reported LOD (if available)
                else:
            
                    #get static reported lower LOD
                    current_static_lower_lod_reported = static_lower_lod_reported[current_meta_index]

                    #is static reported lower lod available?
                    if not pd.isnull(current_static_lower_lod_reported):

                        #is measurement below or equal to reported lower limit of detection?
                        if current_data <= current_static_lower_lod_reported:
                            self.data['qa'][ii].append(below_lod_reported_qa_flag_code)
                            written_lower_lod_reported = True

                        #set flag that have evaluated reported lower LOD
                        evaluated_lower_lod_reported = True

            #have not evaluated reported upper LOD yet?
            if not evaluated_upper_lod_reported:

                #get variable reported upper LOD
                current_variable_upper_lod_reported = self.data['reported_upper_limit_of_detection_per_measurement'][ii]
        
                #is variable reported upper lod available?
                if not pd.isnull(current_variable_upper_lod_reported):
 
                    #is measurement above or equal to reported lower limit of detection?
                    if current_data >= current_variable_upper_lod_reported:
                        self.data['qa'][ii].append(above_lod_reported_qa_flag_code)
                        written_upper_lod_reported = True

                    #set flag that have evaluated reported upper LOD
                    evaluated_upper_lod_reported = True

                #if variable reported LOD not available, evaulate using static reported LOD (if available)
                else:
            
                    #get static reported upper LOD
                    current_static_upper_lod_reported = static_upper_lod_reported[current_meta_index]

                    #is static reported upper lod available?
                    if not pd.isnull(current_static_upper_lod_reported):

                        #is measurement above or equal to reported upper limit of detection?
                        if current_data >= current_static_upper_lod_reported:
                            self.data['qa'][ii].append(above_lod_reported_qa_flag_code)
                            written_upper_lod_reported = True

                        #set flag that have evaluated reported upper LOD
                        evaluated_upper_lod_reported = True

            #-----------------------------------------------------  
            #write preferential flags that are set by first trying to evaluate measurements compared with reported LOD, or if not available then with documented LOD    

            #have evaluated reported lower LOD?
            if evaluated_lower_lod_reported:
                #have written reported lower LOD flag?
                if written_lower_lod_reported:  
                    self.data['qa'][ii].append(below_lod_preferential_qa_flag_code)
            
            #else, have evaluated documented lower LOD?   
            elif evaluated_lower_lod_documented:
                #have written documented lower LOD flag?
                if written_lower_lod_documented: 
                    self.data['qa'][ii].append(below_lod_preferential_qa_flag_code)
            
            #have evaluated reported upper LOD?
            if evaluated_upper_lod_reported:
                #have written reported upper LOD flag?
                if written_upper_lod_reported:  
                    self.data['qa'][ii].append(above_lod_preferential_qa_flag_code)
            
            #else, have evaluated documented upper LOD?   
            elif evaluated_upper_lod_documented:
                #have written documented upper LOD flag?
                if written_upper_lod_documented: 
                    self.data['qa'][ii].append(above_lod_preferential_qa_flag_code)
        

    def check_data_outlier_scientifically_non_feasible(self):

        '''check if measured data is scientifically non-feasible (i.e lower/greater than scientifically feasible).
           for each different measured parameter, minimum and maximums value are set, if data is lower/greater than these values, it is flagged 
           NaNs not caught by this check
        '''

        #get relevant qa flag code
        qa_flag_code = standard_QA_name_to_QA_code['Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit']

        #iterate through measured values
        for ii in range(len(self.data[self.parameter_details['bsc_parameter_name']])):
        
            #if measured value is lower/greater than the set lower/upper limits, flag data
            if (self.data[self.parameter_details['bsc_parameter_name']][ii] < self.parameter_details['extreme_lower_limit']) or (self.data[self.parameter_details['bsc_parameter_name']][ii] > self.parameter_details['extreme_upper_limit']):
                #write out qa flag
                self.data['qa'][ii].append(qa_flag_code)
            
    
    def check_data_outlier_network(self):

        '''Check all data which is flagged in the data provider flags as being outliers in some regard (and not manually checked and verified as valid)
           Flag all of this data with its own QA flag.
        '''

        #get all network decreed outlier data flag codes
        extreme_network_flag_names = ['Extreme/Irregular Data - Unspecified', 'Data Does Not Meet Internal Network Quality Control Criteria', 'High Variability of Data', 'Irregular Data Manually Screened and Rejected', 'Value Below Acceptable Range', 'Value Above Acceptable Range']
        extreme_network_flag_codes = np.array([standard_data_flag_name_to_data_flag_code[extreme_network_flag_name] for extreme_network_flag_name in extreme_network_flag_names], dtype=np.uint8)

        #get all instances where data flags contain codes for 'outlier network decreed data' of some kind
        invalid_inds = np.arange(len(self.data['flag'][:]), dtype=np.int)[np.array([any(x in flag_codes for x in extreme_network_flag_codes) for flag_codes in self.data['flag'][:]])]

        #update relevant inds with qa code
        if len(invalid_inds) > 0:
            #get relevant qa flag code to write 
            qa_flag_code = standard_QA_name_to_QA_code['Data Outlier - Network Decreed']

            for inv in invalid_inds:
                self.data['qa'][inv].append(qa_flag_code)


    def check_data_outlier_manual(self):

        '''function for manually flagging outlying data
           if data has been manually found to be outlying, and has not been flagged by other extreme flags, then it should be manually flagged. 

           input is in the following format:
           extremes = {'parameter-station_reference':'YYYYMMDDHHMM-YYYYMMDDHHMM', 'station_reference2':YYYYMMDDHHMM-YYYYMMDDHHMM;YYYYMMDDHHMM-YYYYMMDDHHMM}
      
           if more than 1 bad period for a reference, it is separated by a semicolon.
           all data inside the given datetime ranges for a station reference is flagged as extreme (if have data for this window)
        '''

        #get relevant qa flag code
        qa_flag_code = standard_QA_name_to_QA_code['Data Outlier - Manually Decreed']

        #dictionary for definition of manually decreed data outlier windows
        extremes = {}
    
        #get all unique parameter-station references keys
        unique_keys = list(extremes.keys())
    
        #iterate through keys
        if len(unique_keys) > 0:
            for key in unique_keys:
                #separate into parameter, station reference 
                key_split = key.split('-')
                current_parameter, current_station_reference = key[0],key[1]
            
                #if are currently processing parameter, and station reference proceed
                if (current_parameter == self.run_details['parameter']) & (current_station_reference == self.metadata['station_reference'][0]):
                
                    #get all reference times for current station_reference/parameter processed
                    reference_time = self.data['time']
                
                    #split extreme window ranges by ; (if more than 1 extreme window)
                    extreme_ranges = extremes[key].split(';')
                
                    #iterate through extreme window ranges
                    for extreme_range in extreme_ranges:
                        #break apart range string
                        extreme_range_split = extreme_range.split('-')
                        start_dt_string, end_dt_string = extreme_range_split[0],extreme_range_split[1]
                        #convert dt strings to python datetimes
                        start_dt = datetime.datetime(start_dt_string[:4],start_dt_string[4:6],start_dt_string[6:8],start_dt_string[8:10],start_dt_string[10:])
                        end_dt = datetime.datetime(end_dt_string[:4],end_dt_string[4:6],end_dt_string[6:8],end_dt_string[8:10],end_dt_string[10:])
                        #get start time/end time in reference time
                        start_reference_time = datetime_to_fractional_minutes_from_reference(start_dt) 
                        end_reference_time = datetime_to_fractional_minutes_from_reference(end_dt) 
                
                        #flag all indices with start times inside extreme window range
                        extreme_inds = np.where((reference_time >= start_reference_time) & (reference_time < end_reference_time))[0]
                    
                        #if have times inside extreme window, then write out flags for all measurements inside window
                        if len(extreme_inds) > 0:
                            for inv in extreme_inds:
                                self.data['qa'][inv].append(qa_flag_code)
                        

    def check_measurement_resolution_by_metadata(self):

        '''check if measurement resolution is coarser than a set limit (variable by parameter measured). This is checked in 3 ways: by taking documented and reported metadata, and also empirically. This is handled separately by 2 different functions.
           this function handles the documented and reported metadata checks.
           function simply takes documented and reported measurement resolutions and checks if the resolutions are coarser than a set limit (variable by parameter measured). Any occasions where these are coarser are flagged.  
           a preferential check is also done where first data is flagged by evaluating by reported measurement resolutions, and if not available, then by documented measurement resolutions
        '''
    
        #get key metadata
        reported_measurement_resolutions = self.metadata['measuring_instrument_reported_measurement_resolution']
        documented_measurement_resolutions = self.metadata['measuring_instrument_documented_measurement_resolution']
        update_metadata_stamps = self.metadata['meta_update_stamp']

        invalid_inds_reported = []
        invalid_inds_documented = []
        invalid_inds_preferential = []
        invalid_inds_nometa = []

        for ii in range(len(reported_measurement_resolutions)):

            #set flags to state when have evaluated reported/documented measurement resolutions
            evaluated_reported_measurement_resolution = False
            evaluated_documented_measurement_resolution = False
        
            #set flags to state when have written reported/documented measurement resolution qa flags
            written_reported_measurement_resolution = False 
            written_documented_measurement_resolution = False 

            #get reported/documented measurement resolutions
            reported_measurement_resolution = reported_measurement_resolutions[ii]
            documented_measurement_resolution = documented_measurement_resolutions[ii]

            #get data indices for period (in case period needs flagging)
            #if measurement window crosses between two metadata periods then classify it as belonging to first metadata period 
            start_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii])
            if ii == (len(reported_measurement_resolutions)-1):
                end_ind = len(self.data['time'])
            else:
                end_ind = bisect.bisect_left(self.data['time'], update_metadata_stamps[ii+1])
            if (start_ind == end_ind) & (start_ind == len(self.data['time'])):
                invalid_inds = []
            elif start_ind == end_ind:    
                invalid_inds = [start_ind]
            else:
                invalid_inds = range(start_ind,end_ind)            
    
            #do you have valid reported measurement resolution?
            if not pd.isnull(reported_measurement_resolution):

                #is reported measurement_resolution coarser than the set minimum resolution for the parameter being processed?
                if reported_measurement_resolution > self.parameter_details['minimum_resolution']:
                    invalid_inds_reported.extend(invalid_inds)
                    written_reported_measurement_resolution = True
            
                #set flag to state you have evaluated reported measurement resolution 
                evaluated_reported_measurement_resolution = True

            #do you have valid documented measurement resolution?
            if not pd.isnull(documented_measurement_resolution):

                #is documented measurement_resolution coarser than the set minimum resolution for the parameter being processed?
                if documented_measurement_resolution > self.parameter_details['minimum_resolution']:
                    invalid_inds_documented.extend(invalid_inds)
                    written_documented_measurement_resolution = True

                #set flag to state you have evaluated documented measurement resolution 
                evaluated_documented_measurement_resolution = True

            #--------------------------------------

            #do preferential evaluation, first evaluating the reported measurement resolution if available, and if not then by documented measurement resolution 
            #have evaluated by reported measurement resolution?
            if evaluated_reported_measurement_resolution:
                #have written reported measurement resolution flag?
                if written_reported_measurement_resolution:
                    invalid_inds_preferential.extend(invalid_inds)

            #else if, have evaluated by documented measurement resolution?
            elif evaluated_documented_measurement_resolution:
                #have written documented measurement resolution flag?
                if written_documented_measurement_resolution:
                    invalid_inds_preferential.extend(invalid_inds)
                
            #else, do you have no valid documented or reported measurement resolution metadata?
            else:
                invalid_inds_nometa.append(ii)
                
        #----------------------------------------
      
        #update relevant invalid inds with qa codes
      
        if len(invalid_inds_reported) > 0:
            #get relevant qa flag code
            qa_flag_code = standard_QA_name_to_QA_code['Insufficient Measurement Resolution - Reported']
            for inv in invalid_inds_reported:
                self.data['qa'][inv].append(qa_flag_code)
      
        if len(invalid_inds_documented) > 0:
            #get relevant qa flag code
            qa_flag_code = standard_QA_name_to_QA_code['Insufficient Measurement Resolution - Documented']
            for inv in invalid_inds_documented:
                self.data['qa'][inv].append(qa_flag_code)

        if len(invalid_inds_preferential) > 0:
            #get relevant qa flag code
            qa_flag_code = standard_QA_name_to_QA_code['Insufficient Measurement Resolution - Preferential']
            for inv in invalid_inds_preferential:
                self.data['qa'][inv].append(qa_flag_code)


    def check_recurring_values(self):

        '''do check for persistently recurring values
           persistently recurring values are symptomatic of when an instrument hits the detection limit, or is malfunctioning.

           check is done by using moving windows of differing lengths. 
           If 5/6 or 9/12 or 16/24 of values are a physical values (i.e. not NaN), and the same value, flag recurring values in window.
        '''

        window_widths = [6,12,24]
        reccuring_value_thresholds = [5,9,16]
        total_n = len(self.data[self.parameter_details['bsc_parameter_name']])    

        #iterate through moving window sizes, and generate list of lists of relevant moving window indices
        for window_width, reccuring_value_threshold in zip(window_widths, reccuring_value_thresholds):
            moving_window_indices = [np.arange(n,n+window_width) for n in range((total_n-window_width)+1)]

            #list to store all indices to flag
            invalid_inds = []

            #iterate though windows
            for window in moving_window_indices:

                #get maximum count of values in valid value list - np.NaN values are not counted additively (i.e. will be multiple nan counts of 1)
                max_occuring_value,n_occurances = Counter(self.data[self.parameter_details['bsc_parameter_name']][window]).most_common(1)[0]
        
                #if number of duplicates in window >= reccuring_value_threshold, flag recurring values in window
                if n_occurances >= reccuring_value_threshold:
                    invalid_inds.append(window[self.data[self.parameter_details['bsc_parameter_name']][window] == max_occuring_value])
             
            #remove duplicate values from invalid inds array
            invalid_inds = list(itertools.chain.from_iterable(invalid_inds))
            invalid_inds = list(set(invalid_inds))
    
            #update relevant invalid inds with qa codes 
            if len(invalid_inds) > 0:
                #get relevant qa flag code
                qa_flag_code = standard_QA_name_to_QA_code['Persistent Recurring Values - {}/{}'.format(reccuring_value_threshold, window_width)]
                for inv in invalid_inds:
                    self.data['qa'][inv].append(qa_flag_code)


    def check_non_integer_local_timezone(self, valid_inds):

        '''check that determines if the local timezone of measurement station is non-integer - important for the temporal alignment of periodic processes (e.g. when using spectral analysis)
           this is done by:
           check alignment of starts of measurement of measurement periods with UTC hours (all those with a temporal resolution of equal to or coarser than 1 hour, and finer than 1 day) 
           this is done by getting remainder of start times divided by 60
           all times with remainder not equal to 0 are not aligned with UTC start hours, this implies the local timezone is non-integer.
           HOWEVER, sometimes times are manually shifted by small amounts by the data providers (i.e. 30 minute) to align perfectly with UTC hours, which this test will not catch.
        '''

        #get relevant qa flag
        qa_flag_ni_code = standard_QA_name_to_QA_code['Non-Integer Local Timezone (relative to UTC)']

        #check alignment of starts of measurement of measurement periods with UTC hours (all those with a temporal resolution of equal to or coarser than 1 hour, and finer than 1 day) 
    
        #get remainder of entire time array divided by 60
        time_modulus = np.mod(self.data['time'][valid_inds],60)
        #all times with remainder not equal to 0 are not aligned with UTC hour starts and ends, therefore flag
        non_matching_time_inds = np.where(time_modulus != 0)[0]
        if len(non_matching_time_inds) != 0:
            for non_matching_time_ind in non_matching_time_inds:
                self.data['qa'][valid_inds[non_matching_time_ind]].append(qa_flag_ni_code)


    def check_unique_values(self, df_month, df_month_group_names):

        '''check which determines the percentage of data in a given window is unique.
           If the number of uniques in the window is <= N% then the entire window is flagged.
                        
           check is only done when have >= 3 valid values after screening by certain qa flags, to ensure have sufficient values to fairly determine the % uniques in a given month

           when grouping measurements per month, the midpoint of the measurement window is taken as the point to group by      
        '''

        #get relevant qa flag codes
        qa_flag_code_1 =  standard_QA_name_to_QA_code['Monthly Fractional Unique Values <= 1%']
        qa_flag_code_5 =  standard_QA_name_to_QA_code['Monthly Fractional Unique Values <= 5%']
        qa_flag_code_10 = standard_QA_name_to_QA_code['Monthly Fractional Unique Values <= 10%']
        qa_flag_code_30 = standard_QA_name_to_QA_code['Monthly Fractional Unique Values <= 30%']
        qa_flag_code_50 = standard_QA_name_to_QA_code['Monthly Fractional Unique Values <= 50%']
        qa_flag_code_70 = standard_QA_name_to_QA_code['Monthly Fractional Unique Values <= 70%']
        qa_flag_code_90 = standard_QA_name_to_QA_code['Monthly Fractional Unique Values <= 90%']

        #define invalid qa flags 
        invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Invalid Data Provider Flags - GHOST Decreed', 'Below Preferential Lower Limit of Detection', 'Above Preferential Upper Limit of Detection', 'Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit', 'Data Outlier - Network Decreed', 'Data Outlier - Manually Decreed']

        #iterate through grouped period data 
        for month in df_month_group_names:
            month_data = df_month.get_group(month)

            #have some valid data to proceed? 
            if len(month_data) > 0:

                #remove invalid data (decreed from qa flags)
                filtered_month_data = filter_by_qa(invalid_qa_flag_names, month_data['data'].values, self.data['qa'][month_data['valid_inds'].values])
            
                #check number of total measurements after filtering in the period
                #if < 3 values in the month, then do not perform qa check
                if len(filtered_month_data) < 3:
                    continue
        
                #otherwise, check % uniques in the period
                else:
                    uniques = np.unique(filtered_month_data)
                    unique_pc = len(uniques)/len(filtered_month_data)
                        
                    #if percent uniques in the period is <= N%, then flag entire period
                    if unique_pc <= 0.01:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code_1)
                    if unique_pc <= 0.05:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code_5)
                    if unique_pc <= 0.1:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code_10)
                    if unique_pc <= 0.3:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code_30)
                    if unique_pc <= 0.5:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code_50)
                    if unique_pc <= 0.7:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code_70)
                    if unique_pc <= 0.9:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code_90)
                    


    def check_measurement_resolution_empirical(self, df_month, df_month_group_names):

        '''empirically determine measurement resolution of measurements in grouped monthly windows (UTC time).     
           if the minimum difference between observations in the monthly window is coarser than a set limit (variable by measured parameter), all monthly measurements are flagged
           check accounts for the improvement of measurement resolution over time (as happens with improved methods and instrumentation over time)
           
           check is only done when have >= 100 valid values after screening by certain qa flags, to ensure have sufficient values to fairly return a true minimum difference value.
        
           when grouping measurements per month, the midpoint of the measurement window is taken as the point to group by
        '''

        #get relevant qa flag code
        qa_flag_code = standard_QA_name_to_QA_code['Insufficient Measurement Resolution - Empirical']
        #define invalid qa flags 
        invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Invalid Data Provider Flags - GHOST Decreed', 'Below Preferential Lower Limit of Detection', 'Above Preferential Upper Limit of Detection', 'Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit', 'Data Outlier - Network Decreed', 'Data Outlier - Manually Decreed']

        #iterate through monthly grouped data and determine minimum difference between observations 
        for month in df_month_group_names:
            month_data = df_month.get_group(month)
        
            #have some valid data to proceed? 
            if len(month_data) > 0:
            
                #remove invalid data (decreed from qa flags)
                filtered_month_data = filter_by_qa(invalid_qa_flag_names, month_data['data'].values, self.data['qa'][month_data['valid_inds'].values])
            
                #check number of total measurements after filtering in the month
                #if < 100 values in the month, then do not perform qa check
                if len(filtered_month_data) < 100:
                    continue
        
                #otherwise, check minimum difference between observations
                else:
            
                    #initiate minimum difference
                    minimum_difference = np.inf
        
                    #remove duplicate values from array
                    filtered_month_data = list(set(filtered_month_data))
    
                    #sort array in ascending order
                    filtered_month_data = sorted(filtered_month_data)                    
 
                    #Find the min diff by comparing adjacent pairs in sorted array
                    for ii in range(len(filtered_month_data)-1):
                        #is difference smaller than current minimum difference
                        if (filtered_month_data[ii+1] - filtered_month_data[ii]) < minimum_difference:
                            #if so set new minimum difference
                            minimum_difference = filtered_month_data[ii+1] - filtered_month_data[ii]

                    #if minimum difference between all points for the monthly window is coarser than the set minimum resolution for the parameter being processed, flag the entire month of measurements.
                    if minimum_difference > self.parameter_details['minimum_resolution']:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code)


        
    def check_monthly_adjusted_boxplot(self, df_month, df_month_group_names):
    
        '''check for data outliers, using adjusted boxplot of monthly data (by UTC month).
           measurements which exceed inner/outer lower or upper constructed fences are flagged.

           check is only done when have >= 20 valid values after screening by certain qa flags, to ensure have sufficient values to identify outliers
           when grouping measurements per month, the midpoint of the measurement window is taken as the point to group by

           Technique is designed to identify outliers based on an adapted version of the classical Tukey boxplot.
           Tukey's boxplot is a very popular tool for detection of outliers. It reveals the location, spread and skewness of the data.
           It works nicely for detection of outliers when the data are symmetric.
           When the data are skewed it covers boundary away from the whisker on the compressed side while declares erroneous outliers on the extended side of the distribution.
           Hubert and Vandervieren (2008) made adjustment to Tukey's technique to overcome this problem.
           They apply a robust measure of skewness to Tukey's technique: the 'Medcouple', incorporated as a power of exponential times some constant on left and right as -3.5 and 4 changing position depending upon sign of medcouple. Incorporating this function condenses the interval from narrow side and extends the interval towards the puffy tail.
           However another problem arises that is the adjusted boxplot constructs the interval of critical values which even exceeds from the extremes of the data.
           In this situation the adjusted boxplot is unable to detect outliers.
           Adil and Irshad (2015) provide a solution for this:
           Where the lower and upper inner fences are defined as:
           [Lf Uf] = [Q1 - 1.5*IQR*e-SK*|MC|, Q3 + 1.5*IQR*eSK*|MC|]
           where, Q1 = the 25th percentile, Q3 = the 75th percentile, IQR = the interquartile range, SK = the classical skewness, MC = the medcouple

           **A restriction is also imposed here that if the classical skewness, SK, is greater than 3.5 then it should be treated as 3.5.
           The reason to fix maximum level of skewness 3.5 is to avoid the problem of constructing the large interval of critical values due to classical skewness that might be higher than 3.5.
           Not allowing the skewness statistic to exceed 3.5 synchronize the interval of critical value with the data sets as against the adjusted box plot and prevents the interval to be very large in case of highly skewed distributions.
           It also constructs smaller interval in case of moderately skewed distributions

           The definition of the inner fences is such that the expected percentage values which exceed is close to 0.7%, which coincides with the outlier rule of the standard Tukey boxplot for a normal distribution.           
           The definition of the outer fences is such that the expected percentage values which exceed is close to 0.0002%, which coincides with the outlier rule of the standard Tukey boxplot for a normal distribution.           
            
           The method is only recommended to be used if a small number of outliers is presumed (at most 5%).

           See References here:
           https://wis.kuleuven.be/stat/robust/papers/2008/adjboxplot-revision.pdf
           https://www.researchgate.net/publication/277943905_A_Modified_Approach_for_Detection_of_Outliers
           https://en.wikipedia.org/wiki/Medcouple
           https://en.wikipedia.org/wiki/Box_plot#Variations

        '''

        #get relevant qa flag codes
        possible_data_outlier_qa_code = standard_QA_name_to_QA_code['Possible Data Outlier - Monthly Adjusted Boxplot']
        probable_data_outlier_qa_code = standard_QA_name_to_QA_code['Probable Data Outlier - Monthly Adjusted Boxplot']
        #define invalid qa flags 
        invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Invalid Data Provider Flags - GHOST Decreed', 'Below Preferential Lower Limit of Detection', 'Above Preferential Upper Limit of Detection', 'Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit', 'Data Outlier - Network Decreed', 'Data Outlier - Manually Decreed']

        #iterate through monthly grouped data 
        for month in df_month_group_names:
            month_data = df_month.get_group(month)
        
            #have some valid data to proceed? 
            if len(month_data) > 0:
        
                #remove invalid data (decreed from qa flags)
                filtered_month_data = filter_by_qa(invalid_qa_flag_names, month_data['data'].values, self.data['qa'][month_data['valid_inds'].values])
            
                #check number of total measurements after filtering in the month
                #if < 20 values in the month, then do not perform qa check
                if len(filtered_month_data) < 20:
                    continue
    
                #otherwise, perform check
                else:    

                    #caluclate medcouple
                    mc = sm.medcouple(filtered_month_data)
                    #calculate absolute value of medcouple
                    mc_abs = np.abs(mc)
            
                    #calculate the 25th percentile 
                    p25 = np.percentile(filtered_month_data, 25)
                    #calculate the 75th percentile 
                    p75 = np.percentile(filtered_month_data, 75)

                    #calculate the interquartile range
                    iqr = p75-p25

                    #caluclate classical skewness
                    skew = st.skew(filtered_month_data)
                    #limit skewness to to be between -3.5 and 3.5
                    if (skew > 3.5):
                        skew = 3.5
                    elif (skew < -3.5):
                        skew = -3.5

                    #calculate lower/upper inner fences
                    lower_inner_fence = p25 - 1.5 * iqr * np.exp(-skew*mc_abs)
                    upper_inner_fence = p75 + 1.5 * iqr * np.exp(skew*mc_abs)

                    #calculate lower/upper outer fences
                    lower_outer_fence = p25 - 3.0 * iqr * np.exp(-skew*mc_abs)
                    upper_outer_fence = p75 + 3.0 * iqr * np.exp(skew*mc_abs)

                    #flag data which exceeds inner fence (possible data outlier)
                    inner_invs = np.where(np.logical_or(filtered_month_data<lower_inner_fence, filtered_month_data>upper_inner_fence))[0]
                    invalid_inner_inds = month_data['valid_inds'].values[inner_invs]
                    if len(invalid_inner_inds) > 0:
                        for inv in invalid_inner_inds: 
                            self.data['qa'][inv].append(possible_data_outlier_qa_code)

                    #flag data which exceeds outer fence (probable data outlier)
                    outer_invs = np.where(np.logical_or(filtered_month_data<lower_outer_fence, filtered_month_data>upper_outer_fence))[0]
                    invalid_outer_inds = month_data['valid_inds'].values[outer_invs]
                    if len(invalid_outer_inds) > 0:
                        for inv in invalid_outer_inds: 
                            self.data['qa'][inv].append(probable_data_outlier_qa_code)
                    

    def check_data_outlier_scientifically_non_feasible_monthly_median(self, df_month, df_month_group_names):

        '''function that checks if monthly median is greater than a scientifically decreed upper limit, variable per parameter

           check is only done when have >= 1 valid values after screening by certain qa flags, to ensure have sufficient values to calculate median
           when grouping measurements per month, the midpoint of the measurement window is taken as the point to group by
        '''

        #get relevant qa flag code
        qa_flag_code = standard_QA_name_to_QA_code['Data Outlier - Monthly Median Exceeds Scientifically Decreed Upper Limit']
        #define invalid qa flags 
        invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Invalid Data Provider Flags - GHOST Decreed', 'Below Preferential Lower Limit of Detection', 'Above Preferential Upper Limit of Detection', 'Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit', 'Data Outlier - Network Decreed', 'Data Outlier - Manually Decreed']               
 
        #iterate through monthly grouped data 
        for month in df_month_group_names:
            month_data = df_month.get_group(month)
        
            #have some valid data to proceed? 
            if len(month_data) > 0:
        
                #remove invalid data (decreed from qa flags)
                filtered_month_data = filter_by_qa(invalid_qa_flag_names, month_data['data'].values, self.data['qa'][month_data['valid_inds'].values])
            
                #check number of total measurements after filtering in the month
                #if < 1 values in the month, then do not perform qa check
                if len(filtered_month_data) < 1:
                    continue
    
                #otherwise, get monthly median
                else:    
                    monthly_median = np.median(filtered_month_data)

                    #if monthly median exceeds an upper limit, then flag entire month   
                    if monthly_median > self.parameter_details['extreme_upper_monthly_median']:
                        for inv in month_data['valid_inds'].values:
                            self.data['qa'][inv].append(qa_flag_code)
                        

    def check_monthly_distribution_consistency(self, df_month, df_month_group_names):
    
        '''Function that determines how consistent a monthly distribution is with other distributions for the same month, across the years.  

           This is done by calculating how consistent a monthly distribution of measurements is with the distributions for the same month of the neighbouring 5 years (backwards and forwards in time), as well as calculating how this metric of consistency differs from all other calculated metrics for the same month, across the time series.
 
           Kernel density estimation (kde) is used to estimate the probability density function (PDF) for the month in question, as well as the neighbouring year months.  
           This PDF is only estimated when there are >= 100 valid values after screening by certain qa flags, and when there is a minimum of 3 unique values in the month, to ensure there are sufficient values to estimate the full monthly distribution. A qa flag is written for the month when the PDF cannot be estimated.
           
           The intersection is then taken between the month in question's PDF and the surrounding months PDF's, for which 1.0 - the median intersection value is taken as the metric of consistency (perfect intersection = 0.0, no intersection = 1.0). 
           Only the neighbouring 5 years are used, to limit the impact of any long-term trends biasing the comparison. 
           The metric of consistency if not calculated for month if have less than 2 intersections with neighbouring months. The month will be flagged as unclassified.

           Additionally determined is to what extent does the median PDF intersection metric per month differ from the typical (median) intersection score for the same month across the entire timerseries (0.0 = perfectly consistent or better than the median intersection score, 1.0 = intersection score is as far below as possible the median intersection score). 
           This metric provides a weighting for the multi-annual variance of the monthly distributions. If there is a low intersection score, but there are typically always low intersection scores due to high variance across the years, then the difference score will be close to 0.0. Conversely, if there is a low intersection score, but there are typically always high intersection scores, then the difference score will be closer to 1.0, as it is outlying.           
           If there are < 3 intersections for a specific month, then do not calculate difference metric for the month, and all relevant yearmonths are flagged as unclassified. 

           These metrics are then both summed to give a total classification score of the normality of the monthly distribution (2.0 = Extremely different from typical case. 0.0 = Extremely consistent with typical case).
           The classification score is split into 10 zones (in range increments of 0.2) from the most consistent monthly distributions in Zone 1 (score of 0.0 to 0.2), to the least consistent monthly distributions in Zone 10 (score of 1.8 to 2.0).
           All months for which a total classification score can be determined are flagged with a flag for the respective classification score zone.

           If 2/3, 4/6, 8/12 consecutive months are classed as zone 6 or higher, then it is suspected there is a systematic reason for the non-typical distributions, and the entire periods are flagged. 

           When grouping measurements per month, the midpoint of the measurement window is taken as the point to group by.       
        '''

        #get min/max of entire time series after filtering by QA 
        invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Invalid Data Provider Flags - GHOST Decreed', 'Erroneous Primary Sampling', 'Erroneous Sample Preparation', 'Below Preferential Lower Limit of Detection', 'Above Preferential Upper Limit of Detection', 'Monthly Fractional Unique Values <= 1%', 'Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit', 'Data Outlier - Network Decreed', 'Data Outlier - Manually Decreed']
        filtered_data = filter_by_qa(invalid_qa_flag_names, self.data[self.parameter_details['bsc_parameter_name']], self.data['qa'])
        #do not proceed if have very low number of data (i.e. impossible to have enough data for even 1 valid monthly distribution), or very low number of uniques, instead add unclassified flag to every measurement
        if (len(filtered_data) < 100) or (len(np.unique(filtered_data)) < 3):
            qa_flag_code = standard_QA_name_to_QA_code['Monthly Distribution Consistency - Unclassified'] 
            for inv in range(len(self.data['qa'])):
                if qa_flag_code not in self.data['qa'][inv]:
                    self.data['qa'][inv].append(qa_flag_code)
            return
        
        #define grid to sample values of the monthly PDFs, between min and max
        #by default sample the the PDF in intervals of the minimum resolution/100., with a minimum of 8000 samples required
        #set 8000 total samples if no minimum resolution is available 
        min_x = np.min(filtered_data)
        max_x = np.max(filtered_data)
        minmax_diff = max_x - min_x
        if pd.isnull(self.parameter_details['minimum_resolution']):
            n_samples = 8000  
        else:
            n_samples = int(np.around(minmax_diff/(self.parameter_details['minimum_resolution']/100.0),0))  
            if n_samples < 8000:
                n_samples = 8000

        # round n_samples to next next power of 2 (for fft optimisation)
        n_samples = 2 ** np.ceil(np.log2(n_samples)) 
        # set x_grid
        x_grid = np.linspace(min_x,max_x,n_samples,endpoint=True)

        #define dict to store monthly PDFs
        monthly_PDFs = {}

        #iterate through monthly grouped data estimating PDF of each valid month 
        for key in df_month_group_names:
            month_data = df_month.get_group(key)
        
            #have some valid data to proceed? 
            if len(month_data) > 0:

                #remove invalid data (decreed from qa flags)
                filtered_month_data = filter_by_qa(invalid_qa_flag_names, month_data['data'].values, self.data['qa'][month_data['valid_inds'].values])

                #check number of total measurements after filtering in the month
                #if < 100 values in the month, or < 3 total unique values, then do not take PDF, and flag month as unclassified 
                if (len(filtered_month_data) < 100) or (len(np.unique(filtered_month_data)) < 3):
                    qa_flag_code = standard_QA_name_to_QA_code['Monthly Distribution Consistency - Unclassified'] 
                    for inv in month_data['valid_inds'].values:
                        self.data['qa'][inv].append(qa_flag_code)
                    continue

                #get kernel density estimation of monthly probability density function (using Gaussian kernels)
                #Bandwidth selection is done using Scott's rule    
                #sometimes if the monthly values lie in a very small band over the distribution, 
                # the following error is thrown: "ValueError: math domain error".
                #in this case fall back to using slower, but reliable scipy kde calculation
                try:
                    monthly_PDFs[key] = kde_fft(filtered_month_data, xgrid=x_grid)      
                except ValueError as error:
                    kde_model = st.gaussian_kde(filtered_month_data, bw_method='scott')
                    monthly_PDFs[key] = kde_model(x_grid)

        #create dicts to store neighbouring PDF intersection medians per month 
        intersection_medians_grouped_by_month = {str(mm):{'median_intersections':[],'yearmonths':[]} for mm in range(1,13)}
        intersection_medians_with_relevant_yearmonths = {} 

        #iterate through monthly PDFs - calculating metric of consistency with neighbouring +/- 5 years of the same month
        #perfect intersection = 0.0, no intersection = 1.0
        #do not calculate metric of consistency for month if have less than 2 intersections with neighbouring months
        for key in monthly_PDFs.keys():

            #get current year/month
            current_year = key.year
            current_month = key.month
            #get current yearmonth PDF 
            current_yearmonth_pdf = monthly_PDFs[key]
    
            #get intersections with with the PDFs for the same month of the neighbouring 5 years (+/- 5 years) 
            intersections = [] 
            for n_years_offset in [-5,-4,-3,-2,-1,1,2,3,4,5]:
                offset_yearmonth_key = key + pd.DateOffset(years=n_years_offset)
                if offset_yearmonth_key in list(monthly_PDFs.keys()):        
                    offset_yearmonth_pdf = monthly_PDFs[offset_yearmonth_key]
                    intersections = np.append(intersections, return_PDF_intersection(current_yearmonth_pdf,offset_yearmonth_pdf))

            #if there less than 2 intersections with neighbouring months, then do not calculate metric of consistency for month       
            if len(intersections) < 2:
                intersection_medians_with_relevant_yearmonths[key] = -99
                intersection_medians_grouped_by_month[str(current_month)]['yearmonths'].append(key)
                intersection_medians_grouped_by_month[str(current_month)]['median_intersections'].append(-99)

            #else, get median intersection with neighbouring months
            else:
                intersection_median = np.median(intersections)
                #take 1.0 - median intersection to be the monthly metric of consistency (0.0 = perfect intersection, 1.0 = no intersection)
                intersection_medians_with_relevant_yearmonths[key] = 1.0 - intersection_median
                #group metric of consistency for current month with other values for the same month 
                intersection_medians_grouped_by_month[str(current_month)]['yearmonths'].append(key)
                intersection_medians_grouped_by_month[str(current_month)]['median_intersections'].append(intersection_median)

        intersection_diffs_with_monthly_median = {}        
        #iterate through monthly metrics of consistency - calculating how the metrics differ from the typical (median) scores for the same month across the entire timerseries 
        #0.0 = perfectly consistent or better than the median intersection score, 1.0 = intersection score is as far below as possible the median intersection score 
        #if the monthly metric of consistency is > than the median score across the timeseries, then the score is 0 (i.e. perfectly consistent with other scores)
        #if there are < 3 intersections for the month, then do not calculate difference metric for the month
        for month in intersection_medians_grouped_by_month.keys():
            
            #get current month intersections/relevant yearmonths
            current_month_intersections = np.array(intersection_medians_grouped_by_month[month]['median_intersections'], dtype=np.float32)
            current_yearmonths = np.array(intersection_medians_grouped_by_month[month]['yearmonths'])
            #get indices of intersections >= 0.0
            valid_month_intersections = current_month_intersections >= 0.0
            current_month_intersections = current_month_intersections[valid_month_intersections]
            #set all yearmonths with intersections < 0.0, to be -99
            for key in current_yearmonths[~valid_month_intersections]:
                intersection_diffs_with_monthly_median[key] = -99
            
            #if have less then 3 intersection scores for specific month, then do not calculate difference score for month       
            if len(current_month_intersections) < 3:
                for key in current_yearmonths[valid_month_intersections]:
                    intersection_diffs_with_monthly_median[key] = -99
             
            else:
                monthly_median_intersection = np.median(current_month_intersections)
                diffs_from_median = monthly_median_intersection - current_month_intersections
                diffs_from_median[np.where(diffs_from_median < 0.0)[0]] = 0.0
                normalised_diffs_from_median = normalise_by_defined_max(diffs_from_median,max_val=monthly_median_intersection)
                for diff, key in zip(normalised_diffs_from_median, current_yearmonths[valid_month_intersections]):
                    intersection_diffs_with_monthly_median[key] = diff

        #iterate through all months, and add the 2 calculated consistency metrics per month to generate a total classification score
        #2.0 = Extremely different from typical case. 0.0 = Extremely consistent with typical case
        #The classification score is split into 10 zones (in range increments of 0.2) from the most consistent monthly distributions in Zone 1 (score of 0.0 to 0.2), to the least consistent monthly distributions in Zone 10 (score of 1.8 to 2.0).      
        #All months for which a total classification score can be determined are flagged with a flag for the respective classification score zone.
        #If 2/3, 4/6, 8/12 consecutive months are classed as zone 6 or higher, then it is suspected there is a systematic reason for the non-typical distributions, and the entire periods are flagged. 
        monthly_classfication_scores = {}
        systematically_biased_months_2_outof_3 = []
        systematically_biased_months_4_outof_6 = []
        systematically_biased_months_8_outof_12 = []    
        for key in intersection_medians_with_relevant_yearmonths.keys():
            classification_score = intersection_medians_with_relevant_yearmonths[key] + intersection_diffs_with_monthly_median[key]    
            if classification_score < 0.0:
                monthly_classfication_scores[key] = -99
            elif classification_score < 0.2:
                monthly_classfication_scores[key] = 1
            elif classification_score < 0.4:
                monthly_classfication_scores[key] = 2
            elif classification_score < 0.6:
                monthly_classfication_scores[key] = 3
            elif classification_score < 0.8:
                monthly_classfication_scores[key] = 4
            elif classification_score < 1.0:
                monthly_classfication_scores[key] = 5
            elif classification_score < 1.2:
                monthly_classfication_scores[key] = 6
            elif classification_score < 1.4:
                monthly_classfication_scores[key] = 7
            elif classification_score < 1.6:
                monthly_classfication_scores[key] = 8
            elif classification_score < 1.8:
                monthly_classfication_scores[key] = 9
            else:
                monthly_classfication_scores[key] = 10

            #if 2/3, 4/6, 8/12 previous consecutive months are >= zone 6, add tag to flag all months as being systematically biased
            last_3_month_classfication_scores = [monthly_classfication_scores[key]]
            last_6_month_classfication_scores = [monthly_classfication_scores[key]]
            last_12_month_classfication_scores = [monthly_classfication_scores[key]]
            for n_month_offset in [-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11]:
               offset_key = key + pd.DateOffset(months=n_month_offset)
               if offset_key in monthly_classfication_scores.keys():
                   offset_classification_score = monthly_classfication_scores[offset_key]
                   if np.abs(n_month_offset) < 3: 
                       last_3_month_classfication_scores.append(offset_classification_score)
                   if np.abs(n_month_offset) < 6:
                       last_6_month_classfication_scores.append(offset_classification_score)
                   last_12_month_classfication_scores.append(offset_classification_score) 
            #2/3 consecutive months >= zone 6?
            if sum(i >= 6 for i in last_3_month_classfication_scores) >= 2: 
                if key not in systematically_biased_months_2_outof_3:
                    systematically_biased_months_2_outof_3.append(key)
                    for n_month_offset in [-1,-2]:
                        offset_key = key + pd.DateOffset(months=n_month_offset)
                        if offset_key not in systematically_biased_months_2_outof_3:
                            systematically_biased_months_2_outof_3.append(offset_key)  
            #4/6 consecutive months >= zone 6?
            if sum(i >= 6 for i in last_6_month_classfication_scores) >= 4: 
                if key not in systematically_biased_months_4_outof_6:
                    systematically_biased_months_4_outof_6.append(key)
                    for n_month_offset in [-1,-2,-3,-4,-5]:
                        offset_key = key + pd.DateOffset(months=n_month_offset)
                        if offset_key not in systematically_biased_months_4_outof_6:
                            systematically_biased_months_4_outof_6.append(offset_key) 
            #8/12 consecutive months >= zone 6?
            if sum(i >= 6 for i in last_12_month_classfication_scores) >= 8: 
                if key not in systematically_biased_months_8_outof_12:
                    systematically_biased_months_8_outof_12.append(key)
                    for n_month_offset in [-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11]:
                        offset_key = key + pd.DateOffset(months=n_month_offset)
                        if offset_key not in systematically_biased_months_8_outof_12:
                            systematically_biased_months_8_outof_12.append(offset_key) 

        #write out qa flags per month
        for key in monthly_classfication_scores.keys():
            month_data = df_month.get_group(key)
            #get classification score qa code
            if monthly_classfication_scores[key] < 0.0:
                qa_flag_code = standard_QA_name_to_QA_code['Monthly Distribution Consistency - Unclassified']
            else:
                qa_flag_code = standard_QA_name_to_QA_code['Monthly Distribution Consistency - Zone {}'.format(monthly_classfication_scores[key])]
            #write classification score qa code for entire month
            for inv in month_data['valid_inds'].values:
                self.data['qa'][inv].append(qa_flag_code)
            #have systematic bias qa flags for month? if so write flags also for entire month. 
            if key in systematically_biased_months_2_outof_3:
               qa_flag_code = standard_QA_name_to_QA_code['Systematic Inconsistent Monthly Distributions - 2/3 Months >= Zone 6'] 
               for inv in month_data['valid_inds'].values:
                   self.data['qa'][inv].append(qa_flag_code)
            if key in systematically_biased_months_4_outof_6:
               qa_flag_code = standard_QA_name_to_QA_code['Systematic Inconsistent Monthly Distributions - 4/6 Months >= Zone 6'] 
               for inv in month_data['valid_inds'].values:
                   self.data['qa'][inv].append(qa_flag_code)
            if key in systematically_biased_months_8_outof_12:
               qa_flag_code = standard_QA_name_to_QA_code['Systematic Inconsistent Monthly Distributions - 8/12 Months >= Zone 6'] 
               for inv in month_data['valid_inds'].values:
                   self.data['qa'][inv].append(qa_flag_code)


    def filter_by_temporal_resolution(self, n_mins_resolution_min, n_mins_resolution_max):

        '''function that isolates indices of measurements that fall within defined temporal resolution range.
           measurements are retained that are <= than the resolution_min, and are >= the resolution_max.
        '''

        valid_indices = []
        for ii, start_time in enumerate(self.data['time']):
            temporal_resolution = self.data['temporal_resolution'][ii]
            if (temporal_resolution <= n_mins_resolution_min) & (temporal_resolution >= n_mins_resolution_max):
                valid_indices.append(ii)
        
        return np.array(valid_indices,dtype=np.int)
        
    
    def group_by_period(self, period, df):

        '''function that groups measurements by a defined period (e.g. daily/weekly/monthly)'''
    
        return df.groupby(pd.Grouper(freq=period))
        

    def grouped_measurement_checks(self):
 
        '''function that runs qa checks which require grouping by temporal_resolution/temporal window.
           function first groups measurements into multiple groups by temporal resolution ranges,
           then subsequently sub-groups data into windows across the time series (i.e. daily, weekly, monthly),
           then finally passes the grouped data through numerous QA checks (which vary per resolution group)

           for grouping data into daily/weekly/monthly windows, the midpoint of the measurement period is used.
        '''

        #define temporal resolution range groups
        temporal_resolution_range_groups = {
        '0hour_to_<1hour':{'min':59, 'max':0},
        '1hour_to_<4hours':{'min':239, 'max':60},
        '4hours_to_<1day':{'min':1439, 'max':240},
        '1day_to_<3days':{'min':4319, 'max':1440},
        '3days_to_<10days':{'min':14399, 'max':4320}, 
        '10days_to_1month':{'min':44640, 'max':14400},
        '1hour_to_<1day':  {'min':1439, 'max':60}
        }

        #get midpoint time as datetime
        dt_time = pd.DatetimeIndex([fractional_minutes_from_reference_to_datetime(t+(res/2.)) for t,res in zip(self.data['time'],self.data['temporal_resolution'])])
        
        #--------------------------------------------#        
        #perform required qa checks for 1hour to <1day range
 
        #get indices of valid measurements 
        valid_indices = self.filter_by_temporal_resolution(temporal_resolution_range_groups['1hour_to_<1day']['min'], temporal_resolution_range_groups['1hour_to_<1day']['max'])

        #non-integer local timezone check
        self.check_non_integer_local_timezone(valid_indices)

        #--------------------------------------------#        
        #perform required qa checks for other temporal resolution ranges

        #iterate through temporal resolution ranges
        for temporal_resolution_range_group in temporal_resolution_range_groups.keys():
            if temporal_resolution_range_group != '1hour_to_<1day':

                #get indices of valid measurements 
                valid_indices = self.filter_by_temporal_resolution(temporal_resolution_range_groups[temporal_resolution_range_group]['min'], temporal_resolution_range_groups[temporal_resolution_range_group]['max'])

                #create dataframe of measured valid data
                pd_data = {'data':self.data[self.parameter_details['bsc_parameter_name']][valid_indices],'valid_inds':valid_indices}
                valid_df = pd.DataFrame(pd_data,index=dt_time[valid_indices],columns=['data','valid_inds'])

                #group data into monthly chunks
                df_month = self.group_by_period('MS', valid_df)

                #get valid group names (i.e. groups with some data)
                df_month_agg = df_month.agg('count')
                df_month_group_names = [k for k,v in zip(df_month_agg.index, df_month_agg.values) if v[0] > 0]   

                #percent unique values in month check
                self.check_unique_values(df_month, df_month_group_names)
            
                #empirical measurement resolution check
                self.check_measurement_resolution_empirical(df_month, df_month_group_names)

                #monthly adjusted boxplot check
                self.check_monthly_adjusted_boxplot(df_month, df_month_group_names)

                #data outlier - scientifically non-feasible monthly median check
                self.check_data_outlier_scientifically_non_feasible_monthly_median(df_month, df_month_group_names)
    
                #monthly distribution consistency check
                self.check_monthly_distribution_consistency(df_month, df_month_group_names)
