#!/usr/bin/env python
# -*- coding: utf-8 -*-

#WRITTEN BY DENE BOWDALO

###------------------------------------------------------------------------------------###
###IMPORT MODULES
###------------------------------------------------------------------------------------###

import bisect
from calendar import monthrange
import copy
from collections import Counter
import crossnetwork_processing_modules
import datetime
import ephem
import itertools
import math
import numpy as np
import pandas as pd
import scipy.stats as st
import standardisation_modules
import statsmodels.stats.stattools as sm

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###

#modules that applies each of modular data quality checks to data from each site
def do_qa(metadata,data):
    
    #key metadata check    
    data, exit_flag = check_key_metadata(metadata,data)
    if exit_flag == True:
        return data, exit_flag
        
    #altitude metadata check
    data, exit_flag = check_altitude_metadata(metadata,data)
    if exit_flag == True:
        return data, exit_flag
    
    #missing measurements check
    data = check_missing_measurement(data)
    
    #negative measurement check (only do check for gas and aerosol matrices)
    if (parameter_details['matrix'] == 'gas') or (parameter_details['matrix'] == 'aerosol'):
        data = check_negative_measurement(data)
    
    #zero values check (only do check for gas and aerosol matrices)
    if (parameter_details['matrix'] == 'gas') or (parameter_details['matrix'] == 'aerosol'):
        data = check_zero_values(data)
    
    #infinite values check
    data = check_infinite_values(data)
    
    #LOD checks (only do check for gas and aerosol matrices)
    if (parameter_details['matrix'] == 'gas') or (parameter_details['matrix'] == 'aerosol'):
        data = check_lod(metadata,data)

    #nighttime check
    data = check_nighttime(metadata,data)
    
    #weekday check
    data = check_weekday(data)
    
    #extreme data - scientifically non-feasible check
    data = check_extreme_scientifically_non_feasible(data)
    
    #extreme data - manually decreed check
    data = check_extreme_manual(metadata,data)
    
    #recurring value check
    data = check_recurring_values(data)
    
    #measurement resolution check - by reported metadata
    data = check_measurement_resolution_by_metadata(metadata,data)
    
    #run qa checks which process data by year/month (UTC)
    #first temporal data is split out in a standalone module and this module then calls the qa checks - this is done for code optimisation
    #the qa checks called are: 
    #non-integer local timezone check
    #extreme data - distributional outlier check
    #measurement resolution check - empirical
    #hitting LOD check
    data = iterate_by_yearmonth(metadata,data)
    
    return data, False
    

###------------------------------------------------------------------------------------###
###------------------------------------------------------------------------------------###
#QUALITY CHECK MODULES

#Check have key metadata (latitude, longitude), i.e. they have a numerical value.
#Also check both latitude and longitude are not set both as 0 (must be bug as nothing but ocean there)
def check_key_metadata(metadata,data):
    
    lats = metadata['latitude']
    lons = metadata['longitude']
    update_metadata_stamp = metadata['meta_update_stamp']
    
    valid = True
    invalid_inds = np.empty(0, dtype=np.uint32)
    
    for ii in range(len(lats)):
        #are any of key metadata variables empty?
        if pd.isnull(lats[ii]) == True:
            valid = False
        elif pd.isnull(lons[ii]) == True:
            valid = False    
    
        #are lat/lon both == 0.0?
        elif (lats[ii] == 0.0) & (lons[ii] == 0.0):
            valid = False
            
        #process start/end invalid date range pairs
        if valid == False:
            
            #is it last index? 
            if ii == (len(lats)-1):
                invalid_inds = np.append(invalid_inds,np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), len(data['time'])))
            else:
                invalid_inds = np.append(invalid_inds,np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), bisect.bisect_left(data['time'], update_metadata_stamp[ii+1])))
                
        #reset valid flag
        valid = True
    
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        
        #if each metadata column has key metadata missing then, no not process further for this station
        if len(invalid_inds)  == len(data['time']):
            return data, True
        
        else: 
            #otherwise just flag period here dont have metadata with relevant qa flag
            qa_flag = standardisation_modules.get_standard_qa_flag('No Key Metadata')
            for inv in invalid_inds:
                data['qa'][inv] = data['qa'][inv] + qa_flag
    
            
    return data, False

###------------------------------------------------------------------------------------###

#Check have valid altitude metadata, i.e. it has a numerical value

def check_altitude_metadata(metadata,data):
    
    alts = metadata['altitude']
    update_metadata_stamp = metadata['meta_update_stamp']
    
    invalid_inds = np.empty(0, dtype=np.uint32)
    
    for ii in range(len(alts)):
    
        #ia altitude value NaN?
        if pd.isnull(alts[ii]) == True:
            
            #is it last index? 
            if ii == (len(lats)-1):
                invalid_inds = np.append(invalid_inds,np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), len(data['time'])))
            else:
                invalid_inds = np.append(invalid_inds,np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), bisect.bisect_left(data['time'], update_metadata_stamp[ii+1])))
                
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        
        #if each metadata column has altitude metadata missing then, no not process further for this station
        if len(invalid_inds)  == len(data['time']):
            return data, True
        
        else: 
            #otherwise just flag period here dont have metadata with relevant qa flag
            qa_flag = standardisation_modules.get_standard_qa_flag('No Altitude Metadata')
            for inv in invalid_inds:
                data['qa'][inv] = data['qa'][inv] + qa_flag
    
            
    return data, False

###------------------------------------------------------------------------------------###

#Check if have missing measurements (i.e NaN)
def check_missing_measurement(data):
    
    invalid_inds = np.arange(len(data[parameter_details['bsc_parameter_name']]))[pd.isnull(data[parameter_details['bsc_parameter_name']])]
    
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        #get relevant qa flag
        qa_flag = standardisation_modules.get_standard_qa_flag('Missing Measurement')
        for inv in invalid_inds:
            data['qa'][inv] = data['qa'][inv] + qa_flag
            
    return data 

###------------------------------------------------------------------------------------###

#Check if have negative measurements (i.e < 0), or have network flags stating negative measurements (X6)
def check_negative_measurement(data):
    
    invalid_inds_empirical = np.where(data[parameter_details['bsc_parameter_name']] < 0.0)[0]
    invalid_inds_flags = [ii for ii,flag in enumerate(data['flag']) if 'X6' in flag]
    
    if (len(invalid_inds_empirical) > 0) & (len(invalid_inds_flags) > 0):
        #get unique indices from both lists
        invalid_inds = list(set(invalid_inds_empirical+invalid_inds_flags))
    elif len(invalid_inds_empirical) > 0:
        invalid_inds = invalid_inds_empirical
    elif len(invalid_inds_flags) > 0:
        invalid_inds = invalid_inds_flags
    else:
        invalid_inds = []
    
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        #get relevant qa flag
        qa_flag = standardisation_modules.get_standard_qa_flag('Negative Measurement')
        for inv in invalid_inds:
            data['qa'][inv] = data['qa'][inv] + qa_flag
            
    return data 

###------------------------------------------------------------------------------------###

#Check if have values == 0
def check_zero_values(data):

    invalid_inds = np.where(data[parameter_details['bsc_parameter_name']] == 0.0)[0]
    
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        #get relevant qa flag
        qa_flag = standardisation_modules.get_standard_qa_flag('Zero Measurement')
        for inv in invalid_inds:
            #check that do not have negative measurement flag first (happens when networks report negative measurements and change values to be zero)
            if standardisation_modules.get_standard_qa_flag('Negative Measurement') not in data['qa'][inv]:
                data['qa'][inv] = data['qa'][inv] + qa_flag
            
    return data 

###------------------------------------------------------------------------------------###

#Check if have infinite values (i.e. np.inf).
#This happens when values are outside of the range that the float32 data type can handle (-3.4E+38 to +3.4E+38)
def check_infinite_values(data):
    
    invalid_inds = np.arange(len(data[parameter_details['bsc_parameter_name']]))[np.isinf(data[parameter_details['bsc_parameter_name']])]
    
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        #get relevant qa flag
        qa_flag = standardisation_modules.get_standard_qa_flag('Infinite Value')
        for inv in invalid_inds:
            data['qa'][inv] = data['qa'][inv] + qa_flag
             
    return data 

###------------------------------------------------------------------------------------###

#iterate through measured data, and check if the values are <=/>= the reported/documented limits of detection
#the reported limits of detection are either variable in time, or as a fixed value for an entire data range.
#additionally network providers often give flags stating if data is <=/>= detection limits
#these flags are first used in preference over testing if measurements are above/below static/variable detection limits
#variable LODs are then taken preferentially over static LODs where both are available.  
#if have no valid LODs or flags, no measurements are flagged

def check_lod(metadata,data):
    
    #get relevant qa flags
    below_lod_documented_qa_flag = standardisation_modules.get_standard_qa_flag('Below Documented Lower Limit of Detection')
    below_lod_reported_qa_flag = standardisation_modules.get_standard_qa_flag('Below Reported Lower Limit of Detection')
    above_lod_documented_qa_flag = standardisation_modules.get_standard_qa_flag('Above Documented Upper Limit of Detection')
    above_lod_reported_qa_flag = standardisation_modules.get_standard_qa_flag('Above Reported Upper Limit of Detection')

    #get needed data/metadata
    
    static_lower_lod_documented = metadata['measuring_instrument_documented_lower_limit_of_detection']
    static_lower_lod_reported = metadata['measuring_instrument_reported_lower_limit_of_detection']
    static_upper_lod_documented = metadata['measuring_instrument_documented_upper_limit_of_detection']
    static_upper_lod_reported = metadata['measuring_instrument_reported_upper_limit_of_detection']
    update_metadata_stamp = metadata['meta_update_stamp']

    #iterate through #iterate through all time points
    for ii in range(len(data['time'])):
        
        #preferentially check if have network data flags informing of hitting of LOD
        current_flag = data['flag'][ii]
        
        #if have network flag stating it is <=/>= detection limits, write reported qa flag
        #have flag stating measurement is <= detection limit?
        if 'E1' in current_flag:
            data['qa'][ii] = data['qa'][ii] + below_lod_reported_qa_flag
                
        #have flag stating measurement is >= detection limit?
        if 'E2' in current_flag:
            data['qa'][ii] = data['qa'][ii] + above_lod_reported_qa_flag 
                    
        #get utc time in minutes
        current_minutes = data['time'][ii]
        
        #get measurement value for timepoint
        current_data = data[parameter_details['bsc_parameter_name']][ii]
        
        #if current data is NaN, skip to next value
        if pd.isnull(current_data) == True:
            continue
        
        #get index of current valid metadata for timepoint
        current_meta_index = bisect.bisect(update_metadata_stamp,current_minutes)-1
        #use index to get current metadata
        current_lower_lod_documented = static_lower_lod_documented[current_meta_index]
        current_static_lower_lod_reported = static_lower_lod_reported[current_meta_index]
        current_upper_lod_documented = static_upper_lod_documented[current_meta_index]
        current_static_upper_lod_reported = static_upper_lod_reported[current_meta_index]
        
        #test measurements by documented LODs
        #is measurement below or equal to documented lower limit of detection?
        if current_data <= current_lower_lod_documented:
            data['qa'][ii] = data['qa'][ii] + below_lod_documented_qa_flag
            
        #is measurement above or equal to documented upper limit of detection?
        if current_data >= current_upper_lod_documented:
            data['qa'][ii] = data['qa'][ii] + above_lod_documented_qa_flag  
        
        #test measurements by reported LODs
        #get variable reported limit of detections
        current_variable_lower_lod_reported = data['reported_lower_limit_of_detection_per_measurement'][ii]
        current_variable_upper_lod_reported = data['reported_upper_limit_of_detection_per_measurement'][ii]
        
        #if both reported variable and static LODs are not NaN, preferentially keep variable reported LODs
        if pd.isnull(current_variable_lower_lod_reported) == False:
            current_lower_lod_reported = current_variable_lower_lod_reported
        else:
            current_lower_lod_reported = current_static_lower_lod_reported
            
        if pd.isnull(current_variable_upper_lod_reported) == False:
            current_upper_lod_reported = current_variable_upper_lod_reported
        else:
            current_upper_lod_reported = current_static_upper_lod_reported
        
        #is measurement below or equal to reported lower limit of detection, and do not have network flag stating below LOD? 
        if (current_data <= current_lower_lod_reported) & ('E1' not in current_flag):
            data['qa'][ii] = data['qa'][ii] + below_lod_reported_qa_flag
            
        #is measurement above or equal to reported upper limit of detection, and do not have network flag stating above LOD?
        if (current_data >= current_upper_lod_reported) & ('E2' not in current_flag):
            data['qa'][ii] = data['qa'][ii] + above_lod_reported_qa_flag  
    
    return data

###------------------------------------------------------------------------------------###

#calculate the solar elevation angle for a latitude/longitude/measurement height at a certain timestamp.
#he solar elevation angle is the angle between the horizon and the centre of the sun's disc.

#the distinction between day and night is partially subjective. 
#there are 3 intermediary states between night and day: 
#1. astronomical twilight (where solar elevation angle is between -18 deg. and -12 deg.)
#2. nautical twilight (where solar elevation angle is between -12 deg. and -6 deg.)
#3. civil twilight (where solar elevation angle is between -6 deg. and 0 deg.)
#depending on which of these states are classed as day/night  

#sunrise is classically defined as when these states end (where solar elevation angle is >= 0)
#sunset is classically defined as when these states start (where solar elevation angle is < 0)

#for atmospheric chemistry, the presence of any light (however weak it may be) will start photochemistry.
#therefore it is defined if the solar elevation angle is <= -18 degrees, then the time is classed as night. 

#station altitude + sampling height is added to produce a measurement height
#if no sampling height is available, only station altitude is used.
#if any of latitude/longitude/measurement height is NaN, do not do check

#only do check if measurement resolution is finer than 1 day
#do check for time in centre of measurement window (therefore if 51% of measurement window is night - the window will be classed as night)

def check_nighttime(metadata,data):

    #get relevant qa flags
    qa_flag_day =   standardisation_modules.get_standard_qa_flag('Daytime')
    qa_flag_night = standardisation_modules.get_standard_qa_flag('Nighttime')
    
    #set ephem location instance
    sun = ephem.Sun()
    observer = ephem.Observer()
    
    #get altitude/sampling height to produce measurement altitudes
    lats = metadata['latitude']
    lons = metadata['longitude']
    alts = metadata['altitude']
    sampling_heights = metadata['sampling_height']
    measurement_alts = copy.deepcopy(alts)
    for ii in range(len(measurement_alts)):
        #if have valid sampling height and altitude, create a measurement altitude
        if (pd.isnull(sampling_heights[ii]) == False) & (pd.isnull(alts[ii]) == False):
            measurement_alts[ii] = sampling_heights[ii] + alts[ii]
    
    #iterate through metadata
    current_ind = 0
    for ii in range(len(metadata['meta_update_stamp'])): 
        
        #get valid time indices for metadata window
        #is it last index? 
        if ii == (len(metadata['meta_update_stamp'])-1):
            valid_time_inds = np.arange(bisect.bisect_left(data['time'], metadata['meta_update_stamp'][ii]), len(data['time']))
        else:
            valid_time_inds = np.arange(bisect.bisect_left(data['time'], metadata['meta_update_stamp'][ii]), bisect.bisect_left(data['time'], metadata['meta_update_stamp'][ii+1]))
        
        #get relevant time for metadata window
        cut_time = data['time'][valid_time_inds]
        
        #convert times to datetimes
        datetimes = []
        for current_minutes in cut_time:
            #get datetime
            datetimes.append(crossnetwork_processing_modules.fractional_minutes_from_reference_to_datetime(float(current_minutes)))
    
        #if any of latitude/longitude/measurement altitude are NaN, do not do check
        if (pd.isnull(lats[ii]) == True) or (pd.isnull(lons[ii]) == True) or (pd.isnull(measurement_alts[ii]) == True):
            current_ind+=len(datetimes)
            continue
    
        #define observer coordinates
        observer.lat = lats[ii]
        observer.lon = lons[ii]
        observer.elevation = measurement_alts[ii]

        #iterate through relevant datetimes, and determine if time is nighttime
        for tt in range(len(datetimes)):
            
            proceed_with_check = False
            #only do check if measurement is finer than 1 day
            temporal_resolution = data['temporal_resolution'][current_ind]
            try:
                temporal_resolution = int(temporal_resolution)
                if temporal_resolution < (60*24):
                    proceed_with_check = True
            except ValueError:
                if temporal_resolution == 'H':
                    proceed_with_check = True
                    
            if proceed_with_check == True:

                #Get datetime in centre of measurement window
                #Get timedelta to get to centre of measurement window
                if temporal_resolution == 'H':
                    timedelta = datetime.timedelta(minutes=30)
                else:
                    timedelta = datetime.timedelta(minutes=float(temporal_resolution)/2.)
                
                #Datetime at centre of measurement window in UTC 
                observer.date = datetimes[tt]+timedelta
                #compute solar altitude 
                sun.compute(observer)
                current_sun_alt = sun.alt
                #compute solar elevation angle 
                solar_elevation_angle_centre_window = (current_sun_alt*180/math.pi)
                
                #if solar elevation angle <= -18 degrees, time is classed as nightime  
                if solar_elevation_angle_centre_window <= -18.0:
                    #write out qa flag
                    data['qa'][current_ind] = data['qa'][current_ind] + qa_flag_night
                #otherwise time is classed as daytime
                else:
                    #write out qa flag
                    data['qa'][current_ind] = data['qa'][current_ind] + qa_flag_day
            
            current_ind+=1
            
    return data

###------------------------------------------------------------------------------------###

#module that checks if the time of measurement is a weekday, or weekend.

#only do check if measurement resolution is finer <= 2 days
#do check for time in centre of measurement window (therefore if 51% of measurement window is weekday - the window will be classed as weekday)

def check_weekday(data):

    #get relevant qa flags
    qa_flag_weekday =   standardisation_modules.get_standard_qa_flag('Weekday')
    qa_flag_weekend =   standardisation_modules.get_standard_qa_flag('Weekend')
    
    time = data['time']
    
    #iterate through times - converting to datetimes
    datetimes = []
    for current_minutes in time:
        #get datetime
        datetimes.append(crossnetwork_processing_modules.fractional_minutes_from_reference_to_datetime(float(current_minutes)))
    
    
    #iterate through relevant datetimes, and determine if period of measurement is weekday or weekend
    for tt in range(len(datetimes)):
    
            proceed_with_check = False
            #only do check if measurement is finer <= 2 days
            temporal_resolution = data['temporal_resolution'][tt]
            try:
                temporal_resolution = int(temporal_resolution)
                if temporal_resolution <= (60*48):
                    proceed_with_check = True
            except ValueError:
                if (temporal_resolution == 'H') or (temporal_resolution == 'D'):
                    proceed_with_check = True
                    
            if proceed_with_check == True:
                
                #Get datetime in centre of measurement window
                #Get timedelta to get to centre of measurement window
                if temporal_resolution == 'H':
                    timedelta = datetime.timedelta(minutes=30)
                else:
                    timedelta = datetime.timedelta(minutes=float(temporal_resolution)/2.)
                centre_datetime = datetimes[tt]+timedelta
                
                #determine if centre datetime is weekday or weekend
                #get n_day of week, where Monday = 0 and Sunday = 6
                n_day_of_week = centre_datetime.weekday()
                #weekday
                if n_day_of_week < 5:
                    #write out qa flag
                    data['qa'][tt] = data['qa'][tt] + qa_flag_weekday
                #weekend
                else:
                    #write out qa flag
                    data['qa'][tt] = data['qa'][tt] + qa_flag_weekend
                     
    return data   

###------------------------------------------------------------------------------------###

#do check for extreme data that is scientifically non-feasible (i.e greater than scientifically feasible).
#for each different measured parameter, a maximum value is set, if data is greater or equal to this value, it is flagged 
#NaNs not caught by this check

def check_extreme_scientifically_non_feasible(data):

    #get relevant qa flag
    qa_flag = standardisation_modules.get_standard_qa_flag('Extreme Data - Scientifically Non-Feasible')

    #iterate through measured values
    for ii in range(len(data[parameter_details['bsc_parameter_name']])):
        
        #if measured value is greater or equal to set upper limit, flag data
        if data[parameter_details['bsc_parameter_name']][ii] >= parameter_details['extreme_upper_limit']:
            #write out qa flag
            data['qa'][ii] = data['qa'][ii] + qa_flag
            
    return data
    
###-----------------------------------------------------------------------------------###
#module for manually flagging extreme data
#if data has been manually found to be extreme, and has not been flagged by other extreme flags, then it should be manually flagged. 

#input is in the following format:
#extremes = {'parameter-station_reference':'YYYYMMDDHHMM-YYYYMMDDHHMM', 'station_reference2':YYYYMMDDHHMM-YYYYMMDDHHMM;YYYYMMDDHHMM-YYYYMMDDHHMM}
#if more than 1 bad period for a reference, it is separated by a semicolon.
#all data inside the given datetime ranges for a station reference is flagged as extreme (if have data for this window)

def check_extreme_manual(metadata, data):

    #get relevant qa flag
    qa_flag = standardisation_modules.get_standard_qa_flag('Extreme Data - Manually Decreed')

    #dictionary for definition of manually decreed extreme data windows
    extremes = {}
    
    #get all unique parameter-station references keys
    unique_keys = extremes.keys()
    
    #iterate through keys
    if len(unique_keys) > 0:
        for key in unique_keys:
            #separate into parameter, station reference 
            key_split = key.split('-')
            current_parameter, current_station_reference = key[0],key[1]
            
            #if are currently processing parameter, and station reference proceed
            if (current_parameter == run_details['parameter']) & (current_station_reference == metadata['station_reference'][0]):
                
                #get all reference times for current station_reference/parameter processed
                reference_time = data['time']
                
                #split extreme window ranges by ; (if more than 1 extreme window)
                extreme_ranges = extremes[key].split(';')
                
                #iterate through extreme window ranges
                for extreme_range in extreme_ranges:
                    #break apart range string
                    extreme_range_split = extreme_range.split('-')
                    start_dt_string, end_dt_string = extreme_range_split[0],extreme_range_split[1]
                    #convert dt strings to python datetimes
                    start_dt = datetime.datetime(start_dt_string[:4],start_dt_string[4:6],start_dt_string[6:8],start_dt_string[8:10],start_dt_string[10:])
                    end_dt = datetime.datetime(end_dt_string[:4],end_dt_string[4:6],end_dt_string[6:8],end_dt_string[8:10],end_dt_string[10:])
                    #get start time/end time in reference time
                    start_reference_time = crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(start_dt) 
                    end_reference_time = crossnetwork_processing_modules.datetime_to_fractional_minutes_from_reference(end_dt) 
                
                    #flag all indices with start times inside extreme window range
                    extreme_inds = np.where((reference_time >= start_reference_time) & (reference_time < end_reference_time))[0]
                    
                    #if have times inside extreme window, then write out flags for all measurements inside window
                    if len(extreme_inds) > 0:
                        for inv in extreme_inds:
                            data['qa'][inv] = data['qa'][inv] + qa_flag
                        
    return data
    
###------------------------------------------------------------------------------------###

#do check for persistently recurring values 
#persistently recurring values are symptomatic of when an instrument hits the detection limit, or is malfunctioning.

#check is done by using a moving window of 12 measurements. If 11/12 of values in the window are a physical value (i.e. not NaN), and the same value, flag values as recurring.

#check is only done for measurements with a temporal resolution of 1 hour or finer.

def check_recurring_values(data):

    #remove data with a temporal resolution of coarser than 1 hour
    valid_inds = np.array([False]*len(data['temporal_resolution']))
    for ii, temporal_resolution in enumerate(data['temporal_resolution']):         
        try:
            temporal_resolution = int(temporal_resolution)
            if temporal_resolution <= (60):
                valid_inds[ii] = True 
        except ValueError:
            if temporal_resolution == 'H':
                valid_inds[ii] = True  

    valid_data = data[parameter_details['bsc_parameter_name']][valid_inds]

    #generate list of lists of moving window indices
    window_width = 12
    total_n = len(valid_data)    
    moving_window_indices = [np.arange(n,n+window_width) for n in range((total_n-window_width)+1)]

    #iterate though windows
    invalid_inds = []
    for window in moving_window_indices:
        #get maximum count of values in valid value list - np.NaN values are not counted additively (i.e. will be multiple nan counts of 1)
        max_occuring_value,n_occurances = Counter(valid_data[window]).most_common(1)[0]
        #if 11 or more of values are duplicates, flag values
        if n_occurances >= 11:
            invalid_inds.append(window[valid_data[window]==max_occuring_value])
             
    #remove duplicate values from invalid inds array
    invalid_inds = list(itertools.chain.from_iterable(invalid_inds))
    invalid_inds = list(set(invalid_inds))
    
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        #get relevant qa flag
        qa_flag = standardisation_modules.get_standard_qa_flag('Recurring Value')
        for inv in invalid_inds:
            data['qa'][inv] = data['qa'][inv] + qa_flag
            
    return data
    
###-----------------------------------------------------------------------------------###

#check if measurement resolution is coarser than a set limit (variable by parameter measured). This is checked in 2 ways: by reported metadata, and empirically. Handled separately by 2 different modules.
#this module is the reported metadata check
#take reported (or if not given, documented) measurement resolution and flag associated measurements if the resolution is coarser than a set limit (variable by parameter measured)  

def check_measurement_resolution_by_metadata(metadata,data):
    
    #get key metadata
    reported_measurement_resolution = metadata['measuring_instrument_reported_measurement_resolution']
    documented_measurement_resolution = metadata['measuring_instrument_documented_measurement_resolution']
    update_metadata_stamp = metadata['meta_update_stamp']

    invalid_inds = []
    for ii in range(len(reported_measurement_resolution)):
    
        #do you have valid reported measurement resolution?
        if pd.isnull(reported_measurement_resolution[ii]) == False:
            measurement_resolution = reported_measurement_resolution[ii]
        #do you have valid documented measurement resolution?
        elif pd.isnull(documented_measurement_resolution[ii]) == False:
            measurement_resolution = documented_measurement_resolution[ii]
        #no valid measurement resolution value, continue to next metadata index
        else:
            continue
            
        #is measurement_resolution coarser (or equal to) than the set minimum resolution for the parameter being processed?
        if measurement_resolution >= parameter_details['minimum_resolution']:
        
            #is it last index? 
            if ii == (len(reported_measurement_resolution)-1):
                invalid_inds = np.append(invalid_inds,np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), len(data['time'])))
            else:
                invalid_inds = np.append(invalid_inds,np.arange(bisect.bisect_left(data['time'], update_metadata_stamp[ii]), bisect.bisect_left(data['time'], update_metadata_stamp[ii+1])))
                
    #update relevant invalid inds with qa codes 
    if len(invalid_inds) > 0:
        #get relevant qa flag
        qa_flag = standardisation_modules.get_standard_qa_flag('Insufficient Measurement Resolution - Reported')
        for inv in invalid_inds:
            data['qa'][inv] = data['qa'][inv] + qa_flag

    return data

###------------------------------------------------------------------------------------###

#check that determines if the local timezone of measurement station is non-integer - important for the temporal alignment of periodic processes (e.g. when using spectral analysis)
#check is done in 2 ways:

#1. all the saved time values (which are the start of the measurement window) with a temporal resolution of equal or coarser than 1 hour, and finer than 1 day are ran through
#if any of the times do not align with the start of each UTC hour - this implies the local timezone is non-integer

#however, sometimes times are manually shifted by small amounts by the data providers (i.e. 30 minute) to align perfectly with UTC hours. These cases should be attempted to be flagged also, which is what the next test does
#** it is important to note that it is possible that the measurements taken in places with non-integer local timezones are recorded truly aligned with UTC time. However, these instances will be flagged also.
#2. a timezone package is used to determine the local timezone UTC offset via latitude/longitude pairings.
#iterate through all lat/lon pairings, getting all relevant times within each metadata window 
#group all relevant times by year+month, and for each group get local timezone offset (in hours) relative to the UTC datetime (midnight on 1st of each month).
#if offset is non-integer, write out a flag for the whole relevant monthly window.
#only do this check if measurement resolution is finer than 1 day

def check_non_integer_local_timezone(metadata,data,day_monthly_inds,daytohour_inds):

    #get relevant qa flag
    qa_flag = standardisation_modules.get_standard_qa_flag('Non-Integer Local Timezone (relative to UTC)')

    #--------------------------------------------------------------#
    #1. check alignment of saved time values (all those with a temporal resolution of equal to or coarser than 1 hour, and as coarse as 1 day) with the start of each UTC hour
    
    #get remainder of entire time array divided by 60
    time_modulus = np.mod(data['time'][daytohour_inds],60)
    #all times with remainder not equal to 0 are not aligned with UTC hour starts and ends, therefore flag
    non_matching_time_inds = np.where(time_modulus != 0)[0]
    if len(non_matching_time_inds) != 0:
        for inv in non_matching_time_inds:
            data['qa'][inv] = data['qa'][inv] + qa_flag
            
    #--------------------------------------------------------------#
    #2. check local timezone offset from UTC in year/month groupings, using a timezone package
    
    #iterate through metadata 
    previous_valid_data = False
    for meta_i in range(len(metadata['meta_update_stamp'])): 
    
        #get valid time indices for metadata window
        #is it last index? 
        if meta_i == (len(metadata['meta_update_stamp'])-1):
            valid_time_inds = np.arange(bisect.bisect_left(data['time'], metadata['meta_update_stamp'][meta_i]), len(data['time']))
        else:
            valid_time_inds = np.arange(bisect.bisect_left(data['time'], metadata['meta_update_stamp'][meta_i]), bisect.bisect_left(data['time'], metadata['meta_update_stamp'][meta_i+1]))
        
        #cut day_monthly_inds groups for those that contain valid inds in the metadata window 
        monthly_start_datetimes = []
        for monthly_inds in day_monthly_inds:
            have_times_in_month = False
            
            #if first valid time ind is after the last ind of monthly window, skip to next month
            if valid_time_inds[0] > monthly_inds[-1]:
                continue
            #if last valid time ind is before the start ind of the monthly window, break out of processing
            if valid_time_inds[-1] < monthly_inds[0]:
                break
                
            #get relevant inds in monthly window
            start_relevant_ind = bisect.bisect_left(valid_time_inds,monthly_inds[0])
            end_relevant_ind = bisect.bisect(valid_time_inds,monthly_inds[-1])
            #cut out relevant inds in month
            relevant_inds = valid_time_inds[start_relevant_ind:end_relevant_ind]
            if len(relevant_inds) != 0:
                #get datetime of first relevant ind in month
                current_minutes = data['time'][relevant_inds[0]]
                current_dt = crossnetwork_processing_modules.fractional_minutes_from_reference_to_datetime(float(current_minutes))
                #force datetime to be first hour of first day of month
                current_dt = current_dt.replace(hour=0, day=1)
                #append list with first element being datetime at midnight on 1st of each month, and the second element a list of indices of the relevant time inds
                monthly_start_datetimes.append([current_dt,relevant_inds]) 
      
                #limit valid_time_inds for next iteration
                reference_ind = np.where(valid_time_inds == relevant_inds[-1])[0][0]
                valid_time_inds = valid_time_inds[reference_ind+1:]
                #if have no more valid inds, then break out of processing
                if len(valid_time_inds) == 0:
                    break
        
        #iterate through year/month datetime groups
        #get local UTC offset in hours relative to the UTC start of month datetime, for each latitude/longitude pairing
        for monthly_start_datetime in monthly_start_datetimes:
            
            #check if have valid latitude/longitude
            if (pd.isnull(metadata['longitude'][meta_i]) == False) & (pd.isnull(metadata['latitude'][meta_i]) == False):
                
                #call to get local utc offset is costly, therefore take shortcut if possible. If latitude/longitude have not changed from previous iteration, pytz string output is the same, therefore take pytz string from previous iteration, and input it to speed up calculation
                #else, process as normal
                if meta_i != 0:
                    if (metadata['longitude'][meta_i] == metadata['longitude'][meta_i-1]) & (metadata['latitude'][meta_i] == metadata['latitude'][meta_i-1]) & (previous_valid_data == True):
                        pytz_str, local_utc_offset = crossnetwork_processing_modules.local_offset_from_utc(monthly_start_datetime[0],longitude=metadata['longitude'][meta_i],latitude=metadata['latitude'][meta_i],pytz_local_tz=pytz_str)
                    else:
                        pytz_str, local_utc_offset = crossnetwork_processing_modules.local_offset_from_utc(monthly_start_datetime[0],longitude=metadata['longitude'][meta_i],latitude=metadata['latitude'][meta_i])
                else:
                    pytz_str, local_utc_offset = crossnetwork_processing_modules.local_offset_from_utc(monthly_start_datetime[0],longitude=metadata['longitude'][meta_i],latitude=metadata['latitude'][meta_i])
        
                #is the hour offset integer or not?
                if local_utc_offset.is_integer() == False:
                    #get relevant inds for grouping
                    relevant_inds = np.array(monthly_start_datetime[1],dtype=np.uint32)
                    
                    #limit relevant inds to those that do not have qa_flag from the first check
                    relevant_inds = np.array([ii for ii,val in enumerate(data['qa'][relevant_inds]) if qa_flag not in val])
                    
                    #write qa flag
                    for inv in relevant_inds:
                        data['qa'][inv] = data['qa'][inv] + qa_flag
    
        
        #modify variable to inform that had valid of sufficient temporal resolution to run check in previous meta period
        if len(monthly_start_datetimes) > 0:
            previous_valid_data = True
        else:
            previous_valid_data = False
        
    return data


###------------------------------------------------------------------------------------###

#do check for extreme data based on distribution of data.
#check is done by each UTC month
#check is only done for measurements resolutions of 1 hour or finer, and when have >= 108 valid values (i.e. not NaN) and not decreed invalid by qa flags, in the monthly window (~15% of hourly values in a month), to ensure have sufficient values to get extreme values
#measurements that start in one group and end in the next are treated as belonging to the first group consistently.

#1. apply adjusted boxplot to determine outliers, robustly for normal and non-normal distributions.

#Technique designed to identify outliers based on an adapted version of the classical Tukey boxplot.
#Tukey's boxplot is very popular tool for detection of outliers. It reveals the location, spread and skewness of the data.
#It works nicely for detection of outliers when the data are symmetric.
#When the data are skewed it covers boundary away from the whisker on the compressed side while declares erroneous outliers on the extended side of the distribution.
#Hubert and Vandervieren (2008) made adjustment to Tukey's technique to overcome this problem.
#They apply a robust measure of skewness to Tukey's technique: the 'Medcouple', incorporated as a power of exponential times some constant on left and right as -3.5 and 4 changing position depending upon sign of medcouple. Incorporating this function condenses the interval from narrow side and extends the interval towards the puffy tail.
#However another problem arises that is the adjusted boxplot constructs the interval of critical values which even exceeds from the extremes of the data.
#In this situation the adjusted boxplot is unable to detect outliers.
#Adil and Irshad (2015) provide a solution for this:

#Where the lower and upper critical limits are defined as:
#[Lf Uf] = [Q1 - 1.5*IQR*e-SK*|MC|, Q3 + 1.5*IQR*eSK*|MC|]
#where, Q1 = the 25th percentile, Q3 = the 75th percentile, IQR = the interquartile range, SK = the classical skewness, MC = the medcouple

#**A restriction is also imposed here that if the classical skewness, SK, is greater than 3.5 then it should be treated as 3.5.
#The reason to fix maximum level of skewness 3.5 is to avoid the problem of constructing the large interval of critical values due classical skewness that might be higher than 3.5.
#Not allowing the skewness statistic to exceed 3.5 synchronize the interval of critical value with the data sets as against the adjusted box plot and prevents the interval to be very large in case of highly skewed distributions.
#It also constructs smaller interval in case of moderately skewed distributions
#The 1.5 factor is found to be too conservative for air quality data, and is multiplied by 7 to give a factor of 10.5, which is found empirically to be much more appropriate. 

#See References here:
#http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.882.3197&rep=rep1&type=pdf
#https://en.wikipedia.org/wiki/Medcouple
#https://en.wikipedia.org/wiki/Box_plot#Variations

#2. Take data flagged as outlier by the adjusted boxplot

def check_extreme_distributional_outlier(data,hour_monthly_inds):
    
    #get relevant qa flag
    qa_flag = standardisation_modules.get_standard_qa_flag('Extreme Data - Distributional Outlier')
    
    #iterate through grouped data 
    for xx, grouped_inds in enumerate(hour_monthly_inds):
        grouped_inds = np.array(grouped_inds)
        
        #get relevant data
        valid_monthly_data = data[parameter_details['bsc_parameter_name']][grouped_inds]
        
        #remove invalid data (decreed from qa flags)
        invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Negative Measurement', 'Zero Measurement', 'No Key Metadata', 'No Altitude Metadata', 'Extreme Data - Scientifically Non-Feasible', 'Extreme Data - Manually Decreed', 'Invalid Data Provider Flags - Scientifically Decreed', 'Recurring Value', 'Insufficient Measurement Resolution - Reported', 'Below Documented Lower Limit of Detection', 'Below Reported Lower Limit of Detection', 'Above Documented Upper Limit of Detection', 'Above Reported Upper Limit of Detection', 'Methodology Not Mapped', 'Erroneous Primary Sampling', 'Erroneous Sample Preparation', 'Erroneous Measurement Methodology', 'Invalid QA Measurement Methodology']
        invalid_qa_flags = [standardisation_modules.get_standard_qa_flag(invalid_flag_name) for invalid_flag_name in invalid_qa_flag_names]
        valids = np.array([all(x not in qa_str for x in invalid_qa_flags) for qa_str in data['qa'][grouped_inds]])
        valid_monthly_data = valid_monthly_data[valids]
        valid_monthly_inds = grouped_inds[valids]
        
        #check number of total measurements reported in the annual/month window
        #if < 108 values (~15% of hourly values in a month - with an average 30 days), continue to next month
        if len(valid_monthly_data) < 108:
            continue
    
        #otherwise, check for distributional extreme values
        else:    

            #caluclate medcouple
            mc = sm.medcouple(valid_monthly_data)
            #calculate absolute value of medcouple
            mc_abs = np.abs(mc)

            #caluclate the 25th percentile (p1)
            p25 = np.percentile(valid_monthly_data, 25)
            #caluclate the 75th percentile (p1)
            p75 = np.percentile(valid_monthly_data, 75)

            #calculate the interquartile range
            iqr = p75-p25

            #caluclate classical skewness
            skew = st.skew(valid_monthly_data)
            #limit skewness to to bet between -3.5 and 3.5
            if (skew > 3.5):
                skew = 3.5
            elif (skew < -3.5):
                skew = -3.5

            #calculate upper critical limit
            lower_critical = p25 - 10.5 * iqr * np.exp(-skew*mc_abs)
            upper_critical = p75 + 10.5 * iqr * np.exp(skew*mc_abs)

            #check which data is below adjusted boxplot upper limit and above lower limit
            invs = np.where(np.logical_or(valid_monthly_data<lower_critical, valid_monthly_data>upper_critical))[0]
            invalid_inds = valid_monthly_inds[invs]
            
            #write out invalid indices
            if len(invalid_inds) > 0:
                
                for inv in invalid_inds:
                    data['qa'][inv] = data['qa'][inv] + qa_flag
            
    return data
                    

###-----------------------------------------------------------------------------------###

#check if measurement resolution is coarser than a set limit (variable by parameter measured). This is checked in 2 ways: by reported metadata, and empirically. Handled separately by 2 different modules.
#this module is the empirical check
#empirically determine measurement resolution of measurements in grouped monthly windows (UTC time). 
#if the minimum difference between observations in the monthly window is coarser than a set limit (variable by measured parameter), flag the measurements
#check accounts for the improvement of measurement resolution over time (as happens with improved methods and instrumentation over time)
#***check is only done for measurements resolutions of 1 hour or finer, and when have >= 108 valid values (i.e. not NaN) and not decreed invalid by qa flags, in the monthly window (~15% of hourly values in a month), to ensure have sufficient values to fairly return a true minimum difference value
#measurements that start in one group and end in the next are treated as belonging to the first group consistently.

def check_measurement_resolution_empirical(data,hour_monthly_inds):

    #get relevant qa flag
    qa_flag = standardisation_modules.get_standard_qa_flag('Insufficient Measurement Resolution - Empirical')
    
    #iterate through grouped data and determine minimum difference between observations
    for xx, grouped_inds in enumerate(hour_monthly_inds):
        grouped_inds = np.array(grouped_inds)
        
        #if have valid data proceed (i.e. have data with sufficient temporal resolution) 
        if len(grouped_inds) > 0:
                
            #get relevant data
            valid_monthly_data = data[parameter_details['bsc_parameter_name']][grouped_inds]
            
            #remove invalid data (decreed from qa flags)
            invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Negative Measurement', 'Zero Measurement', 'No Key Metadata', 'No Altitude Metadata', 'Extreme Data - Scientifically Non-Feasible', 'Extreme Data - Distributional Outlier', 'Extreme Data - Manually Decreed', 'Invalid Data Provider Flags - Scientifically Decreed', 'Recurring Value', 'Insufficient Measurement Resolution - Reported', 'Below Documented Lower Limit of Detection', 'Below Reported Lower Limit of Detection', 'Above Documented Upper Limit of Detection', 'Above Reported Upper Limit of Detection', 'Methodology Not Mapped', 'Erroneous Primary Sampling', 'Erroneous Sample Preparation', 'Erroneous Measurement Methodology', 'Invalid QA Measurement Methodology']
            invalid_qa_flags = [standardisation_modules.get_standard_qa_flag(invalid_flag_name) for invalid_flag_name in invalid_qa_flag_names]
            valids = np.array([all(x not in qa_str for x in invalid_qa_flags) for qa_str in data['qa'][grouped_inds]])
            valid_monthly_data = valid_monthly_data[valids]
            
            #check number of total measurements reported in the annual/month window
            #if < 108 values (~15% of hourly values in a month - with an average 30 days)
            if len(valid_monthly_data) < 108:
                continue
        
            #otherwise, check minimum difference between observations
            else:
            
                #initiate minimum difference
                minimum_difference = 9999999999
        
                #remove duplicate values from array
                valid_monthly_data = list(set(valid_monthly_data))
    
                #sort array in ascending order
                valid_monthly_data = sorted(valid_monthly_data)                    
 
                #Find the min diff by comparing adjacent pairs in sorted array
                for ii in range(len(valid_monthly_data)-1):
                    #is difference smaller than current minimum difference
                    if (valid_monthly_data[ii+1] - valid_monthly_data[ii]) < minimum_difference:
                        #if so set new minimum difference
                        minimum_difference = valid_monthly_data[ii+1] - valid_monthly_data[ii]

                #if minimum difference between all points for the monthly window is coarser (or equal to) than the set minimum resolution for the parameter being processed, flag the measurements.
                if minimum_difference >= parameter_details['minimum_resolution']:
                    for inv in grouped_inds:
                        data['qa'][inv] = data['qa'][inv] + qa_flag
    
    return data
    
###------------------------------------------------------------------------------------###

#module for determining if distribution of measurements per month are significantly biased through hitting of LOD (>=25%).
#measurements hitting the LOD are determined through presence of LOD QA flags and negative/zero/infinite QA flags
#valid measurements each UTC month are compared with the number of LOD hitting associated QA flagged measurements, if >= 25% then whole monthly window is flagged.
#this check is limited by the availability of LOD information

#check is only done for measurements resolutions of 1 hour or finer, and when have >= 108 valid values (i.e. not NaN) and not decreed invalid by qa flags, in the monthly window (~15% of hourly values in a month), to ensure have sufficient values to test
#measurements that start in one group and end in the next are treated as belonging to the first group consistently.

def check_hitting_lod(data,hour_monthly_inds):

    #get relevant qa flag
    qa_flag = standardisation_modules.get_standard_qa_flag('Hitting LOD')
    
    #iterate through grouped data 
    for xx, grouped_inds in enumerate(hour_monthly_inds):
        grouped_inds = np.array(grouped_inds)
        
        #if have valid data proceed (i.e. have data with sufficient temporal resolution) 
        if len(grouped_inds) > 0:
                
            #get relevant data
            valid_monthly_data = data[parameter_details['bsc_parameter_name']][grouped_inds]
            
            #remove invalid data (decreed from qa flags)
            invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Negative Measurement', 'Zero Measurement', 'No Key Metadata', 'No Altitude Metadata', 'Extreme Data - Scientifically Non-Feasible', 'Extreme Data - Distributional Outlier', 'Extreme Data - Manually Decreed', 'Invalid Data Provider Flags - Scientifically Decreed', 'Recurring Value', 'Insufficient Measurement Resolution - Reported', 'Insufficient Measurement Resolution - Empirical', 'Below Documented Lower Limit of Detection', 'Below Reported Lower Limit of Detection', 'Above Documented Upper Limit of Detection', 'Above Reported Upper Limit of Detection', 'Methodology Not Mapped', 'Erroneous Primary Sampling', 'Erroneous Sample Preparation', 'Erroneous Measurement Methodology', 'Invalid QA Measurement Methodology']
            invalid_qa_flags = [standardisation_modules.get_standard_qa_flag(invalid_flag_name) for invalid_flag_name in invalid_qa_flag_names]
            valids = np.array([all(x not in qa_str for x in invalid_qa_flags) for qa_str in data['qa'][grouped_inds]])
            valid_monthly_data = valid_monthly_data[valids]
            
            #check number of total measurements reported in the annual/month window
            #if < 108 values (~15% of hourly values in a month - with an average 30 days), continue to next month
            if len(valid_monthly_data) < 108:
                continue
        
            #otherwise, proceed with check
            else:    
                n_valid_in_month = len(valid_monthly_data)
                
                #get number of measurements in month which are hitting lod
                #this is determined from presence of any of LOD QA flags and negative/zero/infinite QA flags
                hitting_lod_flag_names = ['Infinite Value', 'Below Documented Lower Limit of Detection', 'Below Reported Lower Limit of Detection', 'Above Documented Upper Limit of Detection', 'Above Reported Upper Limit of Detection', 'Negative Measurement', 'Zero Measurement']
                hitting_lod_flags = [standardisation_modules.get_standard_qa_flag(flag_name) for flag_name in hitting_lod_flag_names]
                n_hitting_lod_in_month = [any(x in qa_str for x in hitting_lod_flags) for qa_str in data['qa'][grouped_inds]].count(True)
                
                #get ratio of n valid measurements in month to those hitting LOD
                n_hitting_lod_ratio = (100.0/n_valid_in_month)*n_hitting_lod_in_month
                
                #if percentage hitting LOD is >=25%, then write flag for the entire monthly period
                if n_hitting_lod_ratio >= 25:
                    for inv in grouped_inds:
                        data['qa'][inv] = data['qa'][inv] + qa_flag
                    
    return data


###------------------------------------------------------------------------------------###
#module that groups time by year/month (UTC) for passing to multiple qa modules.
#also handles filtering of times which do not match up the required temporal resolutions required for the specific modules
def iterate_by_yearmonth(metadata,data):

    hour_monthly_inds = []
    day_monthly_inds = []
    daytohour_inds = []
    current_year = -9999
    current_month = -99
    for ii,start_time in enumerate(data['time']):
        #append values for 3 separate temporal resolution instances, for temporal resolutions of 1 hour or finer (hour), for temporal resolutions of finer than 1 day (day), and for resolutions as coarse as 1 hour and less coarse than 1 day (daytohour)
        #the first two are grouped by year/month (UTC), and the last is appended all together
        hour_resolution = False
        day_resolution = False
        daytohour_resolution = False
        
        temporal_resolution = data['temporal_resolution'][ii]
        
        try:
            temporal_resolution = int(temporal_resolution)
            if temporal_resolution < (60*24):
                day_resolution = True
            if (temporal_resolution >= 60) or (temporal_resolution < (60*24)):
                daytohour_resolution = True
            if temporal_resolution <= (60):
                hour_resolution = True
        
        except ValueError:
            if temporal_resolution == 'H':
                hour_resolution = True
                day_resolution = True
                daytohour_resolution = True
                
        if (hour_resolution == True) or (day_resolution == True) or (daytohour_resolution == True):
            #get datetime
            current_dt = crossnetwork_processing_modules.fractional_minutes_from_reference_to_datetime(float(start_time))
            #is datetime in a new year/monthly window?
            if (current_dt.year != current_year)  or (current_dt.month != current_month):
                if hour_resolution == True:
                    hour_monthly_inds.append([ii])
                if day_resolution == True:
                    day_monthly_inds.append([ii])
                if daytohour_resolution == True:
                    daytohour_inds.append(ii)
                current_year = current_dt.year
                current_month = current_dt.month
            else:
                if hour_resolution == True: 
                    hour_monthly_inds[-1].append(ii)
                if day_resolution == True:
                    day_monthly_inds[-1].append(ii)
                if daytohour_resolution == True:
                    daytohour_inds.append(ii)
                    
    #pass grouped year/month indices to qa modules for processing
    
    #first convert daytohour_inds to numpy array
    daytohour_inds = np.array(daytohour_inds, dtype=np.uint32)
    
    #non-integer local timezone check
    data = check_non_integer_local_timezone(metadata,data,day_monthly_inds,daytohour_inds)
    
    #extreme data - distributional outlier check
    data = check_extreme_distributional_outlier(data,hour_monthly_inds)
    
    #measurement resolution check
    data = check_measurement_resolution_empirical(data,hour_monthly_inds)
    
    #hitting LOD check
    data = check_hitting_lod(data,hour_monthly_inds)
    
    return data
        
    
###------------------------------------------------------------------------------------###

#module for temporal representation check of data
#scientifically invalid flagged data is first screened out. 
#representation is checked for 4 temporal periods: diurnal, weekly, monthly, annual
#the starts and ends of the temporal periods are all in UTC time.  
#temporal representation across the different periods is defined as a minimum of 50% of the window represented by valid observations, and a maximum data gap of 15% across the total period length.
#if measurements are determined to be unrepresentative for a period, all measurements across the period are flagged with a specific qa flag 

def check_representation(n_minutes_represented, all_gaps, period_code, qa_data, relevant_inds, current_dt):

    #set minimum percentage limit of period represented to be classed as 'representative'
    #i.e. must have >= than this % represented in window 
    minimum_period_percentage = 50 #50%
    
    #set maximum percentage limit of data gaps allowed within period to be classed as 'representative'
    #i.e. must have max data gap of < than this % in window 
    maximum_data_gap_percentage = 15 #15%
    
    #get maximum gap in all_gaps array (if a gap exists)
    if len(all_gaps) == 0:
        max_temporal_gap = 0
    else:   
        max_temporal_gap = np.max(all_gaps)
    
    #get maximum possible minutes by period
    if period_code == 'H':
        max_minutes_possible = 60
        qa_string = 'Not Hourly Representative'
    elif period_code == 'D':
        max_minutes_possible = 60*24
        qa_string = 'Not Diurnally Representative'
    elif period_code == 'W':
        max_minutes_possible = (60*24)*7
        qa_string = 'Not Weekly Representative'
    elif period_code == 'M':
        current_year = current_dt.year
        current_month = current_dt.month
        days_in_month = monthrange(current_year, current_month)[1]
        max_minutes_possible = (60*24)*days_in_month
        qa_string = 'Not Monthly Representative'
    elif period_code == 'A':
        current_year = current_dt.year
        d1 = datetime.date(current_year, 1, 1) 
        d2 = datetime.date(current_year + 1, 1, 1) 
        days_in_year = (d2 - d1).days 
        max_minutes_possible = (60*24)*days_in_year
        qa_string = 'Not Annually Representative'

    #100./max_minutes_possible (for multiplying by)
    percent_max_minutes = 100./max_minutes_possible
        
    #determine if relevant period is representative or not (set default flag to True)
    representive_period = True
    
    #does it have < minimum permitted percentage of period represented?    
    if (percent_max_minutes*n_minutes_represented) < minimum_period_percentage:
        representive_period = False
    
    #does a data gap exist that is > maximum permitted?
    elif (percent_max_minutes*max_temporal_gap) > maximum_data_gap_percentage:
        representive_period = False
    
    #if period not representative, write flag for all relevant indices contained within period
    if representive_period == False:
        #get relevant qa flag
        qa_flag = standardisation_modules.get_standard_qa_flag(qa_string)
    
        for inv in relevant_inds:
            qa_data[inv] = qa_data[inv] + qa_flag
            
    return qa_data
    
###------------------------------------------------------------------------------------###