import bisect
from calendar import monthrange
from collections import Counter
import copy
import datetime
import time

from dateutil.relativedelta import relativedelta
from netCDF4 import Dataset
import numpy as np
import numpy.lib.recfunctions as rf
import pandas as pd

from crossnetwork_processing_functions import set_file_permissions_ownership, create_GHOST_stage_logfile, get_data_from_reference, cut_data_array, weighted_mode, get_flag_simple
from GHOST_standards import get_standard_data, get_standard_metadata, standard_temporal_resolutions, standard_data_flag_name_to_data_flag_code, standard_QA_name_to_QA_code, providentia_defaults
from time_functions import datetime_to_fractional_minutes_from_reference, fractional_minutes_from_reference_to_datetime, create_local_time, create_solar_time


class StationTemporalAverage:

    '''Class to temporally average station data to a defined temporal resolution.'''

    def __init__(self, station_reference, temporal_resolution_code, region, run_details, parameter_details, network_details):

        #add passed variables to self
        self.station_reference = station_reference
        self.temporal_resolution_code = temporal_resolution_code
        self.region = region
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += '{} -- TEMPORALLY AVERAGE DATA\n'.format(self.station_reference)


    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #temporally average data for station in defined temporal resolution
        self.temporally_average_data()
        
        #proceed to write out to station specific netCDF file
        if self.data_to_write:

            #drop duplicates, sort and pad flag/qa code arrays
            self.padded_qa = self.drop_duplicates_sort_and_pad_codes('qa', self.gridded_data['qa'])
            self.padded_flag = self.drop_duplicates_sort_and_pad_codes('flag', self.gridded_data['flag'])

            #drop qa/flag from gridded_data
            self.gridded_data = rf.drop_fields(self.gridded_data, ['qa','flag'])

            self.run_details['log_file_str'] += 'WRITING TO MONTHLY STATION SPECIFIC TEMPORALLY GRIDDED netCDFs\n'

            #create netCDF file for station
            self.write_station_specific_netCDF_by_month()

            #return valid process logfile (0)
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)

        #otherwise no valid data has been found
        #return error logfile to inform station no longer continues in pipeline (1)
        else:
            self.run_details['log_file_str'] += 'NO VALID DATA FOUND AFTER QA FLAG CHECKS'  
            create_GHOST_stage_logfile(1,self.run_details)


    def temporally_average_data(self):

        '''function for converting native mixed temporal resolution data onto a grid of times of fixed intervals, between a start and end time in UTC time
           data with a resolution coarser than the temporal resolution desired is screened out.
           when converting from higher temporal resolution to lower temporal resolution, some processing is done:
           1. invalid data is first screened out using set of key qa flags (defined in "invalid_qa_flag_names").
           2. numerical data values are averaged
           3. certain qa flags which are no longer relevant are dropped (this includes qa flags used to screen out measurements in step 1, and other flags which make no sense for a temporal resolution different from which they were originally associated with)
           4. other defined qa flags are kept if they appear more often than not over period (i.e. modally)
           5. measurement flags are dropped as are no longer relevant
        '''

        #get output temporal resolution specific variables
        self.temporal_resolution_vars = standard_temporal_resolutions[self.temporal_resolution_code]

        #set network directory where to read/write data --> add to run details
        if self.region == 'GLOBAL':
            self.run_details['network_write_dir'] = self.run_details['network']
        else:
            self.run_details['network_write_dir'] = '{}_{}'.format(self.run_details['network'], self.region)

        #create list of all minutes in temporal window (representing start of temporal window), with spacing in defined temporal resolution between start_date and end_date 
        #for temporal resolutions which are instantaneous the start of the temporal window represents the start of the period around each temporal point in the grid where observations are considered
        if self.temporal_resolution_code in ['HI','3HI','6HI']:
            #for AERONET instantaneous temporal resolutions, observations are considered in the 60 minutes around each temporal point in the grid  
            if self.run_details['network'] in ['AERONET_v3_lev1.5','AERONET_v3_lev2.0']:
                instantaneous_range = 60
            #otherwise, observations are considered in the 30 minutes around each temporal point in the grid  
            else: 
                instantaneous_range = 30
            start_dt = datetime.datetime(int(self.run_details['start_year']),int(self.run_details['start_month']),1,0,0) - datetime.timedelta(minutes=instantaneous_range/2)
            end_dt = datetime.datetime(int(self.run_details['end_year']),int(self.run_details['end_month']),1,0,0) - datetime.timedelta(minutes=instantaneous_range/2)
            all_dt = pd.date_range(start_dt, end_dt, freq=self.temporal_resolution_vars['temporal_resolution_datetime_frequency']).to_pydatetime()
            grid_start_minutes = [datetime_to_fractional_minutes_from_reference(dt) for dt in all_dt]
            orig_start_dt = datetime.datetime(int(self.run_details['start_year']),int(self.run_details['start_month']),1,0,0)
            orig_end_dt = datetime.datetime(int(self.run_details['end_year']),int(self.run_details['end_month']),1,0,0)
            orig_all_dt = pd.date_range(orig_start_dt, orig_end_dt, freq=self.temporal_resolution_vars['temporal_resolution_datetime_frequency']).to_pydatetime()
            orig_grid_start_minutes = [datetime_to_fractional_minutes_from_reference(dt) for dt in orig_all_dt] 
        else:
            start_dt = datetime.datetime(int(self.run_details['start_year']),int(self.run_details['start_month']),1,0,0)
            end_dt = datetime.datetime(int(self.run_details['end_year']),int(self.run_details['end_month']),1,0,0) 
            all_dt = pd.date_range(start_dt, end_dt, freq=self.temporal_resolution_vars['temporal_resolution_datetime_frequency']).to_pydatetime()
            grid_start_minutes = [datetime_to_fractional_minutes_from_reference(dt) for dt in all_dt] 
        #get start/end stamp of defined processing window in minutes from reference time
        defined_start_stamp = datetime_to_fractional_minutes_from_reference(start_dt)
        defined_end_stamp = datetime_to_fractional_minutes_from_reference(end_dt)
    
        #flags for not counting as native minutes when only 1 measurement
        nonnative_qa_flag_names = ['Missing Measurement', 'Infinite Value']
        nonnative_qa_flag_codes = [standard_QA_name_to_QA_code[flag_name] for flag_name in nonnative_qa_flag_names]

        #flags for removing invalid data when averaging
        invalid_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Invalid Data Provider Flags - GHOST Decreed', 'Insufficient Data to Calculate Data Product', 'Below Preferential Lower Limit of Detection', 'Above Preferential Upper Limit of Detection', 'Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit', 'Data Outlier - Network Decreed', 'Data Outlier - Manually Decreed']
        invalid_qa_flag_codes = [standard_QA_name_to_QA_code[flag_name] for flag_name in invalid_qa_flag_names]

        #nonsensical qa flags when concatenated/modal value is taken, therefore are dropped 
        nonsensical_qa_flag_names = ['Missing Measurement', 'Infinite Value', 'Invalid Data Provider Flags - GHOST Decreed', 'Insufficient Data to Calculate Data Product', 'Below Preferential Lower Limit of Detection', 'Above Preferential Upper Limit of Detection', 'Data Outlier - Exceeds Scientifically Decreed Lower/Upper Limit', 'Data Outlier - Network Decreed', 'Data Outlier - Manually Decreed']
        nonsensical_qa_flag_codes = [standard_QA_name_to_QA_code[flag_name] for flag_name in nonsensical_qa_flag_names]

        #qa flags to take modal value of across averaging period to write out (i.e. if flag appears more than not over period then write it out)
        modal_qa_flag_names = ['Negative Measurement', 'Zero Measurement', 'Not Maximum Data Quality Level', 'Preliminary Data', 'Invalid Data Provider Flags - Network Decreed', 'Methodology Not Mapped', 'Assumed Primary Sampling', 'Assumed Sample Preparation', 'Assumed Measurement Methodology', 'Unknown Primary Sampling Type', 'Unknown Primary Sampling Instrument', 'Unknown Sample Preparation Type', 'Unknown Sample Preparation Technique', 'Unknown Measurement Method', 'Unknown Measuring Instrument', 'Erroneous Primary Sampling', 'Erroneous Sample Preparation', 'Erroneous Measurement Methodology', 'Invalid QA Measurement Methodology', 'Corrected Parameter', 'Sample Gas Volume - Network Standard', 'Sample Gas Volume - Unknown', 'Unit Conversion - Network Standard Sample Gas Volume Assumption', 'Unit Conversion - Educated Guess Sample Gas Volume Assumption', 'Station Position Doubt - DEM Decreed', 'Station Position Doubt - Manually Decreed', 'Data Product', 'Local Precipitation', 'Local Extreme Weather', 'Local Atmospheric Obscuration', 'Local Contamination', 'Local Exceptional Event', 'Non-Integer Local Timezone (relative to UTC)', 'Timezone Doubt', 'Below Documented Lower Limit of Detection', 'Below Reported Lower Limit of Detection', 'Above Documented Upper Limit of Detection', 'Above Reported Upper Limit of Detection', 'Insufficient Measurement Resolution - Documented', 'Insufficient Measurement Resolution - Reported', 'Insufficient Measurement Resolution - Preferential', 'Insufficient Measurement Resolution - Empirical', 'Persistent Recurring Values - 5/6', 'Persistent Recurring Values - 9/12', 'Persistent Recurring Values - 16/24', 'Monthly Fractional Unique Values <= 1%', 'Monthly Fractional Unique Values <= 5%', 'Monthly Fractional Unique Values <= 10%', 'Monthly Fractional Unique Values <= 30%', 'Monthly Fractional Unique Values <= 50%', 'Monthly Fractional Unique Values <= 70%', 'Monthly Fractional Unique Values <= 90%', 'Data Outlier - Monthly Median Exceeds Scientifically Decreed Upper Limit', 'Possible Data Outlier - Monthly Adjusted Boxplot', 'Probable Data Outlier - Monthly Adjusted Boxplot', 'Monthly Distribution Consistency - Zone 1', 'Monthly Distribution Consistency - Zone 2', 'Monthly Distribution Consistency - Zone 3', 'Monthly Distribution Consistency - Zone 4', 'Monthly Distribution Consistency - Zone 5', 'Monthly Distribution Consistency - Zone 6', 'Monthly Distribution Consistency - Zone 7', 'Monthly Distribution Consistency - Zone 8', 'Monthly Distribution Consistency - Zone 9', 'Monthly Distribution Consistency - Zone 10', 'Monthly Distribution Consistency - Unclassified', 'Systematic Inconsistent Monthly Distributions - 2/3 Months >= Zone 6', 'Systematic Inconsistent Monthly Distributions - 4/6 Months >= Zone 6', 'Systematic Inconsistent Monthly Distributions - 8/12 Months >= Zone 6']
        modal_qa_flag_codes = [standard_QA_name_to_QA_code[flag_name] for flag_name in modal_qa_flag_names]

        #get metadata/data for reference
        self.metadata, data, station_numpy_filename = get_data_from_reference(self.station_reference,self.run_details,self.parameter_details,read='all')

        #get start and end stamps of read station data
        self.start_stamp = data['time'][0]
        end_time = data['time'][-1]
        end_res = data['temporal_resolution'][-1]
        self.end_stamp = end_time + end_res

        #if start stamp >= defined end stamp OR end stamp <= defined start_stamp, no valid data in range, therefore do not process further
        if (self.start_stamp >= defined_end_stamp) or (self.end_stamp <= defined_start_stamp):

            #return logfile to inform station no longer continues in pipeline (1)
            self.run_details['log_file_str'] += 'NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'  
            create_GHOST_stage_logfile(1,self.run_details)

        #cut station metadata/data for relevant period
        self.metadata = cut_data_array(self.metadata, 'metadata', defined_start_stamp, defined_end_stamp)
        data =          cut_data_array(data, 'data', defined_start_stamp, defined_end_stamp)
    
        #if have no time data after cutting then no valid data in range, therefore do not process further
        if len(data['time']) == 0:
            #return logfile to inform station no longer continues in pipeline (1)
            self.run_details['log_file_str'] += 'NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'  
            create_GHOST_stage_logfile(1,self.run_details)

        #re-get start and end stamps of read data (after cutting)
        self.start_stamp = data['time'][0]
        end_time = data['time'][-1]
        end_res = data['temporal_resolution'][-1]
        self.end_stamp = end_time + end_res

        #if start of first gridded time period is after end of station data range, no valid data in range, therefore do not process further
        if grid_start_minutes[0] >= data['time'][-1]:
            #return logfile to inform station no longer continues in pipeline (1)
            self.run_details['log_file_str'] += 'NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'  
            create_GHOST_stage_logfile(1,self.run_details)
        #if end of last gridded time period is before start of station data range, no valid data in range, therefore do not process further
        if grid_start_minutes[-1] <= data['time'][0]:
            #return logfile to inform station no longer continues in pipeline (1)
            self.run_details['log_file_str'] += 'NO VALID DATA IN DESIRED RANGE TO TEMPORALLY GRID'  
            create_GHOST_stage_logfile(1,self.run_details)

        #define relevant periods whose representativity will be evaluated in the data, different by temporal resolution code
        if self.temporal_resolution_code == 'M':
            relevant_periods = ['monthly','annual']
        elif self.temporal_resolution_code in ['3H','3HI','6H','6HI','D']:
            relevant_periods = ['daily','monthly','annual']
        elif self.temporal_resolution_code in ['H','HI']:
            relevant_periods = ['hourly','daily','monthly','annual']

        #get data keys
        data_keys = list(data.dtype.names)
        #remove temporal resolution key from list of keys
        data_keys.remove('temporal_resolution')

        #create empty numpy dataframe for storing gridded data
        standard_dtype = [(key,data[key].dtype) for key in data_keys]
        #append variables for temporal representativity to dataframe (np.uint8 type)
        for period in relevant_periods:
            standard_dtype.append(('{}_native_representativity_percent'.format(period), np.uint8))
            standard_dtype.append(('{}_native_max_gap_percent'.format(period), np.uint8))
        self.gridded_data = np.empty(len(grid_start_minutes[:-1]),dtype=standard_dtype)

        #iterate through data_keys to instantiate new numpy dataframe with data
        for kk, key in enumerate(data_keys):

            #add gridded time to dataframe
            if key == 'time':   
                if self.temporal_resolution_code in ['HI','3HI','6HI']:
                    self.gridded_data['time'] = orig_grid_start_minutes[:-1]
                else:
                    self.gridded_data['time'] = grid_start_minutes[:-1]

            #add array of NaNs for np.float32 types
            elif self.gridded_data[key].dtype == np.dtype('float32'):
                self.gridded_data[key] = np.array([np.NaN]*len(grid_start_minutes[:-1]))

            #add array of 255's for np.uint8 types
            elif self.gridded_data[key].dtype == np.dtype('uint8'):
                self.gridded_data[key] = np.array([255]*len(grid_start_minutes[:-1]))

            #make list of lists of qa codes - all initialised as empty
            elif key == 'qa':
                self.gridded_data[key] = [[] for i in range(len(grid_start_minutes[:-1]))]

            #make list of list of data flag codes - with a default of Un-flagged data code
            elif key == 'flag':
                valid_data_flag_code = standard_data_flag_name_to_data_flag_code['Valid Data']
                unflagged_data_flag_code = standard_data_flag_name_to_data_flag_code['Un-Flagged Data']
                self.gridded_data[key] = [[unflagged_data_flag_code] for i in range(len(grid_start_minutes[:-1]))]

        #get valid start and end times
        #screen out data that has coarser resolution than desired output temporal resolution
        valid_start_times = []
        valid_end_times = []
        valid_timedeltas = []
        valid_inds = []
        for ii, start_time in enumerate(data['time']):

            process_flag = False

            #get window end time by adding temporal resolution as timedelta
            timedelta = data['temporal_resolution'][ii]
            if self.temporal_resolution_code in ['HI','3HI','6HI']:
                if timedelta == 1:
                    process_flag = True
            elif self.temporal_resolution_code == 'H':
                if timedelta <= 60:
                    process_flag = True
            elif self.temporal_resolution_code == '3H':
                if timedelta <= (3*60):
                    process_flag = True
            elif self.temporal_resolution_code == '6H':
                if timedelta <= (6*60):
                    process_flag = True
            elif self.temporal_resolution_code == 'D':
                if timedelta <= (24*60):
                    process_flag = True
            elif self.temporal_resolution_code == 'M':
                if timedelta <= (31*24*60):
                    process_flag = True

            #get measurement window end time
            if process_flag:
                valid_start_times.append(start_time)
                valid_end_times.append(start_time + timedelta)
                if timedelta == 0:
                    valid_timedeltas.append(1)
                else:
                    valid_timedeltas.append(timedelta)
                valid_inds.append(ii)

        #check have some valid data after removing coarse resolution data
        if len(valid_inds) == 0:
            #return logfile to inform station no longer continues in pipeline (1)
            self.run_details['log_file_str'] += 'AFTER REMOVING COARSE RESOLUTION DATA, NO VALID DATA TO TEMPORALLY GRID'  
            create_GHOST_stage_logfile(1,self.run_details)
        #next check have some data reported between start_date and end_date - after removing coarse resolutions
        #if start of first gridded time period is after end of station data range, no valid data in range, therefore do not process further
        elif grid_start_minutes[0] >= valid_end_times[-1]:
            #return logfile to inform station no longer continues in pipeline (1)
            self.run_details['log_file_str'] += 'AFTER REMOVING COARSE RESOLUTION DATA, NO VALID DATA TO TEMPORALLY GRID'  
            create_GHOST_stage_logfile(1,self.run_details)
        #if end of last gridded time period is before start of station data range, no valid data in range, therefore do not process further
        elif grid_start_minutes[-1] <= valid_start_times[0]:
            #return logfile to inform station no longer continues in pipeline (1)
            self.run_details['log_file_str'] += 'AFTER REMOVING COARSE RESOLUTION DATA, NO VALID DATA TO TEMPORALLY GRID'  
            create_GHOST_stage_logfile(1,self.run_details)

        #convert cut arrays to numpy arrays
        valid_start_times = np.array(valid_start_times, dtype=np.uint32)
        valid_end_times = np.array(valid_end_times, dtype=np.uint32)
        valid_timedeltas = np.array(valid_timedeltas , dtype=np.uint32)
        valid_inds = np.array(valid_inds , dtype=np.uint32)

        #flag that determines if have valid data to write for station netCDF file
        self.data_to_write = False

        #--------------------------------------------------------------------------------#
        #--------------------------------------------------------------------------------#
        #initialise variables for evaluating temporal representation across periods

        #get QA flag for flagging windows which have no measurements provided within them
        missing_measurement_qa_flag_code = standard_QA_name_to_QA_code['Missing Measurement']

        #get QA flag for flagging windows which have no contain no valid data to average (after screening by key QA flags --> those in invalid_qa_flags)
        no_valid_data_qa_flag_code = standard_QA_name_to_QA_code['No Valid Data to Average']

        #initialise dictionary for keeping track of % data represented across different periods, and max % data gaps
        #this is handled slightly different for temporal resolutions which are instantaneous
        period_tracker = {}
        for period in relevant_periods: 
            if self.temporal_resolution_code in ['HI','3HI','6HI']:
                representativity_var_name = 'n_hours_represented'
            else:
                representativity_var_name = 'n_minutes_represented'
            period_tracker[period] = {representativity_var_name:0, 'gaps':[], 'inds':[], 'open_gap':False, 'new_period':False}

       
        #set variable that will hold last written qa codes, for the purpose of filling blank windows (i.e. where have no measurements), to ensure qa flags do not jump wildly.
        #makes assumption qa flags hold for blank windows
        most_recent_qa_codes = []

        #get index of first window which contains measurements
        window_start_index = bisect.bisect(grid_start_minutes, valid_start_times[0])-1
        #get index of last window which contains measurements
        window_end_index = bisect.bisect(grid_start_minutes, valid_end_times[-1])
        #adjust start/end indices to be start/end of UTC year which contains first/last measurements (if start/end index not already at edges of period)
        if window_start_index != 0:
            window_start_index =  bisect.bisect(grid_start_minutes, datetime_to_fractional_minutes_from_reference(datetime.datetime(fractional_minutes_from_reference_to_datetime(float(grid_start_minutes[window_start_index])).year, 1, 1, 0, 0)))-1
        if window_end_index != len(grid_start_minutes):
            window_end_index =  bisect.bisect(grid_start_minutes, datetime_to_fractional_minutes_from_reference(datetime.datetime(fractional_minutes_from_reference_to_datetime(float(grid_start_minutes[window_end_index])).year+1, 1, 1, 0, 0)))
        #don't allow end index be last index in grid of start minutes (this index is reserved to be stopping point of all entire period) 
        if window_end_index == len(grid_start_minutes):
            window_end_index = window_end_index - 1

        #initialise variable for finding relevant measurement indices
        last_relevant_index = 0

        #--------------------------------------------------------------------------------#
        #--------------------------------------------------------------------------------#

        #iterate through gridded time windows
        for ii, window_start_minute in enumerate(grid_start_minutes[window_start_index:window_end_index]):

            #get actual index of window (as have skipped through windows before first index)
            actual_ii = window_start_index+ii
            #if temporal resolution code is instantaneous then modify window end minute to be the window start + considered instantaneous range
            if self.temporal_resolution_code in ['HI','3HI','6HI']:
                window_end_minute = window_start_minute + instantaneous_range
            #otherwise it is simply the next start minute
            else:
                window_end_minute = grid_start_minutes[actual_ii+1]

            #self.run_details['log_file_str'] += '{} {}\n'.format(fractional_minutes_from_reference_to_datetime(float(window_start_minute)), fractional_minutes_from_reference_to_datetime(float(window_end_minute)))

            #update period group indices
            for period in relevant_periods:
                period_tracker[period]['inds'].append(actual_ii)

            #--------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------#

            #update current window start month/day/hour
            current_dt = fractional_minutes_from_reference_to_datetime(float(window_start_minute))
            current_year = current_dt.year
            current_month = current_dt.month
            current_day = current_dt.day
            current_hour = current_dt.hour

            #update next window start month/day/hour
            next_dt = fractional_minutes_from_reference_to_datetime(float(window_end_minute))
            next_year = next_dt.year
            next_month = next_dt.month
            next_day = next_dt.day
            next_hour = next_dt.hour

            #determine if next window start datetime is a new hour/day/month/year

            #is it a new year
            if (next_year != current_year):
                period_tracker['annual']['new_period'] = True
            else:
                period_tracker['annual']['new_period'] = False

            #is it a new month?
            if (next_year != current_year) or (next_month != current_month):
                period_tracker['monthly']['new_period'] = True
            else:
                period_tracker['monthly']['new_period'] = False

            #is it a new day?
            if 'daily' in relevant_periods:
                if (next_year != current_year)  or (next_month != current_month) or (next_day != current_day):
                    period_tracker['daily']['new_period'] = True
                else:
                    period_tracker['daily']['new_period'] = False

            #is it a new hour?
            if 'hourly'in relevant_periods:
                if (next_year != current_year)  or (next_month != current_month) or (next_day != current_day) or (next_hour != current_hour):
                    period_tracker['hourly']['new_period'] = True
                else:
                    period_tracker['hourly']['new_period'] = False

            #--------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------#
            #get relevant measurement indices in current window

            #get index where valid start times >= window start minute
            valid_start_times_begin = bisect.bisect_left(valid_start_times, window_start_minute, lo=last_relevant_index, hi=len(valid_start_times))
            #get index where valid start times < window end minute
            valid_start_times_end = bisect.bisect_left(valid_start_times, window_end_minute, lo=last_relevant_index, hi=len(valid_start_times))
            #get index of first start time < window start minute
            start_time_index_before_window = valid_start_times_begin - 1 

            #get index where valid end times > window start minute
            valid_end_times_begin = bisect.bisect_right(valid_end_times, window_start_minute, lo=last_relevant_index, hi=len(valid_end_times))
            #get index where valid end times <= window end minute
            valid_end_times_end = bisect.bisect_right(valid_end_times, window_end_minute, lo=last_relevant_index, hi=len(valid_end_times))
            #get index of first end time > window end minute
            end_time_index_after_window = copy.deepcopy(valid_end_times_end)
    
            #get indices of all measurement periods which entirely contain window
            if start_time_index_before_window == end_time_index_after_window:
                window_in_measurement_indices = np.array([start_time_index_before_window])  
            else:
                window_in_measurement_indices = np.array([], dtype=np.uint32)
    
            #get indices of measurements entirely contained within window
            start_time_indices_in_window = np.arange(valid_start_times_begin, valid_start_times_end, dtype=np.uint32)
            end_time_indices_in_window = np.arange(valid_end_times_begin, valid_end_times_end, dtype=np.uint32)
            measurement_in_window_indices = np.intersect1d(start_time_indices_in_window,end_time_indices_in_window, assume_unique=True)

            #get indices of measurements which overlap on left/right edges of window
            left_overlap_indices = np.setdiff1d(end_time_indices_in_window, measurement_in_window_indices, assume_unique=True)
            left_overlap_indices = np.setdiff1d(left_overlap_indices, window_in_measurement_indices, assume_unique=True)
            right_overlap_indices = np.setdiff1d(start_time_indices_in_window, measurement_in_window_indices, assume_unique=True)
            right_overlap_indices = np.setdiff1d(right_overlap_indices, window_in_measurement_indices, assume_unique=True)

            #deal with cases where instantaneous measurements start and stop on left/right borders of window
            #in these cases measurements on the left border will be flagged as right overlap indices and vice versa
            #remove these border indices from the overlap indices and define them as their own "border" indices
            #left_border_indices = right_overlap_indices[np.where((valid_start_times[right_overlap_indices] == window_start_minute) & (valid_end_times[right_overlap_indices] == window_start_minute))[0]]
            #right_border_indices = left_overlap_indices[np.where((valid_start_times[left_overlap_indices] == window_end_minute) & (valid_end_times[left_overlap_indices] == window_end_minute))[0]]
            #if len(left_border_indices) > 0:
            #    right_overlap_indices = np.setdiff1d(right_overlap_indices, left_border_indices, assume_unique=True)
            #if len(right_border_indices) > 0:
            #    left_overlap_indices = np.setdiff1d(left_overlap_indices, right_border_indices, assume_unique=True)
    
            #deal with cases where left/right borders align but measurement period entirely contains window
            #add indices of these cases to window_in_measurement_indices
            #remove these indices also from the overlap indices
            left_window_in_measurement_indices = right_overlap_indices[np.where(valid_start_times[right_overlap_indices] == window_start_minute)]
            right_window_in_measurement_indices = left_overlap_indices[np.where(valid_end_times[left_overlap_indices] == window_end_minute)]
            if len(left_window_in_measurement_indices) > 0:
                window_in_measurement_indices = np.concatenate((window_in_measurement_indices, left_window_in_measurement_indices))
                right_overlap_indices = np.setdiff1d(right_overlap_indices, left_window_in_measurement_indices, assume_unique=True)
            if len(right_window_in_measurement_indices) > 0:
                window_in_measurement_indices = np.concatenate((window_in_measurement_indices, right_window_in_measurement_indices))
                left_overlap_indices = np.setdiff1d(left_overlap_indices, right_window_in_measurement_indices, assume_unique=True)

            #if there is a left border overlap, get the number of minutes the measurement period overlaps the measurement window
            if len(left_overlap_indices) > 0:
                left_overlap = valid_end_times[left_overlap_indices[0]] - window_start_minute
            #otherwise left overlap == 0
            else:
                left_overlap = 0
            #if there is a right border overlap, get the number of minutes the measurement period overlaps the measurement window
            if len(right_overlap_indices) > 0:
                right_overlap = window_end_minute - valid_start_times[right_overlap_indices[0]]
            #otherwise right overlap == 0
            else:
                right_overlap = 0

            #concatenate all relevant measurements indices in current window
            window_indices = np.sort(np.concatenate((measurement_in_window_indices,window_in_measurement_indices,left_overlap_indices,right_overlap_indices)))

            #for the case of the border indices (from instantaneous measurements), when averaging only the left border indices are preserved (to avoid counting measurements in 2 separate windows)
            #if self.temporal_resolution_code not in ['HI','3HI','6HI']:
            #window_indices = np.sort(np.concatenate((measurement_in_window_indices,window_in_measurement_indices,left_overlap_indices,right_overlap_indices,left_border_indices)))
            #otherwise, when taking instaneous measurement on the top of the hour then count both border indices
            #else:
            #    window_indices = np.sort(np.concatenate((measurement_in_window_indices,window_in_measurement_indices,left_overlap_indices,right_overlap_indices,left_border_indices,right_border_indices)))
            

            #--------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------#

            #if have 0 window indices, it is because the window is in a void between measurements
            if len(window_indices) == 0:
                native_gap = window_end_minute - window_start_minute
                for period in relevant_periods:
                    if period_tracker[period]['open_gap']:
                        period_tracker[period]['gaps'][-1] = period_tracker[period]['gaps'][-1] + native_gap
                    else:
                        period_tracker[period]['gaps'].append(native_gap)
                        period_tracker[period]['open_gap'] = True   

                for period in relevant_periods:
                    #if new period then calculate representativity of previous period
                    if period_tracker[period]['new_period']: 
                        #if have no data in window set all relevant period windows to have a % data representativity of 0 and max data gap of 100 %
                        if period_tracker[period][representativity_var_name] == 0:
                            self.gridded_data['{}_native_representativity_percent'.format(period)][period_tracker[period]['inds']] = 0
                            self.gridded_data['{}_native_max_gap_percent'.format(period)][period_tracker[period]['inds']] = 100
                        #if have data in window, evaluate data representation across previous period
                        else:
                            rep_pc, max_gap_pc = self.check_representation(period, period_tracker[period], current_dt)
                            self.gridded_data['{}_native_representativity_percent'.format(period)][period_tracker[period]['inds']] = rep_pc
                            self.gridded_data['{}_native_max_gap_percent'.format(period)][period_tracker[period]['inds']] = max_gap_pc
                    #update period variables
                        period_tracker[period][representativity_var_name] = 0
                        period_tracker[period]['gaps'] = []
                        period_tracker[period]['inds'] = []
                        period_tracker[period]['open_gap'] = []

                #append QA flags to state have no measurements in window
                self.gridded_data['qa'][actual_ii].append(missing_measurement_qa_flag_code)

                #copy most recent qa codes to this empty window
                self.gridded_data['qa'][actual_ii].extend(most_recent_qa_codes)

                continue

            #record last index of current window indices for next iteration
            last_relevant_index = window_indices[-1] 

            #if temporal resolution is instantaneous, then limit the window_indices to be just the index closest to the top of the hour (after first removing invalid data)
            if self.temporal_resolution_code in ['HI','3HI','6HI']:
                #have more than 1 relevant index?
                if len(window_indices) > 1:
                    #filter window indices by removing indices associated with invalid data
                    data_respective_window_indices = valid_inds[window_indices]
                    valid_window_indices = window_indices[np.array([all(x not in qa_flag_codes for x in invalid_qa_flag_codes) for qa_flag_codes in data['qa'][data_respective_window_indices]])]
                    #have no indices remaining? --> get index closest to top of the hour from the initial window indices
                    if len(valid_window_indices) == 0:
                        top_of_hour = window_start_minute + (instantaneous_range/2.)
                        closest_index = window_indices[np.abs(top_of_hour - valid_start_times[window_indices]).argmin()]
                        window_indices = np.array([closest_index])
                    #have just 1 index remaining? --> keep this index
                    elif len(valid_window_indices) == 1:
                        window_indices = copy.deepcopy(valid_window_indices)
                    #have > 1 indices remaining? --> get index closest to top of the hour from the filtered window indices
                    elif len(valid_window_indices) > 1:
                        top_of_hour = window_start_minute + (instantaneous_range/2.)
                        closest_index = valid_window_indices[np.abs(top_of_hour - valid_start_times[valid_window_indices]).argmin()]
                        window_indices = np.array([closest_index])

            #print('Window = {}:{}'.format(fractional_minutes_from_reference_to_datetime(float(window_start_minute)), fractional_minutes_from_reference_to_datetime(float(window_end_minute))))
            #for st,et in zip(valid_start_times[window_indices],valid_end_times[window_indices]):
            #    print('{}-{}'.format(fractional_minutes_from_reference_to_datetime(float(st)), fractional_minutes_from_reference_to_datetime(float(et))))
            #    print()
            #--------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------#

            #get indices of measurements within window, respective to full data arrays
            data_respective_window_indices = valid_inds[window_indices]

            #if just have 1 index, get number of minutes represented in window and gaps
            if len(window_indices) == 1:

                #if have 1 index will always write out 
                write_out = True

                relevant_start_time = valid_start_times[window_indices][0]
                relevant_end_time = valid_end_times[window_indices][0]
                #if relevant start/end time exceed start/end of window, modify relevant start/end time to be that of window start/end
                if relevant_start_time < window_start_minute:
                    relevant_start_time = window_start_minute
                if relevant_end_time > window_end_minute:
                    relevant_end_time = window_end_minute

                #determine if data is flagged as missing/infinite or not, if so then do not count minutes represented
                validity = [all(x not in qa_flag_codes for x in nonnative_qa_flag_codes) for qa_flag_codes in data['qa'][data_respective_window_indices]][0]
                if validity == False:
                    window_gap = window_end_minute - window_start_minute
                    for period in relevant_periods:
                        if period_tracker[period]['open_gap']:
                            period_tracker[period]['gaps'][-1] = period_tracker[period]['gaps'][-1] + window_gap
                        else:
                            period_tracker[period]['gaps'].append(window_gap)
                            #set open gap to True
                            period_tracker[period]['open_gap'] = True

                else:
                    #get n minutes/n hours represented
                    n_minutes_represented = relevant_end_time - relevant_start_time
                    #add n_minutes_represented/ n hours represented for all relevant temporal resolutions
                    for period in relevant_periods:
                        if self.temporal_resolution_code == 'HI':
                            period_tracker[period]['n_hours_represented'] = period_tracker[period]['n_hours_represented'] + 1
                        elif self.temporal_resolution_code == '3HI':
                            period_tracker[period]['n_hours_represented'] = period_tracker[period]['n_hours_represented'] + 3
                        elif self.temporal_resolution_code == '6HI':
                            period_tracker[period]['n_hours_represented'] = period_tracker[period]['n_hours_represented'] + 6
                        else:
                            period_tracker[period]['n_minutes_represented'] = period_tracker[period]['n_minutes_represented'] + n_minutes_represented

                    #if temporal resolution is instantaneous, have data in this window therefore open gap is False
                    if self.temporal_resolution_code in ['HI','3HI','6HI']:
                        for period in relevant_periods:
                            #set open gap to False
                            period_tracker[period]['open_gap'] = False
                    else: 
                        #append/process gaps across other temporal resolutions
                        left_edge_gap = relevant_start_time - window_start_minute
                        right_edge_gap = window_end_minute - relevant_end_time
        
                        if left_edge_gap > 0:
                            for period in relevant_periods:
                                if period_tracker[period]['open_gap']:
                                    period_tracker[period]['gaps'][-1] = period_tracker[period]['gaps'][-1] + left_edge_gap
                                else:
                                    period_tracker[period]['gaps'].append(left_edge_gap)
                        #reset open gap flags to be False
                        for period in relevant_periods:
                            period_tracker[period]['open_gap'] = False

                        if right_edge_gap > 0:
                            for period in relevant_periods:
                                period_tracker[period]['gaps'].append(right_edge_gap)
                                #set open gap to True
                                period_tracker[period]['open_gap'] = True

            #--------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------#

            #if are averaging data within window (i.e. more than 1 measurement, screen data to remove invalid data first)
            #also calculate number of minutes represented in window and gaps
            elif len(window_indices) > 1:

                #weight data by timedeltas for averaging different temporal resolution data
                window_weights = valid_timedeltas[window_indices]
                #modify weights on left and right edges to adjust for actual time sampled within window, if measurement overlaps edge
                if left_overlap > 0.0:
                    window_weights[0] = left_overlap
                if right_overlap > 0.0:
                    window_weights[-1] = right_overlap

                #cut arrays solely for valid values (by screening by key qa flags)
                valid_mask = np.array([all(x not in qa_flag_codes for x in invalid_qa_flag_codes) for qa_flag_codes in data['qa'][data_respective_window_indices]])
                window_indices = window_indices[valid_mask]
                window_weights = window_weights[valid_mask]
                valid_data_respective_window_indices = data_respective_window_indices[valid_mask]

                if len(valid_data_respective_window_indices) > 0:

                    #have valid data after screening by key qa flags, therefore write out
                    write_out = True

                    relevant_start_times = valid_start_times[window_indices]
                    relevant_end_times = valid_end_times[window_indices]
                    #if relevant first start time/ last end time exceed start/end of window, modify relevant start/end time to be that of window start/end
                    if relevant_start_times[0] < window_start_minute:
                        relevant_start_times[0] = window_start_minute
                    if relevant_end_times[-1] > window_end_minute:
                        relevant_end_times[-1] = window_end_minute

                    #get n minutes represented
                    n_minutes_represented = np.sum(window_weights)

                    #add n_minutes_represented for all relevant temporal resolutions
                    for period in relevant_periods:
                        period_tracker[period]['n_minutes_represented'] = period_tracker[period]['n_minutes_represented'] + n_minutes_represented

                    #deal with open gaps on left edge of window
                    left_edge_gap = relevant_start_times[0] - window_start_minute
                    if left_edge_gap > 0:
                        for period in relevant_periods:
                            if period_tracker[period]['open_gap']:
                                period_tracker[period]['gaps'][-1] = period_tracker[period]['gaps'][-1] + left_edge_gap
                            else:
                                period_tracker[period]['gaps'].append(left_edge_gap)
                    #reset open gap flags to be False
                    for period in relevant_periods:
                        period_tracker[period]['open_gap'] = False

                    #deal with open gaps in centre of window
                    for obs_start,obs_end in zip(relevant_start_times[1:],relevant_end_times[:-1]):
                        gap = obs_start - obs_end
                        if gap > 0:
                            for period in relevant_periods:
                                period_tracker[period]['gaps'].append(gap)

                   #deal with open gaps on right edge of window
                    right_edge_gap = window_end_minute - relevant_end_times[-1]
                    if right_edge_gap > 0:
                        for period in relevant_periods:
                            period_tracker[period]['gaps'].append(right_edge_gap)
                            #set open gap to True
                            period_tracker[period]['open_gap'] = True

                #no valid data
                else:
                    #have no valid data after screening by key qa flags, therefore do not write out
                    write_out = False

                    n_minutes_represented = 0
                    window_gap = window_end_minute - window_start_minute
                    for period in relevant_periods:
                        if period_tracker[period]['open_gap']:
                            period_tracker[period]['gaps'][-1] = period_tracker[period]['gaps'][-1] + window_gap
                        else:
                            period_tracker[period]['gaps'].append(window_gap)
                            #set open gap to True
                            period_tracker[period]['open_gap'] = True

            #--------------------------------------------------------------------------------#
            #--------------------------------------------------------------------------------#
            #now process QA flags for the evaluation of the representativeness of periods

            for period in relevant_periods:
                #if new period then calculate representativity of previous period
                if period_tracker[period]['new_period']: 
                    #if have no data in window set all relevant period windows to have a % data representativity of 0 and max data gap of 100 %
                    if period_tracker[period][representativity_var_name] == 0:
                        self.gridded_data['{}_native_representativity_percent'.format(period)][period_tracker[period]['inds']] = 0
                        self.gridded_data['{}_native_max_gap_percent'.format(period)][period_tracker[period]['inds']] = 100
                    #if have data in window, evaluate data representation across previous period
                    else:
                        rep_pc, max_gap_pc = self.check_representation(period, period_tracker[period], current_dt)
                        self.gridded_data['{}_native_representativity_percent'.format(period)][period_tracker[period]['inds']] = rep_pc
                        self.gridded_data['{}_native_max_gap_percent'.format(period)][period_tracker[period]['inds']] = max_gap_pc
                    #update period variables
                    period_tracker[period][representativity_var_name] = 0
                    period_tracker[period]['gaps'] = []
                    period_tracker[period]['inds'] = []
                    period_tracker[period]['open_gap'] = []

            #--------------------------------------------------------------------------------#

            #if have no valid data to average (after screening by key qa flags), update with no valid data QA flag, but do not write other variables
            if not write_out:
                self.gridded_data['qa'][actual_ii].append(no_valid_data_qa_flag_code)
                #also copy most recent qa codes to this empty window
                self.gridded_data['qa'][actual_ii].extend(most_recent_qa_codes)

            #proceed if have valid indices for window after screening GHOST Decreed invalid data (when have > 1 window indices), or have chosen to filter by QA flags (when have 1 index)
            else:

                #update flag stating have some valid data for station
                self.data_to_write = True

                #iterate through data variables and process data onto desired temporal resolution grid
                for key in data_keys:

                    #measured variable
                    if key == self.parameter_details['bsc_parameter_name']:

                        #more than 1 index?
                        if len(window_indices) > 1:
                            #if calculating average wind direction then first convert to complex coordinates (assume wind speed is 1 m/s)
                            if self.run_details['parameter'] in ['WND_DIR', 'WND_DIR_10M']:   
                                u = -1.0*np.sin(np.deg2rad(data[key][valid_data_respective_window_indices]))
                                v = -1.0*np.cos(np.deg2rad(data[key][valid_data_respective_window_indices]))
                                window_weighted_average = np.rad2deg(np.arctan2(np.average(u, weights=window_weights),np.average(v, weights=window_weights))) + 180.0
                            #if calculating precipitation/snow accumulation then rather than averaging the values, sum them
                            elif self.run_details['parameter'] in ['PREC_ACCUM','SNOW_ACCUM']:
                                window_weighted_average = np.sum(data[key][valid_data_respective_window_indices])
                            #if calculating average oktas then force average to be an integer amount (note .5 decimals round down)
                            elif self.run_details['parameter'] == 'CLOUD_CVG':
                                window_weighted_average = np.around(np.average(data[key][valid_data_respective_window_indices], weights=window_weights))   
                            #else, take weighted average                            
                            else:
                                window_weighted_average = np.average(data[key][valid_data_respective_window_indices], weights=window_weights)
                            #assign weighted average to grid
                            self.gridded_data[key][actual_ii] = window_weighted_average
                        #otherwise the only index gives the value
                        else:
                            self.gridded_data[key][actual_ii] = data[key][data_respective_window_indices][0]

                    #reported_uncertainty_per_measurement/derived_uncertainty_per_measurement 
                    elif key in ['reported_uncertainty_per_measurement', 'derived_uncertainty_per_measurement']:
                        #more than 1 index?
                        if len(window_indices) > 1:
                            #take weighted average (excluding NaNs first)
                            non_nan_indices = np.arange(len(valid_data_respective_window_indices), dtype=np.int32)[~pd.isnull(data[key][valid_data_respective_window_indices])]                          
                            #if non-nan indices?
                            if len(non_nan_indices) > 0:
                                self.gridded_data[key][actual_ii] = np.average(data[key][valid_data_respective_window_indices][non_nan_indices], weights=window_weights[non_nan_indices])
                        #otherwise the only index gives the value
                        else:
                            self.gridded_data[key][actual_ii] = data[key][data_respective_window_indices][0]

                    #day_night_code/weekday_weekend_code/season_code
                    elif key in ['day_night_code', 'weekday_weekend_code', 'season_code']:
                        #more than 1 index?
                        if len(window_indices) > 1:
                            #take weighted mode                         
                            self.gridded_data[key][actual_ii] = weighted_mode(data[key][valid_data_respective_window_indices],window_weights)
                        #otherwise the only index gives the value
                        else:
                            self.gridded_data[key][actual_ii] = data[key][data_respective_window_indices][0]

                    #qa codes
                    elif key == 'qa':

                        #if have more than 1 measurement in window, need to deal with multiple qa flag codes
                        #this involves 2 steps:
                        #1. drop qa codes which are no longer relevant when taken across an averaged period (most of which have already been applied to screen invalid data)
                        #2. other qa codes are retained if they are more frequently reported across period than not, otherwise they are dropped
                        if len(window_indices) > 1:

                            #get all unique qa flag codes across measurements that are averaging
                            unique_qa = list(set(x for l in data[key][valid_data_respective_window_indices] for x in l))
                            #some qa flags in unique list no longer make sense when taking average over period, therefore drop them
                            unique_qa = [x for x in unique_qa if x not in nonsensical_qa_flag_codes]

                            #for some qa codes need to check if they are more frequently reported or not across the period, if more frequently not reported, drop the code
                            for modal_flag in modal_qa_flag_codes:
                                #first only proceed if flag is actually at all reported in the window
                                if modal_flag in unique_qa:
                                    reported_count = 0
                                    not_reported_count = 0
                                    for ww, qa_codes in enumerate(data[key][valid_data_respective_window_indices]):
                                        if modal_flag in qa_codes:
                                            reported_count += window_weights[ww]
                                        else:
                                            not_reported_count += window_weights[ww]
                                    if not_reported_count > reported_count:
                                        unique_qa.remove(modal_flag)

                            #append unique relevant qa codes for averaged period
                            self.gridded_data[key][actual_ii].extend(unique_qa)

                            #make copy of processed qa codes, for copying when have empty window
                            most_recent_qa_codes = copy.deepcopy(unique_qa)

                        else:
                            #add previously processed qa codes for this window to those processed through temporal gridding
                            self.gridded_data[key][actual_ii].extend(data[key][data_respective_window_indices][0])

                            #make copy of previously processed qa codes for this window, for copying when have empty window
                            most_recent_qa_codes = copy.deepcopy(data[key][data_respective_window_indices][0])

                    #data flags
                    elif key == 'flag':

                        #if have more than 1 measurement in window, then discard flag as will have already taken account of them (through associated qa codes) and do not make sense as an average over a period
                        #instead replace with valid data flag (as will have passed invalid flag screening)
                        if len(window_indices) > 1:

                            #append valid data flag code as will have passed invalid flag screening
                            self.gridded_data[key][actual_ii] = [valid_data_flag_code]

                        else:
                            self.gridded_data[key][actual_ii] = data[key][data_respective_window_indices][0]
        

    def check_representation(self, period, period_tracker, current_dt):

        '''function for temporal representation checks of data (after screening by key QA flags). 
           representation checks are handled for 4 different temporal periods: hourly, diurnal, monthly, annual
           hourly period:  start/end NN:00 UTC
           daily period:   start/end NN 00:00 UTC
           monthly period: start/end NN:01 00:00 UTC
           annual period:  start/end NNNN:01:01 00:00 UTC

           two types are checks are done to evaluate the representativity of each of the different periods.
           first is Data Completeness, i.e. what percentage across the period is represented by valid data (after screening by key QA flags)?
           second is Maximum Data Gap, i.e. what is the maximum data gap percentage in the provided valid data across the period (after screening by key QA flags)? 
    
           if the temporal resolution is instantaneous rather then getting the representation over the total period, calculate how many hours inside the period have some data, out of total number of required hours

           the representativity percentages are returned as rounded integers.
        '''

        #get maximum gap in all_gaps array (if a gap exists)
        if len(period_tracker['gaps']) == 0:
            max_temporal_gap = 0
        else:   
            max_temporal_gap = np.max(period_tracker['gaps'])
    
        #get maximum possible minutes by period 
        if period == 'hourly':
            max_minutes_possible = 60
        elif period == 'daily':
            max_minutes_possible = 60*24
        elif period == 'monthly':
            current_year = current_dt.year
            current_month = current_dt.month
            days_in_month = monthrange(current_year, current_month)[1]
            max_minutes_possible = (60*24)*days_in_month
        elif period == 'annual':
            current_year = current_dt.year
            d1 = datetime.date(current_year, 1, 1) 
            d2 = datetime.date(current_year + 1, 1, 1) 
            days_in_year = (d2 - d1).days 
            max_minutes_possible = (60*24)*days_in_year

        #get the total number of hours in the period
        n_total_hours = max_minutes_possible/60

        #100./max_minutes_possible (for multiplying by)
        percent_max_minutes = 100./max_minutes_possible

        #if temporal resolution code is instantaneous then calculate how many hours inside the period have some data, out of total number of relevant hours
        if 'n_hours_represented' in list(period_tracker.keys()):
            #100./n_total_hours (for multiplying by)
            percent_max_hours = 100./n_total_hours
        
            #calculate % number of hours represented by valid data
            percent_period_represented = int(np.around(percent_max_hours*period_tracker['n_hours_represented']))

        else:    
            #100./max_minutes_possible (for multiplying by)
            percent_max_minutes = 100./max_minutes_possible    

            #calculate % of period represented by valid data
            percent_period_represented = int(np.around(percent_max_minutes*period_tracker['n_minutes_represented']))

        #calculate % max data gap across period
        percent_period_max_data_gap = int(np.around(percent_max_minutes*max_temporal_gap))
            
        return percent_period_represented, percent_period_max_data_gap


    def drop_duplicates_sort_and_pad_codes(self, array_name,array,pad_value=255):

        '''take integer 2D code arrays, drop duplicates and sort alphabetically (natural human order)
           then pad arrays with fill_value (255) - converting to 2D numpy arrays for preparation to writing to netCDF
        '''

        #get unique codes in each list of codes per measurement and sort them
        unique_sorted_array = [list(sorted(set(current_list))) for current_list in array]

        #get maximum possible number of codes
        if array_name == 'qa':
            N_codes_max = len(list(standard_QA_name_to_QA_code.keys()))
        elif array_name == 'flag':
            N_codes_max = len(list(standard_data_flag_name_to_data_flag_code.keys()))

        #pad lists with fill value to maximum N codes - converting all into 2D numpy array strcture
        padded_unique_sorted_array = np.array([i + [pad_value]*(N_codes_max-len(i)) for i in unique_sorted_array], dtype=np.uint8)

        return padded_unique_sorted_array


    def write_station_specific_netCDF_by_month(self):

        '''write netCDF file, month by month, in desired temporal range for specific station'''

        #get metadata updata stamp variable
        update_metadata_stamp = self.metadata['meta_update_stamp']
    
        #dictionary to map python types to netcdf types
        type_map = {np.uint8:'u1', np.uint32:'u4', object:str, np.float32:'f4', np.float64:'f8'}

        #some metadata variables make no sense when taken as a modal value across the month (i.e. timestamps)
        #therefore do write/process variables of this type
        #remove from metadata record array
        self.metadata = rf.drop_fields(self.metadata, ['meta_update_stamp','data_download_stamp','data_revision_stamp'])

        #get metadata/data keys
        metadata_keys = list(self.metadata.dtype.names)
        data_keys =     list(self.gridded_data.dtype.names)

        #get start and end datetimes of raw data
        start_dt = fractional_minutes_from_reference_to_datetime(float(self.start_stamp))
        end_dt = fractional_minutes_from_reference_to_datetime(float(self.end_stamp))

        #get data start_year/start_month and end_year/end month
        data_start_year = start_dt.year
        data_start_month = start_dt.month
        data_end_year = end_dt.year
        data_end_month = end_dt.month
        data_end_day = end_dt.day
        data_end_hour = end_dt.hour
        #adjust data end year (if end date is 20150101 0000 for example, the end year should be 2014, not 2015)
        if (data_end_month == 1) & (data_end_day == 1) & (data_end_hour == 0):
            data_end_year = data_end_year - 1
            data_end_month = 12
        #adjust data end month (if end date is 20150301 0000 for example, the end month should be 2, not 3)
        elif (data_end_day == 1) & (data_end_hour == 0):
            data_end_month = data_end_month - 1

        #create list of years to process from provided start_date and end_date
        desired_start_year = int(self.run_details['start_year'])
        desired_end_year = int(self.run_details['end_year'])
        desired_start_month = int(self.run_details['start_month'])
        desired_end_month = int(self.run_details['end_month'])
        process_years = np.arange(desired_start_year, desired_end_year+1)

        #cut list of years to process for years which are >= data_start_year and <= data_end_year
        process_years = process_years[process_years >= data_start_year]
        process_years = process_years[process_years <= data_end_year]

        #loop through years and months to process, and write to monthly station specific .npz files
        #iterate through process_years
        for year in process_years:

            #create list of months to process

            #if year == data start year
            if year == desired_start_year:
                process_months = np.arange(desired_start_month, 13)

            #if year == desired end year
            elif year == desired_end_year:
                process_months = np.arange(1,desired_end_month)

            #full year of months otherwise
            else:
                process_months = np.arange(1,13)

            #if in year that data finishes or ends then cut list of months to process for months which are >= data_start_month and <= data_end_month
            if year == data_start_year:
                process_months = process_months[process_months>=data_start_month]

            elif year == data_end_year:
                process_months = process_months[process_months<=data_end_month]

            #iterate through process_months
            for month in process_months:

                if month < 10:
                    month_string = '0'+str(month)
                else:
                    month_string = str(month)

                #create yearmonth string
                yearmonth_str = '{}{}'.format(year,month_string)

                #get minutes of start and end of monthly window
                start_dt = datetime.datetime(year,month,1,0,0)
                end_dt = start_dt + relativedelta(months=+1)
                start_window_minutes = datetime_to_fractional_minutes_from_reference(start_dt)
                end_window_minutes = datetime_to_fractional_minutes_from_reference(end_dt)

                #cut data,metadata for monthly period
                #after gridding, data times are guaranteed to align to window edges
                start_data_i = bisect.bisect_left(self.gridded_data['time'], start_window_minutes)
                end_data_i = bisect.bisect_left(self.gridded_data['time'], end_window_minutes)
                cut_data = self.gridded_data[start_data_i:end_data_i]
                #check you have some valid data for the month (i.e. not all NaNs), if not continue to next month
                if all(pd.isnull(cut_data[self.parameter_details['bsc_parameter_name']])):
                    continue

                #cut qa/flags
                cut_qa = self.padded_qa[start_data_i:end_data_i,:]
                cut_flag = self.padded_flag[start_data_i:end_data_i,:]

                #get timestep spacing
                if self.temporal_resolution_vars['temporal_resolution_datetime_frequency'][0] == '3':
                    timestep = 3
                elif self.temporal_resolution_vars['temporal_resolution_datetime_frequency'][0] == '6':
                    timestep = 6
                else:
                    timestep = 1

                #make time array (hours/days/months) from start of month to end of month
                monthly_time_array = np.arange(start=0, stop=len(cut_data['time'])*timestep, step=timestep, dtype=cut_data['time'].dtype)

                #meta stamp times are not guaranteed however to align with window edges
                start_meta_i = bisect.bisect(update_metadata_stamp,start_window_minutes)-1
                #if meta starts partially through window, start index will be returned as -1, change this to 0 (as want all from start)
                if start_meta_i < 0:
                    start_meta_i = 0
                end_meta_i = bisect.bisect_left(update_metadata_stamp,end_window_minutes)
                cut_metadata = self.metadata[start_meta_i:end_meta_i]
                cut_update_metadata_stamp = update_metadata_stamp[start_meta_i:end_meta_i]

                #-----------------------------------------------------------------#

                #get modal metadata in month
                #if have more than 1 row of metadata in month, at least 1 field has multiple metadata values, take modal value across month for each field (by most minutes occurring)
                if cut_metadata.shape[0] > 1:

                    #create numpy dataframe to place modal metadata in month
                    #create lists for standardising data formats of the input data dictionary
                    standard_dtype = [(key,self.metadata[key].dtype) for key in metadata_keys]
                    #create numpy dataframe to store standardised forms of variables inside data dictionary
                    modal_metadata = np.empty(1, dtype=standard_dtype)

                    for key in metadata_keys:

                        #metadata values for field
                        metadata_values = cut_metadata[key]

                        #take mode of weighted metadata (by number of minutes that it is relevant for) that does not equal NaN
                        #if are multiple items with same number of instances, the lowest sorted instance will be returned (i.e. 'a' rather 'b'; 1 rather than 2)
                        weighted_meta = {}
                        for xx, metadata_value in enumerate(metadata_values):
                            #last index
                            if (xx+1) == len(metadata_values):
                                minutes_represented = end_window_minutes - cut_update_metadata_stamp[xx]
                            else:
                                minutes_represented = cut_update_metadata_stamp[xx+1] - cut_update_metadata_stamp[xx]

                            if not pd.isnull(metadata_value):
                                if metadata_value in list(weighted_meta.keys()):
                                    weighted_meta[metadata_value] = weighted_meta[metadata_value]+minutes_represented
                                else:
                                    weighted_meta[metadata_value] = minutes_represented

                        #if have no valid metadata, modal value is np.NaN
                        if len(weighted_meta) == 0:
                             modal_value = np.NaN
                        else:
                            counter_instance = Counter(weighted_meta)
                            modal_value = counter_instance.most_common(1)[0][0]

                        #append model value to new metadata frame
                        modal_metadata[key] = modal_value

                    modal_metadata = modal_metadata[0]

                #only 1 value in month for each metadata variable, so existing array is model metadata array
                else:
                    modal_metadata = cut_metadata[0]
    
                #-----------------------------------------------------------------#
                #create while loop to ensure netCDF is created correctly 
                netCDF_not_valid = True
                while netCDF_not_valid:

                    #Create netCDF file for specific yearmonth, and output temporal resolution code
                    netCDF_fname = self.run_details['data_root'] + '{}/{}/{}/{}/temporary/{}/{}/{}.nc'.format(self.run_details['network_write_dir'],self.run_details['ghost_version'],self.temporal_resolution_vars['temporal_resolution_path'],self.parameter_details['bsc_parameter_name'],year,month_string,self.station_reference)
    
                    #create netCDF root
                    root_grp = Dataset(netCDF_fname, 'w', format="NETCDF4")

                    #auto mask arrays
                    #root_grp.set_auto_mask(True)

                    #file contents
                    root_grp.title = '{} data in the {} network in {}-{}.'.format(self.parameter_details['long_parameter_name'],self.run_details['network'], year, month_string)
                    root_grp.institution = 'Barcelona Supercomputing Center'
                    root_grp.source = 'Observations'
                    root_grp.creator_name = 'Dene R. Bowdalo'
                    root_grp.creator_email = 'dene.bowdalo@bsc.es'
                    root_grp.version = modal_metadata['GHOST_version']

                    #add WIGOS global attributes
                    root_grp.application_area = 'Monitoring atmospheric composition'
                    root_grp.domain = 'Atmosphere'
                    if self.parameter_details['matrix'] in ['aod','extaod','absaod','absaod-bc','absaod-brc','scataod','ssa','asy','rin','vconc','size']:
                        root_grp.observed_layer = 'Total atmospheric column'
                    else:
                        root_grp.observed_layer = 'Land surface'

                    #set data licence
                    root_grp.data_licence = 'BSD-3-Clause. Copyright 2023 Dene Bowdalo.'    

                    #get names of all metadata/data variable fields
                    metadata_keys = list(modal_metadata.dtype.names)
                    data_keys =     list(cut_data.dtype.names)
                    #add qa/flag to data_keys
                    data_keys.extend(['qa','flag','flag_simple','{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name'])])    

                    #get dicts with standard formats for metadata/data
                    metadata_format_dict = get_standard_metadata(self.parameter_details)
                    data_format_dict     = get_standard_data(self.parameter_details)

                    #netcdf dimensions (station as unlimited/record dimension)
                    root_grp.createDimension('station', None)
                    root_grp.createDimension('time', len(monthly_time_array))
                    root_grp.createDimension('N_qa_codes', len(list(standard_QA_name_to_QA_code.keys())))
                    root_grp.createDimension('N_flag_codes', len(list(standard_data_flag_name_to_data_flag_code.keys())))

                    #set variables for data parameters
                    for data_key in data_keys:
                        if (data_key in ['qa','flag','flag_simple']) or ('representativity_percent' in data_key) or ('max_gap_percent' in data_key):
                            current_data_type = 'u1'
                        elif data_key == '{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name']):
                            current_data_type = type_map[data_format_dict[self.parameter_details['bsc_parameter_name']]['data_type']]
                        else:
                            current_data_type = type_map[data_format_dict[data_key]['data_type']]

                        if data_key == 'time':
                            if not self.run_details['compression_level']:
                                var = root_grp.createVariable(data_key, current_data_type, ('time',), compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                            else:
                                var = root_grp.createVariable(data_key, current_data_type, ('time',), compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                        elif data_key == 'qa':
                            if not self.run_details['compression_level']:
                                var = root_grp.createVariable(data_key, current_data_type, ('station','time','N_qa_codes'), fill_value=255, compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                            else:
                                var = root_grp.createVariable(data_key, current_data_type, ('station','time','N_qa_codes'), fill_value=255, compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                        elif data_key == 'flag':
                            if not self.run_details['compression_level']:
                                var = root_grp.createVariable(data_key, current_data_type, ('station','time','N_flag_codes'), fill_value=255, compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                            else:
                                var = root_grp.createVariable(data_key, current_data_type, ('station','time','N_flag_codes'), fill_value=255, compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                        elif data_key == 'flag_simple':
                            if not self.run_details['compression_level']:
                                var = root_grp.createVariable(data_key, current_data_type, ('station','time'), fill_value=255, compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                            else:
                                var = root_grp.createVariable(data_key, current_data_type, ('station','time'), fill_value=255, compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                        else:
                            if current_data_type == str:
                                var = root_grp.createVariable(data_key, current_data_type, ('station','time'))
                            else:
                                if not self.run_details['compression_level']:
                                    var = root_grp.createVariable(data_key, current_data_type, ('station','time'), compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                                else:
                                    var = root_grp.createVariable(data_key, current_data_type, ('station','time'), compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])

                        #set window representativity percent variable attributes
                        if 'representativity_percent' in data_key:
                            #get window resolution of variable
                            period = data_key.split('_')[0] 
                            var.standard_name = '{} native representativity percent'.format(period)  
                            var.long_name = '{} native representativity percent'.format(period)
                            var.units = '%'
                            var.description = 'Percentage of the {} averaged measurement UTC window represented by native resolution data.'.format(period) 
                        #set window max data gap percent variable attributes
                        elif 'max_gap_percent' in data_key:
                            #get window resolution of variable
                            period = data_key.split('_')[0] 
                            var.standard_name = '{} native max gap percent'.format(period)    
                            var.long_name = '{} native max gap percent'.format(period) 
                            var.units = '%'
                            var.description = 'Percentage of the maximum data gap in the {} averaged measurement UTC window filled by native resolution data, relative to the total window length.'.format(period) 
                        #prefiltered data array
                        elif data_key == '{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name']):
                            var.standard_name = data_format_dict[self.parameter_details['bsc_parameter_name']]['standard_name']
                            var.long_name = data_format_dict[self.parameter_details['bsc_parameter_name']]['long_name']
                            var.units = data_format_dict[self.parameter_details['bsc_parameter_name']]['units']
                            var.description = '{} Prefiltered by default QA.'.format(data_format_dict[self.parameter_details['bsc_parameter_name']]['description'])
                        #simple flags
                        elif data_key == 'flag_simple':
                            var.standard_name = 'simplified flags'
                            var.long_name = 'data reporter provided simplified standardised flags'
                            var.units = 'unitless'
                            var.description = 'List of associated simplified data flag codes per measurement, indicating the data quality of a specific measurement, provided by the data reporter. The template for the flag standardisations is taken from WaterML2. Fill value code of 255.'                      
                        #set other variable attributes 
                        else:
                            var.standard_name = data_format_dict[data_key]['standard_name']
                            var.long_name = data_format_dict[data_key]['long_name']
                            var.units = data_format_dict[data_key]['units']
                            var.description = data_format_dict[data_key]['description']
                        
                        #time variable specific attributes
                        if data_key == 'time':
                            var.units = '{} since {}-{}-01 00:00:00'.format(self.temporal_resolution_vars['descriptive_temporal_resolution'],year,month_string)
                            var.description = 'Time in {} since {}-{}-01 00:00 UTC. Time given refers to the start of the time window the measurement is representative of (temporal resolution).'.format(self.temporal_resolution_vars['descriptive_temporal_resolution'],year,month_string)
                            var.axis = 'T'
                            var.calendar = 'standard'
                            var.tz = 'UTC'
                        
                        #add data variable identifers (if not empty str)
                        # also add matrix for variable as attribute
                        if data_key in [self.parameter_details['bsc_parameter_name'], '{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name'])]:
                            var_dict = self.parameter_details
                            var.matrix = self.parameter_details['matrix']
                        else:
                            if data_key in data_format_dict:
                                var_dict = data_format_dict[data_key]
                            else:
                                continue

                        if 'identifiers' in var_dict:
                            identifier_dict = var_dict['identifiers']
                            for identifier_name, identifier in identifier_dict.items():
                                if identifier != '':
                                    var.setncattr(identifier_name,identifier)
                        
                    #create and fill local/mean solar time variables (only if temporal resolution is H/3H/6H/HI/3HI/6HI)
                    if 'hourly' in self.temporal_resolution_vars['temporal_resolution_path']:
                        local_time = create_local_time(modal_metadata['station_timezone'], cut_data['time'][:]) 
                        if not self.run_details['compression_level']:
                            var = root_grp.createVariable('local_time', 'u4', ('station','time'), compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                        else:
                            var = root_grp.createVariable('local_time', 'u4', ('station','time'), compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                        var.standard_name = 'local time'
                        var.long_name = 'local time'
                        var.units = 'minutes since 0001-01-01 00:00:00'
                        var.description = 'Time in minutes since 0001-01-01 00:00 local time. Time given refers to the start of the time window the measurement is representative of (temporal resolution).'
                        var.axis = 'T'
                        var.calendar = 'proleptic_gregorian'
                        var.tz = 'local'
                        root_grp['local_time'][0,:] = local_time 

                        solar_time = create_solar_time(modal_metadata['longitude'], cut_data['time'][:]) 
                        if not self.run_details['compression_level']:
                            var = root_grp.createVariable('mean_solar_time', 'u4', ('station','time'), compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                        else:
                            var = root_grp.createVariable('mean_solar_time', 'u4', ('station','time'), compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                        var.standard_name = 'mean solar time'
                        var.long_name = 'mean solar time'
                        var.units = 'minutes since 0001-01-01 00:00:00'
                        var.description =  'Time in minutes since 0001-01-01 00:00 mean solar time. Time given refers to the start of the time window the measurement is representative of (temporal resolution).'
                        var.axis = 'T'
                        var.calendar = 'proleptic_gregorian'
                        var.tz = 'mean solar'
                        root_grp['mean_solar_time'][0,:] = solar_time 

                    #set variables for metadata parameters
                    for metadata_key in metadata_keys:

                        #current_data_type = type_map[metadata[metadata_key].dtype]
                        current_data_type = type_map[metadata_format_dict[metadata_key]['data_type']]
                        if current_data_type == str:
                            var = root_grp.createVariable(metadata_key, current_data_type, ('station',))
                        else:
                            if not self.run_details['compression_level']:
                                var = root_grp.createVariable(metadata_key, current_data_type, ('station',), compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                            else:
                                var = root_grp.createVariable(metadata_key, current_data_type, ('station',), compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])

                        #set variable attributes
                        var.standard_name = metadata_format_dict[metadata_key]['standard_name']
                        var.long_name = metadata_format_dict[metadata_key]['long_name']
                        var.units = metadata_format_dict[metadata_key]['units']
                        var.description = metadata_format_dict[metadata_key]['description']

                        #variable specific attributes
                        if metadata_key == 'longitude':
                            var.axis = 'X'
                        elif metadata_key == 'latitude':
                            var.axis = 'Y'

                        #add metadata variable identifers (if not empty str)
                        if metadata_key in metadata_format_dict:
                            if 'identifiers' in metadata_format_dict[metadata_key]:
                                for identifier_name, identifier in metadata_format_dict[metadata_key]['identifiers'].items():
                                    if identifier != '':
                                        var.setncattr(identifier_name,identifier)

                    #write station data to netCDF
                    for data_key in data_keys:
                        if data_key == 'time':
                            root_grp[data_key][:] = monthly_time_array
                        elif data_key == 'qa':
                            root_grp[data_key][0,:,:] = cut_qa
                        elif data_key == 'flag':
                            root_grp[data_key][0,:,:] = cut_flag
                        elif data_key == 'flag_simple':
                            flag_simple = get_flag_simple(cut_flag,cut_qa)
                            root_grp[data_key][0,:] = flag_simple
                        elif data_key == '{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name']):
                            if self.parameter_details['extreme_lower_limit'] < 0.0: 
                                default_qa = [standard_QA_name_to_QA_code[qa] for qa in providentia_defaults['qa_non_negative']]
                            else:
                                default_qa = [standard_QA_name_to_QA_code[qa] for qa in providentia_defaults['qa_standard']]
                            prefiltered_data = copy.deepcopy(cut_data[self.parameter_details['bsc_parameter_name']])
                            prefiltered_data[np.isin(cut_qa, default_qa).any(axis=1)] = np.NaN
                            root_grp[data_key][0,:] = prefiltered_data
                        else:
                            root_grp[data_key][0,:] = cut_data[data_key]

                    #write station metadata to netCDF
                    for metadata_key in metadata_keys:
                        #convert float NaNs to str for string metadata fields
                        if root_grp[metadata_key].dtype == str:
                            root_grp[metadata_key][0] = str(modal_metadata[metadata_key])
                        else:
                            root_grp[metadata_key][0] = modal_metadata[metadata_key]

                    #close writing to netCDF 
                    root_grp.close()

                    #attempt to read netCDF root, if successful leave while loop
                    try:
                        test_read = Dataset(netCDF_fname)
                        netCDF_not_valid = False
                    except:
                        continue

                #give 770 permissions for file and make owner bsc32
                set_file_permissions_ownership(netCDF_fname)
