import copy
import os
import subprocess
import time

from netCDF4 import Dataset
import numpy as np

from crossnetwork_processing_functions import set_file_permissions_ownership, check_directory_existence
from crossnetwork_processing_functions import get_default_product_parameters_per_network, create_GHOST_stage_logfile
from data_product_formulae import Data_Products
from GHOST_standards import get_standard_data, get_standard_metadata
from GHOST_standards import standard_parameters, standard_temporal_resolutions
from GHOST_standards import providentia_defaults, standard_QA_name_to_QA_code, standard_data_flag_name_to_data_flag_code
from GHOST_standards import standard_simple_data_flag_name_to_simple_data_flag_code

class DataProduct:

    '''Class which calculates data product from other processed components.'''
    
    def __init__(self, yearmonth_str, temporal_resolution_code, region, run_details, parameter_details, network_details):

        #add passed variables to self
        self.yearmonth_str = yearmonth_str
        self.year = self.yearmonth_str[:4]
        self.month = self.yearmonth_str[4:6]
        self.temporal_resolution_code = temporal_resolution_code
        self.region = region
        self.run_details = run_details
        self.parameter_details = parameter_details
        self.network_details = network_details

        self.run_details['log_file_str'] += 'CALCULATING DATA PRODUCT IN {}\n'.format(self.yearmonth_str)

    def do_work(self):
        '''Function which does executes the functions of the class in a pipeline'''

        #create monthly netCDF
        self.calculate_data_product()

    def calculate_data_product(self):
        '''Function which calculates data product from other processed components'''

        #define used flag codes
        unflagged_flag = standard_data_flag_name_to_data_flag_code['Un-Flagged Data']
        preliminary_flag = standard_data_flag_name_to_data_flag_code['Preliminary Data']
        missing_flag = standard_data_flag_name_to_data_flag_code['Missing Data']
        estimated_flag_names = ['Estimated Data - Unspecified','Estimated Data - Measured Negative Value',
                                'Estimated Data - No Value Detected','Estimated Data - Value Below Detection Limit',
                                'Estimated Data - Value Above Detection Limit',
                                'Estimated Data - Value Substituted from Secondary Monitor',
                                'Estimated Data - Multiple Parameters Aggregated']
        estimated_flags = np.array([standard_data_flag_name_to_data_flag_code[estimated_flag_name] for estimated_flag_name in estimated_flag_names], dtype=np.uint8)
        suspect_flag_names = ['Extreme/Irregular Data - Unspecified','High Variability of Data',
                            'No Value Detected','Reconstructed/Recalculated Data','Value Close to Detection Limit',
                            'Value Below Acceptable Range','Value Above Acceptable Range',
                            'Value Below Detection Limit','Value Above Detection Limit']
        suspect_flags = np.array([standard_data_flag_name_to_data_flag_code[suspect_flag_name] for suspect_flag_name in suspect_flag_names], dtype=np.uint8)

        #define used QA codes
        product_code = standard_QA_name_to_QA_code['Data Product']
        insufficient_code = standard_QA_name_to_QA_code['Insufficient Data to Calculate Data Product']
        ghost_invalid_qa = standard_QA_name_to_QA_code['Invalid Data Provider Flags - GHOST Decreed']

        #define simple data flag codes
        unchecked_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['unchecked']
        missing_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['missing']
        estimate_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['estimate']
        suspect_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['suspect']
        poor_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['poor']
        good_simple_flag = standard_simple_data_flag_name_to_simple_data_flag_code['good']

        #set network directory where to read/write data
        if self.region == 'GLOBAL':
            network_write_dir = self.run_details['network']
        else:
            network_write_dir = '{}_{}'.format(self.run_details['network'], self.region)

        #get output temporal resolution specific variables
        temporal_resolution_vars = standard_temporal_resolutions[self.temporal_resolution_code]

        # get units of concentration parameter and wet dep parameter
        if 'WETDEP' in self.run_details['parameter']:
            input_parameter = self.run_details['parameter'].split('_WET')[0]+'_PREC'
            unit_convert_from_to = {'concentration_standard_unit': standard_parameters[input_parameter]['standard_units'], 'deposition_standard_unit': self.parameter_details['standard_units']}
        else:
            unit_convert_from_to = {}

        #get parameters and function name needed for calculation
        product_dict = get_default_product_parameters_per_network(self.run_details['network'])[self.run_details['parameter']]
        product_function_name = product_dict['function']
        calculation_parameters = product_dict['components']
        calculation_parameters_bsc = [standard_parameters[calculation_parameter]['bsc_parameter_name'] for calculation_parameter in calculation_parameters]
        max_calculation_parameters = len(calculation_parameters_bsc)
        if 'option' in product_dict:
            max_calculation_parameters = max_calculation_parameters - (len(product_dict['option']) - 1)

        #iterate through calculation parameters and get intersecting station references across files
        station_references = []
        for calculation_parameter_bsc in calculation_parameters_bsc:
            param_fname = '{}{}/{}/{}/{}/{}_{}.nc'.format(self.run_details['data_root'],network_write_dir,self.run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],calculation_parameter_bsc,calculation_parameter_bsc,self.yearmonth_str)
            if os.path.isfile(param_fname):
                root = Dataset(param_fname)
                station_references.extend(root['station_reference'][:])
                #close netCDF
                root.close()

        # station_reference: stationcode_measurementinstrument
        # parameters needed for calculation are measured with different instruments -> have different station reference -> cut them only take first part aka 'true station reference'
        if 'WETDEP' in self.run_details['parameter']:
            station_references = [station_reference.split('_')[0] for station_reference in station_references]

        # station references of new netcdf files
        #get intersecting station references across files
        intersecting_station_references = []
        for (name, count) in zip(*np.unique(station_references, return_counts=True)):
            if count == max_calculation_parameters: # =2
                intersecting_station_references.append(name)

        #for special cases where can have option of different middle wavelengths, 
        #ensure have outer wavelengths to proceed with station
        #preferentially only proceed with first available middle wavelength
        #only keep intersecting station references where have outer wavelengths and a middle wavlength was selected
        calculation_parameters_station = {}
        if 'option' in product_dict:
            for intersecting_station_reference in intersecting_station_references:
                valid_station = True
                resolved_option = False
                params_to_save = {'ghost':[], 'bsc':[]}

                for calculation_parameter, calculation_parameter_bsc in zip(calculation_parameters, calculation_parameters_bsc):
                    param_fname = '{}{}/{}/{}/{}/{}_{}.nc'.format(self.run_details['data_root'],network_write_dir,self.run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],calculation_parameter_bsc,calculation_parameter_bsc,self.yearmonth_str)

                    if (calculation_parameter in product_dict['option']) & (resolved_option):
                        continue

                    if not os.path.isfile(param_fname):
                        if calculation_parameter not in product_dict['option']:
                            valid_station = False
                            break
                        else:
                            continue
                    
                    root = Dataset(param_fname)
                    if intersecting_station_reference not in root['station_reference'][:]:
                        if calculation_parameter not in product_dict['option']:
                            valid_station  = False
                            break
                        else:
                            continue
                    
                    if calculation_parameter in product_dict['option']:
                        resolved_option = True
                    params_to_save['ghost'].append(calculation_parameter)
                    params_to_save['bsc'].append(calculation_parameter_bsc)
                    root.close()

                if (valid_station) & (resolved_option):
                    calculation_parameters_station[intersecting_station_reference] = {'ghost':params_to_save['ghost'],'bsc':params_to_save['bsc']}

            #update intersecting station references
            intersecting_station_references = list(calculation_parameters_station.keys())

        #if no intersecting station references, then do not write file and write log informing so
        if len(intersecting_station_references) == 0:

            #return logfile to inform yearmonth has no data to write (1)
            self.run_details['log_file_str'] += 'YEARMONTH HAS NO INTERSECTING STATIONS ACROSS COMPONENTS NEEDED TO CALCULATE DATA PRODUCT'  
            create_GHOST_stage_logfile(1,self.run_details)
            
        #otherwise start creating file
        else:

            #create new netCDF file for yearmonth
            netCDF_fname = self.run_details['data_root'] + '{}/{}/{}/{}/{}_{}.nc'.format(network_write_dir,self.run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],self.parameter_details['bsc_parameter_name'],self.parameter_details['bsc_parameter_name'],self.yearmonth_str)
            check_directory_existence('/'.join(netCDF_fname.split('/')[:-1])+'/', directories_not_to_test='{}{}/'.format(self.run_details['data_root'],network_write_dir))
            root_new = Dataset(netCDF_fname, "w", format="NETCDF4")

            #get dicts with standard formats for metadata/data
            metadata_format_dict = get_standard_metadata(self.parameter_details)
            data_format_dict     = get_standard_data(self.parameter_details)

            #copy create dimensions, variables and attributes in new netCDF
            #but do not fill variables
            root = Dataset('{}{}/{}/{}/{}/{}_{}.nc'.format(self.run_details['data_root'],network_write_dir,self.run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],calculation_parameters_bsc[0],calculation_parameters_bsc[0],self.yearmonth_str))

            #copy global attributes to new netCDF
            for name in root.ncattrs():
                if name == 'title': 
                    root_new.title = '{} data in the {} network in {}-{}.'.format(self.parameter_details['long_parameter_name'],self.run_details['network'], self.year, self.month)
                elif name == 'observed_layer':
                    if self.parameter_details['matrix'] in ['aod','extaod','absaod','absaod-bc','absaod-brc','scataod','ssa','asy','rin','vconc','size']:
                        root_new.observed_layer = 'Total atmospheric column'
                    else:
                        root_new.observed_layer = 'Land surface'
                else:
                    root_new.setncattr(name, root.getncattr(name))

            #copy dimensions to new netCDF
            #modify station dimension to be length of 
            for name, dimension in root.dimensions.items():
                if name == 'station':
                    root_new.createDimension(name, len(intersecting_station_references))
                else:
                    root_new.createDimension(name, len(dimension))

            # copy variables to net netCDF, without flling with data
            for name, variable in root.variables.items():

                #get current var dtype
                current_dtype = variable.dtype
                        
                # create variables
                if name in [calculation_parameters_bsc[0], '{}_prefiltered_defaultqa'.format(calculation_parameters_bsc[0])]:
                    if name == calculation_parameters_bsc[0]:
                        if not self.run_details['compression_level']:
                            var_root = root_new.createVariable(self.parameter_details['bsc_parameter_name'], variable.datatype, variable.dimensions, compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                        else:
                            var_root = root_new.createVariable(self.parameter_details['bsc_parameter_name'], variable.datatype, variable.dimensions, compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                    elif name == '{}_prefiltered_defaultqa'.format(calculation_parameters_bsc[0]):
                        if not self.run_details['compression_level']:
                            var_root = root_new.createVariable('{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name']), variable.datatype, variable.dimensions, compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                        else:
                            var_root = root_new.createVariable('{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name']), variable.datatype, variable.dimensions, compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                    var_root.standard_name = data_format_dict[self.parameter_details['bsc_parameter_name']]['standard_name']
                    var_root.long_name = data_format_dict[self.parameter_details['bsc_parameter_name']]['long_name']
                    var_root.units = data_format_dict[self.parameter_details['bsc_parameter_name']]['units']
                    if name == calculation_parameters_bsc[0]:
                        var_root.description = data_format_dict[self.parameter_details['bsc_parameter_name']]['description']
                    elif name == '{}_prefiltered_defaultqa'.format(calculation_parameters_bsc[0]):
                        var_root.description = '{} Prefiltered by default QA.'.format(data_format_dict[self.parameter_details['bsc_parameter_name']]['description'])
                    var_root.matrix = self.parameter_details['matrix']
                    if 'identifiers' in self.parameter_details:
                        identifier_dict = self.parameter_details['identifiers']
                        for identifier_name, identifier in identifier_dict.items():
                            if identifier != '':
                                var_root.setncattr(identifier_name,identifier)
                else:
                    if '_FillValue' in root[name].ncattrs():
                        if not self.run_details['compression_level']:
                            var_root = root_new.createVariable(name, variable.datatype, variable.dimensions, fill_value=255, compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                        else:
                            var_root = root_new.createVariable(name, variable.datatype, variable.dimensions, fill_value=255, compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])
                    else:
                        if current_dtype == str:
                            var_root = root_new.createVariable(name, variable.datatype, variable.dimensions)
                        else:
                            if not self.run_details['compression_level']:
                                var_root = root_new.createVariable(name, variable.datatype, variable.dimensions, compression=self.run_details['compression_type'], shuffle=self.run_details['compression_shuffle'])
                            else:
                                var_root = root_new.createVariable(name, variable.datatype, variable.dimensions, compression=self.run_details['compression_type'], complevel=self.run_details['compression_level'], shuffle=self.run_details['compression_shuffle'])

                    # copy variable attributes
                    for att in root[name].ncattrs():
                        if att != '_FillValue':
                            if (name in ['reported_lower_limit_of_detection_per_measurement','reported_upper_limit_of_detection_per_measurement','reported_uncertainty_per_measurement','derived_uncertainty_per_measurement']) & (att == 'units'):
                                var_root.units = data_format_dict[name]['units']    
                            elif (name in ['measuring_instrument_reported_lower_limit_of_detection','measuring_instrument_documented_lower_limit_of_detection','measuring_instrument_reported_upper_limit_of_detection',
                                           'measuring_instrument_documented_upper_limit_of_detection','measuring_instrument_reported_uncertainty','measuring_instrument_documented_uncertainty','measuring_instrument_reported_accuracy',
                                           'measuring_instrument_documented_accuracy','measuring_instrument_reported_precision','measuring_instrument_documented_precision','measuring_instrument_reported_zero_drift',
                                           'measuring_instrument_documented_zero_drift','measuring_instrument_reported_span_drift','measuring_instrument_documented_span_drift','measuring_instrument_reported_zonal_drift',
                                           'measuring_instrument_documented_zonal_drift','measuring_instrument_reported_measurement_resolution','measuring_instrument_documented_measurement_resolution']) & (att == 'units'):
                                var_root.units = metadata_format_dict[name]['units']
                            elif (name in ['reported_lower_limit_of_detection_per_measurement','reported_upper_limit_of_detection_per_measurement','reported_uncertainty_per_measurement','derived_uncertainty_per_measurement']) & (att == 'description'):
                                var_root.description = data_format_dict[name]['description']
                            else:
                                var_root.setncattr(att, root[name].getncattr(att))

            #iterate through intersecting station references 
            for station_ii, intersecting_station_reference in enumerate(intersecting_station_references):

                #create dictionary for storing station data
                station_data = {}

                #get calculation parameters to iterate through 
                if intersecting_station_reference in calculation_parameters_station:
                    station_calculation_parameters = calculation_parameters_station[intersecting_station_reference]['ghost']
                    station_calculation_parameters_bsc = calculation_parameters_station[intersecting_station_reference]['bsc']
                else:   
                    station_calculation_parameters = copy.deepcopy(calculation_parameters)
                    station_calculation_parameters_bsc = copy.deepcopy(calculation_parameters_bsc)

                #iterate through all calculation parameters
                for calculation_parameter_ii, calculation_parameter_bsc in enumerate(station_calculation_parameters_bsc):

                    #load calculation parameter netCDF
                    root = Dataset('{}{}/{}/{}/{}/{}_{}.nc'.format(self.run_details['data_root'],network_write_dir,self.run_details['ghost_version'],temporal_resolution_vars['temporal_resolution_path'],calculation_parameter_bsc,calculation_parameter_bsc,self.yearmonth_str))

                    #get index for station
                    station_index = np.where(root['station_reference'][:] == intersecting_station_reference)[0]
                    if 'WETDEP' in self.run_details['parameter']:
                        try:
                            station_index = [b.split('_')[0] for b in root['station_reference'][:]].index(str(intersecting_station_reference)) # root['station_reference'][:] need to be cut by _
                        except:
                            station_index = -1

                    #only proceed if station is in file
                    continue_processing = False
                    if type(station_index) == int:  # WETDEP
                        if station_index > -1:
                            continue_processing = True
                    else:            # OTHER PRODUCTS
                        if len(station_index) > 0:
                            station_index = station_index[0]
                            continue_processing = True

                    if continue_processing == True:

                        #iterate through data / metadata variables
                        for data_varname, data_var in root.variables.items(): 
                            #get variable dtype and shape
                            data_var_dtype = data_var.dtype
                            data_var_shape = data_var.shape

                            #if first calculation parameter, create station specific array for saving data
                            if calculation_parameter_ii == 0:
                                
                                #set shape of new station array by removing station dimension and adding calculation parameters dimension
                                if data_varname == 'time':
                                    data_var_new_shape = (max_calculation_parameters,root.dimensions['time'].size)
                                elif data_varname == 'qa':
                                    data_var_new_shape = (max_calculation_parameters,root.dimensions['time'].size,root.dimensions['N_qa_codes'].size)
                                elif data_varname == 'flag':
                                    data_var_new_shape = (max_calculation_parameters,root.dimensions['time'].size,root.dimensions['N_flag_codes'].size)
                                elif len(data_var.dimensions) == 2:
                                    data_var_new_shape = (max_calculation_parameters,root.dimensions['time'].size)
                                elif len(data_var.dimensions) == 1:
                                    data_var_new_shape = (max_calculation_parameters)

                                #create array
                                if data_varname == calculation_parameter_bsc:  # parameters needed for calculation
                                    station_data[self.parameter_details['bsc_parameter_name']] = np.full(data_var_new_shape, np.NaN, dtype=data_var_dtype)
                                elif data_varname == '{}_prefiltered_defaultqa'.format(calculation_parameter_bsc):
                                    station_data['{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name'])] = np.full(data_var_new_shape, np.NaN, dtype=data_var_dtype)
                                elif data_var_dtype == str:
                                    station_data[data_varname] = np.full(data_var_new_shape, np.NaN, dtype=object)
                                else:
                                    station_data[data_varname] = np.full(data_var_new_shape, np.NaN, dtype=data_var_dtype)

                            #place calculation parameter data inside station array
                            if data_varname == 'time':
                                station_data[data_varname][calculation_parameter_ii,:] = root[data_varname][:]
                            elif data_varname == calculation_parameter_bsc: 
                                station_data[self.parameter_details['bsc_parameter_name']][calculation_parameter_ii,:] = root[data_varname][station_index,:]
                            elif data_varname == '{}_prefiltered_defaultqa'.format(calculation_parameter_bsc):
                                station_data['{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name'])][calculation_parameter_ii,:] = root[data_varname][station_index,:]
                            elif len(data_var.dimensions) == 3:
                                station_data[data_varname][calculation_parameter_ii,:,:] = root[data_varname][station_index,:,:]
                            elif len(data_var.dimensions) == 2:
                                station_data[data_varname][calculation_parameter_ii,:] = root[data_varname][station_index,:]
                            elif len(data_var.dimensions) == 1:
                                if data_var_dtype == str:
                                    station_data[data_varname][calculation_parameter_ii] = str(root[data_varname][station_index])
                                else:
                                    station_data[data_varname][calculation_parameter_ii] = root[data_varname][station_index]
                
                    #close netCDF
                    root.close()

                #calculate data product and handle other metadata / data variables
                #gather arguments for processing new product
                if product_function_name == 'calculate_aod':
                    new_wavelength = int(self.run_details['parameter'].split('_')[-1][:-2])
                    function_arguments = {'new_wavelength':new_wavelength, 'calculation_params':station_calculation_parameters, 'calculation_arr':station_data[self.parameter_details['bsc_parameter_name']]}
                else:
                    function_arguments = {'calculation_params':station_calculation_parameters, 'calculation_arr':station_data[self.parameter_details['bsc_parameter_name']], 'temporal_resolution_minutes': temporal_resolution_vars['minutes'], 'parameter_units': unit_convert_from_to}

                #calculate new product for station (filtered and unfiltered)
                root_new[self.parameter_details['bsc_parameter_name']][station_ii,:] = getattr(Data_Products, product_function_name)(**function_arguments)
                function_arguments['calculation_arr'] = station_data['{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name'])]
                root_new['{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name'])][station_ii,:] = getattr(Data_Products, product_function_name)(**function_arguments)

                #merge other data / metadata variables
                for data_varname, data_var in root_new.variables.items(): 

                    #do not process component variable (filtered and unfiltered)
                    if data_varname in [self.parameter_details['bsc_parameter_name'], '{}_prefiltered_defaultqa'.format(self.parameter_details['bsc_parameter_name'])]:
                        continue

                    #merge other data variables across calculation parameters
                    elif 'time' in data_var.dimensions:
                        #if time variable then just take first instance of variable
                        if data_varname == 'time':
                            root_new[data_varname][:] = station_data[data_varname][0,:]

                        #if local time / mean solar time then just take first instance of variable
                        elif data_varname in ['local_time','mean_solar_time']:
                            root_new[data_varname][station_ii,:] = station_data[data_varname][0,:]
    
                        # if variable is temporal classification then just take first instance of variable  
                        elif data_varname in ['day_night_code','weekday_weekend_code','season_code']:
                            root_new[data_varname][station_ii,:] = station_data[data_varname][0,:]

                        # if variable is reported or derived limits of detection or uncertainty, then drop it
                        elif data_varname in ['reported_lower_limit_of_detection_per_measurement','reported_upper_limit_of_detection_per_measurement','reported_uncertainty_per_measurement','derived_uncertainty_per_measurement']:
                            continue

                        # if variable is qa, then get all unique flags per measurement
                        # also add data product flag / insufficient data for calculating data product flags
                        elif data_varname == 'qa':
                            new_qa = np.full((station_data[data_varname].shape[1], station_data[data_varname].shape[2]), 255, dtype=np.uint8)
                            #loop through all times
                            for time_ii in range(root_new.dimensions['time'].size):
                                unique_qa = np.unique(station_data[data_varname][:,time_ii,:])
                                new_qa[time_ii, :len(unique_qa)] = unique_qa
                            new_qa[:, -2] = product_code
                            nan_obs = np.isnan(root_new[self.parameter_details['bsc_parameter_name']][station_ii,:])
                            new_qa[nan_obs, -1] = insufficient_code
                            root_new[data_varname][station_ii,:,:] = np.sort(new_qa, axis=-1)

                        # if variable is flag, then get all unique flags per measurement
                        elif data_varname == 'flag':
                            new_flag = np.full((station_data[data_varname].shape[1], station_data[data_varname].shape[2]), 255, dtype=np.uint8)
                            #loop through all times
                            for time_ii in range(root_new.dimensions['time'].size):
                                unique_flag = np.unique(station_data[data_varname][:,time_ii,:])
                                new_flag[time_ii, :len(unique_flag)] = unique_flag
                            root_new[data_varname][station_ii,:,:] = np.sort(new_flag, axis=-1)

                        # if variable is flag_simple, then drop and recalculate based on new qa / flag
                        elif data_varname == 'flag_simple':
                            #loop through all times
                            for time_ii in range(root_new.dimensions['time'].size):

                                #1. unchecked data determined if have un-flagged data flag or preliminary data flag
                                if (unflagged_flag in root_new['flag'][station_ii,time_ii,:]) or (preliminary_flag in root_new['qa'][station_ii,time_ii,:]):
                                    root_new[data_varname][station_ii,time_ii] = unchecked_simple_flag

                                #2. missing data determined by missing measurement data flag
                                elif missing_flag in root_new['qa'][station_ii,time_ii,:]:
                                    root_new[data_varname][station_ii,time_ii] = missing_simple_flag

                                #3. estimate data determined by any of estimated data flags
                                elif np.isin(root_new['flag'][station_ii,time_ii,:], estimated_flags).any():
                                    root_new[data_varname][station_ii,time_ii] = estimate_simple_flag

                                #4. suspect data determined by any of suspect data flags + GHOST invalid QA
                                elif (np.isin(root_new['flag'][station_ii,time_ii,:], suspect_flags).any()) & (ghost_invalid_qa in root_new['qa'][station_ii,time_ii,:]):
                                    root_new[data_varname][station_ii,time_ii] = suspect_simple_flag

                                #5. poor data determined by GHOST invalid QA
                                elif ghost_invalid_qa in root_new['qa'][station_ii,time_ii,:]:
                                    root_new[data_varname][station_ii,time_ii] = poor_simple_flag

                                #6. otherwise data is considered good
                                else:
                                    root_new[data_varname][station_ii,time_ii] = good_simple_flag  

                        # if variable is representativity, then take average
                        elif data_varname in ['hourly_native_representativity_percent','daily_native_representativity_percent','monthly_native_representativity_percent','annual_native_representativity_percent',
                                              'hourly_native_max_gap_percent','daily_native_max_gap_percent','monthly_native_max_gap_percent','annual_native_max_gap_percent']:
                            root_new[data_varname][station_ii,:] = np.average(station_data[data_varname][:,:], axis=0)

                    #merge metadata variables across calculation parameters, by taking first instance
                    #do not copy measurement variables that are specific to each calculation parameter 
                    else:
                        if data_varname not in ['measuring_instrument_reported_lower_limit_of_detection','measuring_instrument_documented_lower_limit_of_detection','measuring_instrument_reported_upper_limit_of_detection',
                                                'measuring_instrument_documented_upper_limit_of_detection','measuring_instrument_reported_uncertainty','measuring_instrument_documented_uncertainty',
                                                'measuring_instrument_reported_accuracy','measuring_instrument_documented_accuracy','measuring_instrument_reported_precision','measuring_instrument_documented_precision',
                                                'measuring_instrument_reported_zero_drift','measuring_instrument_documented_zero_drift','measuring_instrument_reported_span_drift','measuring_instrument_documented_span_drift',
                                                'measuring_instrument_reported_zonal_drift','measuring_instrument_documented_zonal_drift','measuring_instrument_reported_measurement_resolution','measuring_instrument_documented_measurement_resolution']:
                            root_new[data_varname][station_ii] = station_data[data_varname][0]

            #close new netCDF
            root_new.close()

            #apply compression (-L1 = deflation level 1), and do not write operation history (-h)
            #and set to original name
            submit_list = ['ncks', '-h', '-O', '-L', '1', netCDF_fname, netCDF_fname]
            record_process = subprocess.Popen(submit_list, stdout=subprocess.PIPE)
            record_status = record_process.communicate()[0]
            record_return_code = record_process.returncode

            #give 770 permissions for file and make owner bsc32
            set_file_permissions_ownership(netCDF_fname)

            #return valid process logfile (0)
            self.run_details['log_file_str'] += str((time.time() - self.run_details['stage_process_start_time'])/60.)
            create_GHOST_stage_logfile(0,self.run_details)