import copy
import os
import shutil
import sys
import time
import subprocess

import numpy as np
import pandas as pd

from crossnetwork_processing_functions import (get_configuration_variables, default_parameters_per_network, 
                                               dependent_parameters_per_stage, ordered_stages, 
                                               get_default_temporal_resolutions_per_matrix, 
                                               check_GHOST_pipeline_outcome, networks_to_merge)
from GHOST_standards import standard_parameters, standard_temporal_resolutions
from submit_greasy import SubmitGreasy

#start timer on GHOST processing
start_GHOST_time = time.time()

#get slurm job ID --> passed from job submittal script
slurm_job_ID = sys.argv[1]
    
#get configuration variables
run_details, _, _ = get_configuration_variables(slurm_job_ID, startup=True)

#make a copy of the configuration file indexed by the slurm job ID
shutil.copy2(run_details['process_root']+'/run/GHOST_configuration.py',run_details['process_root']+'/run/GHOST_configurations/configuration_{}.py'.format(slurm_job_ID))

#create variable to track submit iterations
submit_iteration = 0

#get yearmonth strs between start date and end date
yearmonth_strs = [date.strftime("%Y%m") for date in pd.date_range('{}-{}'.format(run_details['start_date'][:4], 
                                                                                 run_details['start_date'][4:6]), 
                                                                                 '{}-{}'.format(run_details['end_date'][:4], 
                                                                                 run_details['end_date'][4:6]), 
                                                                                 freq='MS', inclusive='left')]

#create a dictionary of GHOST matrices and associated parameters (from standard_parameters)
matrices = {}
for param in standard_parameters:
    matrix = standard_parameters[param]['matrix']
    if matrix not in matrices:
        matrices[matrix] = []
    matrices[matrix].append(param)

#create a dictionary of BSC parameter names and associated GHOST parameter names 
BSC_parameters = {}
for param in standard_parameters:
    BSC_parameters[standard_parameters[param]['bsc_parameter_name']] = param

#set valid stage pipelines
valid_pipelines = [['S1','S2','S3','T1','T2'], ['P1','E1','C1'], ['M1','C1']]

#set lists for types of failure pipeline strings
pipeline_fail_no_data = ['NOT PROCESSED AS HAVE NO', 'BUT 0 STATIONS REMAIN']
pipeline_fail_error = ['WITH CAUGHT ERROR', 'UNCAUGHT ERROR', 'DID NOT FINISH DUE TO UNFINISHED JOB']

#iterate through submitting pipelines until have nothing left to process
while True:

    #if first iteration, then gather arguments for pipelines
    if submit_iteration == 0:

        #initialise list to store arguments to submit pipelines
        arguments = []

        # create dict to store current stage of pipeline per network and parameter, and if pipeline is stil alive or not 
        pipeline_status = {}

        # iterate through each defined GHOST stage and populate jobs to submit in pipeline_status
        for stage_ii, stage in enumerate(run_details['stages_to_process']):
            
            #define networks to process for stage
            
            #for EBAS split stage, set the network to be just EBAS (if EBAS set in defined networks that is)
            if stage == 'E1':
                if 'EBAS' in run_details['networks']:
                    networks = ['EBAS']
                else:
                    networks = []
            #for merge stage, the networks are the defined merge networks
            elif stage == 'M1':
                networks = run_details['merge_networks']
            #for compress / zip stages, the networks are the reporting networks + individual EBAS networks (if E1 set),
            #and merge networks (if M1 set)
            #otherwise they are just the reporting networks
            elif stage in ['C1','Z1']:
                networks = run_details['compress_networks']
            #for other stages, the networks are the invidual reporting networks
            else:
                networks = run_details['networks']

            #iterate through networks and add information about stage to pipeline_status
            for network in networks:

                #add network to pipeline_status if not yet added
                if network not in pipeline_status:
                    pipeline_status[network] = {}

                #get standard parameter and product defaults
                #if network is a merge network, get unique defaults across all relevant networks 
                if network in run_details['merge_networks']:
                    merge_networks = networks_to_merge(network)
                    parameter_defaults_standard = []
                    parameter_defaults_product = []
                    for merge_network in merge_networks:
                        if 'EBAS' in merge_network:
                            merge_network = 'EBAS'
                        elif 'NOAA_ISD' in merge_network:
                            merge_network = 'NOAA_ISD'
                        parameter_defaults_standard.extend(default_parameters_per_network[merge_network])
                        parameter_defaults_product.extend([param for param in dependent_parameters_per_stage['P1'] 
                                                           if merge_network in dependent_parameters_per_stage['P1'][param]['networks']])
                    parameter_defaults_standard = list(np.unique(parameter_defaults_standard))
                    parameter_defaults_product = list(np.unique(parameter_defaults_product))

                #otherwise, get default parameters for the specific network 
                else:
                    if 'EBAS' in network:
                        network_to_retrieve = 'EBAS'
                    elif 'NOAA_ISD' in network:
                        network_to_retrieve = 'NOAA_ISD'
                    else:
                        network_to_retrieve = copy.deepcopy(network)
                    parameter_defaults_standard = default_parameters_per_network[network_to_retrieve]
                    parameter_defaults_product = [param for param in dependent_parameters_per_stage['P1'] 
                                                  if network_to_retrieve in dependent_parameters_per_stage['P1'][param]['networks']]

                #set parameters to process for stage and network
                #set parameter defaults per stage
                if stage in ['S1','S2','S3','T1','T2']:
                    parameter_defaults = copy.deepcopy(parameter_defaults_standard)
                elif stage == 'P1':
                    parameter_defaults = copy.deepcopy(parameter_defaults_product)
                elif stage in ['E1','M1','C1']:
                    parameter_defaults = list(np.unique(parameter_defaults_standard + parameter_defaults_product))
                elif stage == 'Z1':
                    parameter_defaults = ['ZIP']
                    
                #if parameters set to be defaults, set parameters to be all defaults
                if run_details['parameters'] == 'default':
                    parameters = copy.deepcopy(parameter_defaults)
                #otherwise take set parameters
                #if any parameters are matrices, then convert them to the associated parameters
                else:
                    parameters = copy.deepcopy(run_details['parameters'])

                    #if the stage is Z1 force the parameters to be 'ZIP'
                    if stage == 'Z1':
                        parameters = ['ZIP']

                    # convert matrices to the associated parameters
                    parameters_to_delete = []
                    parameters_to_add = []
                    for parameter in parameters:
                        if parameter in matrices:
                            parameters_to_delete.append(parameter)
                            parameters_to_add.extend(matrices[parameter])
                    if len(parameters_to_delete) > 0:
                        parameters = [param for param in parameters if param not in parameters_to_delete]
                    if len(parameters_to_add) > 0:
                        parameters_to_add = [param for param in parameters_to_add if param not in parameters]
                        parameters.extend(parameters_to_add)

                    #convert BSC name parameters to GHOST names
                    parameters = [BSC_parameters[param] if param in BSC_parameters else param for param in parameters]
                        
                #iterate through parameters, and add information about stage to pipeline_status
                for parameter in parameters:

                    #ensure parameter is can actually be processed in current stage (i.e. in stage parameter defaults)
                    #if not, continue to next parameter
                    if parameter not in parameter_defaults:
                        continue

                    #work out if stage has other parameter dependencies
                    dependent_stage = False
                    #check if need to wait for any dependent parameters to process 
                    if stage in dependent_parameters_per_stage:
                        if parameter in dependent_parameters_per_stage[stage]:
                            if 'networks' in dependent_parameters_per_stage[stage][parameter]:
                                if network in dependent_parameters_per_stage[stage][parameter]['networks']:
                                    dependent_stage = True
                            else:
                                dependent_stage = True

                    #work out status of current stage
                    #if parameter not yet in pipeline_status dict then it is the first valid stage, so is alive
                    #but if stage has a dependency put it on hold
                    #as well as parameter dependencies, there are network dependencies where need to wait for other 
                    #networks to finish before starting
                    join_stages = False
                    if parameter not in pipeline_status[network]:
                        if dependent_stage:
                            status = 'hold'
                        else:
                            #for merge stage if have E1 stage or before, then put it on hold
                            if (stage == 'M1') & (np.any([stage in ['S1','S2','S3','T1','T2','P1','E1'] for stage in run_details['stages_to_process']])):
                                status = 'hold'
                            #for compress stage, if network is an EBAS split network, and E1 is set to be processed, and EBAS is defined as a network, then put it on hold
                            elif (stage == 'C1') & (network in run_details['EBAS_networks']) & ('E1' in run_details['stages_to_process']) & ('EBAS' in run_details['networks']):
                                status = 'hold'
                            #for zip stage if have C1 stage or before, then put it on hold
                            elif (stage == 'Z1') & (np.any([stage in ['S1','S2','S3','T1','T2','P1','E1','M1','C1'] for stage in run_details['stages_to_process']])):
                                status = 'hold'
                            else: 
                                status = 'alive'
                    #otherwise, work out if stage can be processed in same pipeline as previous stage
                    #iterate through valid stage pipelines, and if both current stage and previous stage, are
                    #in a valid pipeline then the stages can be processed together, if the current stage does 
                    #not have dependencies 
                    else:
                        #get previous stage
                        if len(pipeline_status[network][parameter]['stages_remaining']) > 0:
                            previous_stage = pipeline_status[network][parameter]['stages_remaining'][-1].split(',')[-1]
                        else:
                            previous_stage = pipeline_status[network][parameter]['current_stages'].split(',')[-1]

                        #work out if to join stage to previous stage in a pipeline
                        for valid_pipeline in valid_pipelines:
                            if (stage in valid_pipeline) & (previous_stage in valid_pipeline):
                                if dependent_stage:
                                    join_stages = False
                                else:
                                    join_stages = True
                                break

                    #now set information about stage to pipeline_status
                    #need to set current stage in a new standalone pipeline?
                    if not join_stages:
                        #parameter not yet set in pipeline_status?
                        if parameter not in pipeline_status[network]:
                            #create dictionary in pipeline_status for network/parameter
                            pipeline_status[network][parameter] = {'current_stages':stage, 'stages_remaining':[], 
                                                                   'processed_stages':[], 
                                                                   'stages_with_dependencies':[], 'status':status, 
                                                                   'pipeline_start_time':0}
                        #otherwise add stage to stages remaining
                        else:
                            #append stage to stages_remaining
                            pipeline_status[network][parameter]['stages_remaining'].append(stage)

                    
                    #need to join current stage to previous stage/s in a pipeline?
                    else:
                        #have stages set in stages_remaining?
                        if len(pipeline_status[network][parameter]['stages_remaining']) > 0:
                            #connect stage with previous stage/s in stages_remaining
                            pipeline_status[network][parameter]['stages_remaining'][-1] = '{},{}'.format(pipeline_status[network][parameter]['stages_remaining'][-1],
                                                                                                            stage)
                        #otherwise, just have stages set in current_stages?
                        else:
                            #connect stage with previous stage/s in current_stages
                            pipeline_status[network][parameter]['current_stages'] = '{},{}'.format(pipeline_status[network][parameter]['current_stages'],
                                                                                                    stage)
                    # if stage has dependency, append it to stages_with_dependencies
                    if dependent_stage:
                        pipeline_status[network][parameter]['stages_with_dependencies'].append(stage)

        #--------------------------------------------------------------------------------------------#
        #--------------------------------------------------------------------------------------------#

        #now have pipeline_status populated with jobs
        #iterate through pipeline status dict, and create arguments to submit pipelines which are alive

        #iterate through networks
        for network in list(pipeline_status.keys()):

            #iterate through parameters  
            for parameter in list(pipeline_status[network].keys()):

                # get parameter details (if parameter is not 'ZIP')
                if parameter != 'ZIP':
                    parameter_details = standard_parameters[parameter]

                    #get temporal resolutions
                    if run_details['temporal_resolutions'] == 'default':
                        temporal_resolution_codes = get_default_temporal_resolutions_per_matrix(parameter_details['matrix'])
                    else:
                        temporal_resolution_codes = copy.deepcopy(run_details['temporal_resolutions'])
                    temporal_resolutions = [standard_temporal_resolutions[temporal_resolution_code]['temporal_resolution_path'] for temporal_resolution_code in temporal_resolution_codes] 

                #set variable to determine if can process parameter
                can_process_parameter = True

                #determine if the current parameter has any dependencies to be processed first, if so put on hold
                #if no dependencies are queued, and no files exist for them, then write logfile for parameter informing that it cannot be processed    
                for stage_with_dependency in list(pipeline_status[network][parameter]['stages_with_dependencies']):

                    # if a previous stage was determined to lack dependencies, break out of loop
                    if not can_process_parameter:
                        break

                    #get parameter dependencies
                    parameter_dependencies = dependent_parameters_per_stage[stage_with_dependency][parameter]['components']

                    #set variable to count how many dependent parameters have existing files and are not to to be processed in pipeline
                    parameter_exist_files = 0

                    #iterate through dependent parameters
                    for parameter_dependency in parameter_dependencies:

                        #if parameter dependency not set to be processed, check if previously processed files exist for current date range 
                        if parameter_dependency not in pipeline_status[network]:

                            # get dependent parameter details
                            parameter_dependency_details = standard_parameters[parameter_dependency]

                            #check if dependent parameter has been previously processed and exist for current date range
                            exist_files = False

                            #check if file exist for yearmonth and temporal resolution
                            #iterate through yearmonths
                            for yearmonth_str in yearmonth_strs:
                                if exist_files:
                                    break
                                #iterate through temporal resolutions
                                for temporal_resolution in temporal_resolutions:
                                    #check if file exists
                                    if os.path.isfile(run_details['data_root']+'{}/{}/{}/{}/{}_{}.nc'.format(network,run_details['ghost_version'],temporal_resolution,parameter_dependency_details['bsc_parameter_name'],parameter_dependency_details['bsc_parameter_name'],yearmonth_str)):
                                        exist_files = True
                                        break

                            #if all neccessary files exist then iterate parameter_exist_files by 1
                            if exist_files:
                                parameter_exist_files += 1
                            #if not all neccessary files exist, then do not submit pipeline for parameter, and set error for logfile when writing
                            else:
                                pipeline_status[network][parameter]['status'] = 'nodata'
                                pipeline_error = 'PIPELINE CANNOT BE PROCESSED AS DEPENDENT PARAMETERS NOT PROCESSED FOR {}.\nTERMINATING PIPELINE HERE'.format(stage_with_dependency)
                                can_process_parameter = False
                                break

                    #if all stage dependencies have existing files and are not to be processed in the pipeline, 
                    #remove dependency for stage, as no longer need to wait for other stages to finish,
                    #join together the pipeline the stage is in with the previous pipeline
                    #also, if stage is in current stages, set pipeline to alive
                    if parameter_exist_files == len(parameter_dependencies):
                        #remove dependency for stage, as no longer need to wait for other stages to finish
                        pipeline_status[network][parameter]['stages_with_dependencies'].remove(stage_with_dependency)
                        #iterate through remaining stage pipelines, if dependent stage is in one of them, then join it
                        #with the previous pipeline as can be processed togetehr 
                        for stage_ii, remaining_stage_pipeline in enumerate(pipeline_status[network][parameter]['stages_remaining']):
                            #dependent stage in remaining stage pipeline?
                            if stage_with_dependency in remaining_stage_pipeline:
                                #if first remaining stage pipeline, join it to current_stages
                                if stage_ii == 0:
                                    pipeline_status[network][parameter]['current_stages'] = '{},{}'.format(pipeline_status[network][parameter]['current_stages'], remaining_stage_pipeline)
                                #otherwise, join current remaining stage pipeline to previous one
                                else:
                                    pipeline_status[network][parameter]['stages_remaining'][stage_ii-1] = '{},{}'.format(pipeline_status[network][parameter]['stages_remaining'][stage_ii-1], remaining_stage_pipeline)                                
                                #modify remaining stage pipelines to not remove current one
                                pipeline_status[network][parameter]['stages_remaining'].remove(remaining_stage_pipeline)
                                break

                        #if stage with dependency is in current stages, set pipeline to alive
                        if stage_with_dependency in pipeline_status[network][parameter]['current_stages']:
                            pipeline_status[network][parameter]['status'] = 'alive'

                #check if first stage to process for specific stages have neccessary files for processing to begin

                #get first stage to process
                first_stage = pipeline_status[network][parameter]['current_stages'].split(',')[0]

                #first stage is E1, M1, C1 or Z1, and current status for network / parameter is 'alive' 
                if (first_stage in ['E1','M1','C1','Z1']) and (pipeline_status[network][parameter]['status'] == 'alive'): 
                    #set variable to see if files exist or not for stage
                    exist_files = False

                    #for E1/C1, check if files exist for yearmonth and temporal resolution
                    if first_stage in ['E1','C1']:
                        #iterate through yearmonths
                        for yearmonth_str in yearmonth_strs:
                            if exist_files:
                                break
                            #iterate through temporal resolutions
                            for temporal_resolution in temporal_resolutions:
                                #check if file exists
                                if os.path.isfile(run_details['data_root']+'{}/{}/{}/{}/{}_{}.nc'.format(network,run_details['ghost_version'],temporal_resolution,parameter_details['bsc_parameter_name'],parameter_details['bsc_parameter_name'],yearmonth_str)):
                                    exist_files = True
                                    break
                    
                    #for M1, check if any of the networks to merge have any data
                    elif first_stage == 'M1':
                        #get networks to merge
                        merge_networks = networks_to_merge(network)
                        #iterate through networks to merge
                        for merge_network in merge_networks:
                            if exist_files:
                                break
                            for yearmonth_str in yearmonth_strs:
                                if exist_files:
                                    break
                                #iterate through temporal resolutions
                                for temporal_resolution in temporal_resolutions:
                                    #check if file exists
                                    if os.path.isfile(run_details['data_root']+'{}/{}/{}/{}/{}_{}.nc'.format(merge_network,run_details['ghost_version'],temporal_resolution,parameter_details['bsc_parameter_name'],parameter_details['bsc_parameter_name'],yearmonth_str)):
                                        exist_files = True
                                        break

                    #for Z1, check if network directory exists to compress 
                    elif first_stage == 'Z1':
                        if os.path.isdir('{}GHOST_v{}/{}'.format(run_details['compress_root'],run_details['ghost_version'],network)):
                            exist_files = True

                    #if not all neccessary files exist, then do not submit pipeline for parameter, and set error for logfile when writing
                    if not exist_files:
                        #if first stage is C1, and network is split EBAS network, then do not make pipeline logfile
                        if (first_stage == 'C1') & (network in run_details['EBAS_networks']):
                            del pipeline_status[network][parameter]
                            continue
                        #elif first stage is Z1, and network is zip stage, then do not make pipeline logfile
                        elif first_stage == 'Z1':
                            del pipeline_status[network][parameter]
                            continue
                        #otherwise set pipeline error
                        else:
                            pipeline_status[network][parameter]['status'] = 'nodata'
                            pipeline_error = 'PIPELINE CANNOT BE PROCESSED AS FILES DO NOT EXIST TO PROCESS FIRST STAGE {}.\nTERMINATING PIPELINE HERE'.format(first_stage)

                #proceed if pipeline status for network/parameter is alive, killed or nodata
                if pipeline_status[network][parameter]['status'] in ['alive','killed','nodata']:

                    #create pipeline_logfile
                    pipeline_logfile = open('{}/run/pipeline_logs/{}_{}.out'.format(run_details['process_root'],network,parameter),"w")
                    pipeline_logfile.write('**************************************************************************************************************\n')        
                    pipeline_logfile.write('BEGINNING PROCESSING OF STAGES FOR {}:{}\n'.format(network,parameter))
                    pipeline_logfile.write('**************************************************************************************************************\n')
                    pipeline_logfile.close()

                    #add argument for pipeline if it is alive
                    if pipeline_status[network][parameter]['status'] == 'alive':

                        #add start time to pipeline_status if starting pipeline
                        if len(pipeline_status[network][parameter]['processed_stages']) == 0:
                            pipeline_status[network][parameter]['pipeline_start_time'] = '{:.0f}'.format(time.time())

                        #add argument
                        arguments.append([slurm_job_ID, network, parameter, 
                                        pipeline_status[network][parameter]['current_stages'], 
                                        pipeline_status[network][parameter]['pipeline_start_time']])

                    #if pipeline has been killed or nodata, then write logfile error
                    elif pipeline_status[network][parameter]['status'] in ['killed','nodata']:

                        #write pipeline logfile error
                        pipeline_logfile = open('{}/run/pipeline_logs/{}_{}.out'.format(run_details['process_root'],network,parameter),"a")
                        pipeline_logfile.write(pipeline_error)
                        pipeline_logfile.close()

                        #set current stages to be empty
                        pipeline_status[network][parameter]['current_stages'] = [] 

    print(submit_iteration, pipeline_status)
    print(arguments)
    print()

    #if no more pipelines to submit, break out of loop
    if len(arguments) == 0:
        break

    #submit pipeline for jobs (and monitor outcomes)
    SubmitGreasy('GPipe_{}_{}'.format(slurm_job_ID,submit_iteration), 'pipeline', arguments, run_details)

    #--------------------------------------------------------------------------------------------#
    #--------------------------------------------------------------------------------------------#

    #now determine if submitted pipelines were successful, and gather arguments to submit new pipelines

    #iterate through arguments and check if pipeline did not finish correctly
    for arg_ii, (pipeline_arguments, network, parameter, stages_to_process, pipeline_start_time) in enumerate(arguments):

        #get pipeline logfile name
        pipeline_logfile = '{}/run/pipeline_logs/{}_{}.out'.format(run_details['process_root'],network,parameter)

        #get second last line of output
        line = subprocess.check_output(['tail', '-2', pipeline_logfile], encoding='utf8').strip()

        #if pipeline did not finish correctly then do not submit more jobs for that pipeline
        #if job ended because of failure then set job status as 'killed' 
        for fail_str in pipeline_fail_error:
            if fail_str in line:
                pipeline_status[network][parameter]['status'] = 'killed'
                pipeline_status[network][parameter]['current_stages'] = [] 
                break
        #if job ended because of no data then set job status as 'nodata' 
        for fail_str in pipeline_fail_no_data:
            if fail_str in line:
                pipeline_status[network][parameter]['status'] = 'nodata'
                pipeline_status[network][parameter]['current_stages'] = [] 
                break
    
    #iterate through pipeline status dict, and create arguments to submit next set of pipelines
    arguments = []

    #iterate through networks
    for network in list(pipeline_status.keys()):

        #iterate through parameters
        for parameter in list(pipeline_status[network].keys()):

            # get parameter details (if parameter is not 'ZIP')
            if parameter != 'ZIP':
                parameter_details = standard_parameters[parameter]

                #get temporal resolutions
                if run_details['temporal_resolutions'] == 'default':
                    temporal_resolution_codes = get_default_temporal_resolutions_per_matrix(parameter_details['matrix'])
                else:
                    temporal_resolution_codes = copy.deepcopy(run_details['temporal_resolutions'])
                temporal_resolutions = [standard_temporal_resolutions[temporal_resolution_code]['temporal_resolution_path'] for temporal_resolution_code in temporal_resolution_codes] 

            #move all alive pipelines to next stages

            #if pipeline is alive then check if current stage/s finished correctly 
            if pipeline_status[network][parameter]['status'] == 'alive':

                #add current stages to processed_stages
                pipeline_status[network][parameter]['processed_stages'].append(pipeline_status[network][parameter]['current_stages'])

                #check if have stages remaining, if so, then update current_stages and stages_remaining
                if len(pipeline_status[network][parameter]['stages_remaining']) > 0:
                    pipeline_status[network][parameter]['current_stages'] = pipeline_status[network][parameter]['stages_remaining'][0] 
                    pipeline_status[network][parameter]['stages_remaining'] = pipeline_status[network][parameter]['stages_remaining'][1:]
                #else, if have no stages remaining, finalise pipeline logfile, and continue to next parameter
                else:
                    pipeline_status[network][parameter]['current_stages'] = [] 
                    pipeline_status[network][parameter]['stages_remaining'] = []
                    pipeline_status[network][parameter]['status'] = 'finished'
                    total_pipeline_time = (time.time() - float(pipeline_status[network][parameter]['pipeline_start_time']))/60.
                    pipeline_logfile = open('{}/run/pipeline_logs/{}_{}.out'.format(run_details['process_root'],network,parameter),"a")
                    pipeline_logfile.write('\n') 
                    pipeline_logfile.write('PROCESSING OF ALL DESIRED STAGES FOR {}:{} COMPLETE IN {:.2f} MINUTES'.format(network,parameter,total_pipeline_time))
                    pipeline_logfile.close()
                    continue

            #check if on hold pipelines can be started
            elif pipeline_status[network][parameter]['status'] == 'hold':

                #set variable to determine if ready to start pipeline
                ready_to_start = True

                #if job is on hold because of dependencies then check if dependent components have now been processed
                stages_waiting_for = [stage for stage in pipeline_status[network][parameter]['stages_with_dependencies'] if stage in pipeline_status[network][parameter]['current_stages']]
                if stages_waiting_for:
                    for stage_waiting_for in stages_waiting_for:

                        #if any previous stage has been found not able to be processed, then break out of loop
                        if not ready_to_start:
                            break

                        #get parameter dependencies
                        parameter_dependencies = dependent_parameters_per_stage[stage_with_dependency][parameter]['components']

                        #iterate through dependent parameters
                        for parameter_dependency in parameter_dependencies:
                            
                            #if dependent parameter set to be processed, check if pipeline finished or was killed
                            if parameter_dependency in pipeline_status[network]:

                                #check if dependent pipeline was killed early or has nodata, if so, terminate pipeline logfile
                                if pipeline_status[network][parameter_dependency]['status'] in ['killed','nodata']:
                                    pipeline_status[network][parameter]['status'] = copy.deepcopy(pipeline_status[network][parameter_dependency]['status'])
                                    pipeline_error = '{} CANNOT BE NOT PROCESSED AS DEPENDENT PARAMETERS DID NOT FINISH PROCESSING CORRECTLY.\nTERMINATING PIPELINE HERE'.format(stage_with_dependency)
                                    ready_to_start = False
                                    break

                                #elif dependent pipeline is still running processing / product stages then cannot start pipeline yet
                                elif np.any([stage in pipeline_status[network][parameter_dependency]['current_stages'] for stage in ['S1','S2','S3','T1','T2','P1']]):
                                    ready_to_start = False
                                    break
                    
                #for EBAS split stage, check if have existing files to start stage
                elif pipeline_status[network][parameter]['current_stages'] == 'E1':

                    #set variable to see if files exist or not for stage
                    exist_files = False

                    #check if file exist for yearmonth and temporal resolution
                    #iterate through yearmonths
                    for yearmonth_str in yearmonth_strs:
                        if exist_files:
                            break
                        #iterate through temporal resolutions
                        for temporal_resolution in temporal_resolutions:
                            #check if file exists
                            if os.path.isfile(run_details['data_root']+'{}/{}/{}/{}/{}_{}.nc'.format(network,run_details['ghost_version'],temporal_resolution,parameter_details['bsc_parameter_name'],parameter_details['bsc_parameter_name'],yearmonth_str)):
                                exist_files = True
                                break
                                            
                    #if do not exist files, then cannot process EBAS split stage. Therefore terminate pipeline logfile
                    if not exist_files:
                        pipeline_status[network][parameter]['status'] = 'nodata'
                        pipeline_error = 'E1 CANNOT BE PROCESSED AS FILES DO NOT EXIST TO PROCESS STAGE.\nTERMINATING PIPELINE HERE'
                        ready_to_start = False

                #for merge stage, check if all network pipelines for parameter have proceeded past E1, and have existing files to start stage
                elif pipeline_status[network][parameter]['current_stages'] == 'M1':

                    #set variable to see if files exist or not for stage
                    exist_files = False

                    #get networks to merge
                    merge_networks = networks_to_merge(network)

                    #iterate through networks to merge
                    for merge_network in merge_networks:
                    
                        # ensure network/parameter pair are being processed
                        if merge_network in pipeline_status:
                            if parameter in pipeline_status[merge_network]:

                                #if dependent pipeline has been killed, then cannot process merge stage. Therefore terminate pipeline logfile
                                if pipeline_status[merge_network][parameter]['status'] == 'killed':
                                    pipeline_status[network][parameter]['status'] = 'killed'
                                    pipeline_error = 'M1 CANNOT BE PROCESSED AS AT LEAST 1 NETWORK PROCESSING PIPELINE FAILED.\nTERMINATING PIPELINE HERE'    
                                    ready_to_start = False
                                    break

                                #elif pipeline is not past P1 then continue to hold on processing merge stage 
                                elif np.any([stage in pipeline_status[merge_network][parameter]['current_stages'] for stage in ['S1','S2','S3','T1','T2','P1','E1']]):
                                    ready_to_start = False
                                    break

                                #else, check actually have files to merge
                                elif not exist_files:

                                    #check if file exists for yearmonth and temporal resolution
                                    #iterate through yearmonths
                                    for yearmonth_str in yearmonth_strs:
                                        if exist_files:
                                            break
                                        #iterate through temporal resolutions
                                        for temporal_resolution in temporal_resolutions:
                                            #check if file exists
                                            if os.path.isfile(run_details['data_root']+'{}/{}/{}/{}/{}_{}.nc'.format(merge_network,run_details['ghost_version'],temporal_resolution,parameter_details['bsc_parameter_name'],parameter_details['bsc_parameter_name'],yearmonth_str)):
                                                exist_files = True
                                                break

                    #if do not exist files, then cannot process merge stage. Therefore terminate pipeline logfile
                    if (ready_to_start) & (not exist_files):
                        pipeline_status[network][parameter]['status'] = 'nodata'
                        pipeline_error = 'M1 CANNOT BE PROCESSED AS FILES DO NOT EXIST TO PROCESS STAGE.\nTERMINATING PIPELINE HERE' 
                        ready_to_start = False

                #for compress stage, check if have existing files to start stage, and if network is an EBAS split network, 
                #check if relevant EBAS pipeline for parameter has proceeded past E1
                elif pipeline_status[network][parameter]['current_stages'] == 'C1':

                    #for split EBAS networks, if dependent EBAS pipeline has been killed or has nodata, then cannot process compress stage. Therefore terminate pipeline logfile
                    if network in run_details['EBAS_networks']:
                        if pipeline_status['EBAS'][parameter]['status'] in ['killed','nodata']:
                            pipeline_status[network][parameter]['status'] = copy.deepcopy(pipeline_status['EBAS'][parameter]['status'])
                            pipeline_error = 'C1 CANNOT BE PROCESSED AS AT LEAST 1 NETWORK PROCESSING PIPELINE FAILED.\nTERMINATING PIPELINE HERE'   
                            ready_to_start = False

                        #elif pipeline is not past E1 then continue to hold on processing zip stage 
                        elif np.any([stage in pipeline_status['EBAS'][parameter]['current_stages'] for stage in ['S1','S2','S3','T1','T2','P1','E1']]):
                            ready_to_start = False

                    #if ready to start, then check files exist for parameter to be processed.
                    if ready_to_start:
                        #set variable to see if files exist or not for stage
                        exist_files = False

                        #check if file exist for yearmonth and temporal resolution
                        #iterate through yearmonths
                        for yearmonth_str in yearmonth_strs:
                            if exist_files:
                                break
                            #iterate through temporal resolutions
                            for temporal_resolution in temporal_resolutions:
                                #check if file exists
                                if os.path.isfile(run_details['data_root']+'{}/{}/{}/{}/{}_{}.nc'.format(network,run_details['ghost_version'],temporal_resolution,parameter_details['bsc_parameter_name'],parameter_details['bsc_parameter_name'],yearmonth_str)):
                                    exist_files = True
                                    break

                        #if do not exist files, and network is EBAS split network, then do not make pipeline logfile
                        #otherwise, terminate pipeline logfile
                        if not exist_files:
                            if network in run_details['EBAS_networks']:
                                del pipeline_status[network][parameter]
                                continue
                            else:
                                pipeline_status[network][parameter]['status'] = 'nodata'
                                pipeline_error = 'C1 CANNOT BE PROCESSED AS FILES DO NOT EXIST TO PROCESS STAGE.\nTERMINATING PIPELINE HERE' 
                                ready_to_start = False 

                #for zip stage, check if all parameter pipelines for network have proceeded past C1, and check if network directory exists to compress 
                elif pipeline_status[network][parameter]['current_stages'] == 'Z1':

                    #iterate through parameters
                    for test_parameter in pipeline_status[network]:

                        #ignore 'ZIP' parameter
                        if test_parameter != 'ZIP':
            
                            #if dependent pipeline has been killed, then cannot process zip stage. Therefore terminate pipeline logfile
                            if pipeline_status[network][test_parameter]['status'] == 'killed':
                                pipeline_status[network][parameter]['status'] = 'killed'
                                pipeline_error = 'Z1 CANNOT BE PROCESSED AS AT LEAST 1 NETWORK PROCESSING PIPELINE FAILED.\nTERMINATING PIPELINE HERE'    
                                ready_to_start = False
                                break

                            #elif dependent pipeline is not nodata, and not past C1 then continue to hold on processing zip stage 
                            elif np.any([stage in pipeline_status[network][test_parameter]['current_stages'] for stage in ['S1','S2','S3','T1','T2','P1','E1','M1','C1']]):
                                ready_to_start = False
                                break
                            
                    #if ready to start, then check if network directory exists to compress 
                    #if do not exist files, then do not make pipeline logfile
                    if ready_to_start:
                        if not os.path.isdir('{}GHOST_v{}/{}'.format(run_details['compress_root'],run_details['ghost_version'],network)):
                            del pipeline_status[network][parameter]
                            continue

                # if pipeline can be started, update pipeline status
                if ready_to_start:
                    pipeline_status[network][parameter]['status'] = 'alive'

                #proceed if pipeline status for network/parameter is alive, killed or nodata
                if pipeline_status[network][parameter]['status'] in ['alive','killed','nodata']:

                    #create pipeline_logfile
                    pipeline_logfile = open('{}/run/pipeline_logs/{}_{}.out'.format(run_details['process_root'],network,parameter),"w")
                    pipeline_logfile.write('**************************************************************************************************************\n')        
                    pipeline_logfile.write('BEGINNING PROCESSING OF STAGES FOR {}:{}\n'.format(network,parameter))
                    pipeline_logfile.write('**************************************************************************************************************\n')
                    pipeline_logfile.close()

                    #if pipeline has been killed or nodata then write logfile error
                    if pipeline_status[network][parameter]['status'] in ['killed','nodata']:

                        #write pipeline logfile error
                        pipeline_logfile = open('{}/run/pipeline_logs/{}_{}.out'.format(run_details['process_root'],network,parameter),"a")
                        pipeline_logfile.write(pipeline_error)
                        pipeline_logfile.close()

                        #set current stages to be empty
                        pipeline_status[network][parameter]['current_stages'] = [] 

            #add argument for pipeline if it is alive
            if pipeline_status[network][parameter]['status'] == 'alive':

                #add start time to pipeline_status if starting pipeline
                if len(pipeline_status[network][parameter]['processed_stages']) == 0:
                    pipeline_status[network][parameter]['pipeline_start_time'] = '{:.0f}'.format(time.time())

                #add argument for pipeline
                arguments.append([slurm_job_ID, network, parameter, 
                                  pipeline_status[network][parameter]['current_stages'], 
                                  pipeline_status[network][parameter]['pipeline_start_time']])

    #add 1 to submit_iteration
    submit_iteration += 1

#end timer on GHOST processing
run_details['end_GHOST_time'] = (time.time() - start_GHOST_time)/60.

#check summary outcome of all pipelines
check_GHOST_pipeline_outcome(pipeline_status, run_details)