import os

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###

#GHOST Processing Configuration File 
ghost_version = '1.5.1'

#define the data root directory (i.e. where the observations are stored)
data_root = '/gpfs/projects/bsc32/AC_cache/obs/ghost/'
#data_root = '/gpfs/scratch/bsc32/bsc032715/GHOST_data/'

#define the processing root directory (i.e. where the processing is being run from)
process_root = os.getcwd().split('/run')[0]

#define the BSC account to do the processing on
#OPTIONS: bsc32, cns101 -> this is for prace account
bsc_account = 'cns101' 

#define the QOS (Quality of Service) used to manage jobs on the SLURM system
#it is possible to define the qos of the pipeline/stage levels of submission
#OPTIONS: gp_bsces (max walltime of 2 days), gp_resa (max walltime of 3 days), gp_debug (max walltime of 2 hours)
#         gp_hbm (max walltime of 3 days)
qos_pipeline = 'gp_resa'
qos_stage = 'gp_resa'

#define the chunk size to process the GHOST pipeline tasks (i.e. the size of CPU chunks used to process pipeline tasks)
#this should not exceed the number of CPUs per node on machine (112 physical cores on Marenostrum5)
pipeline_chunk_size = 6

#define the chunk size to process GHOST stage tasks (i.e. the size of CPU chunks used to process stage tasks)
#this should not exceed the number of CPUs per node on machine (112 physical cores on Marenostrum5)
stage_chunk_size = 55

#define the maximum walltime to process the pipeline/stages
#if 'default' is set the maximum allowed walltime for the qos is taken
#pipeline_max_walltime = 'default'
#stage_max_walltime = 'default'
pipeline_max_walltime = '04:00:00'
stage_max_walltime = '03:00:00'

#define job array limit to process GHOST stage tasks (i.e. the number of packed chunks allowed to be submitted simultaneously)
job_array_limit = 100

#define the maximum number of stations than can be should be processed for network
max_N_stations = 10000

#define if previous stage temporary files should be removed after stage completion
remove_temporary_data_files_after_stage = False

#define compression type to compress netCDF files
#OPTIONS: zlib, szip, zstd, bzip2, blosc_lz, blosc_lz4, blosc_lz4hc, blosc_zlib, blosc_zstd
compression_type = 'zlib'

#define compression level to compress netCDF files (0-9)
#1 being fastest and lowest compression ratio, 9 being slowest and highest compression ratio
#0 turns off compression
#should only be set for 'zlib', 'bzip' and 'zstd' (otherwise set as None)
compression_level = 1

#turn netCDF shuffle on or off
#this will de-interlace a block of data before compression by reordering the bytes
#should only be 'True' if compression type is 'zlib'
compression_shuffle = True

#define if multithreading should be used when processing tasks (i.e. use all threads per physical CPU)
multithreading = False

#define if need to use high memory nodes to process S1 tasks (where memory bottleneck can occur). Only valid on MN5
high_memory_S1 = False

#define if need to use high memory nodes to process T2 tasks (where memory bottleneck can occur). Only valid on MN5
high_memory_T2 = False

#define if there is an error .out file created for submitted process
error_out_file = True

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###
#START_DATE = YYYYMM (start year of processed data file - should be fixed as '197001')
start_date = '197001'

#END_DATE   = YYYYMM (end year of processed data file - should be first month of current year)
end_date = '202401'

# NETWORKS (list of networks whose data you wish to process)
# default = all networks
# VALID OPTIONS: AERONET_v3_lev1.5, AERONET_v3_lev2.0, BJMEMC, CANADA_NAPS, CAPMoN, CHILE_SINCA, CNEMC, EANET, EBAS, 
#                EEA_AIRBASE, EEA_AQ_eReporting, JAPAN_NIES, MEXICO_CDMX, MITECO, NAPMD, NOAA_ISD, NZ_ECAN, NZ_LAWA, 
#                UK_AIR, US_EPA_AirNow_DOS, US_EPA_AQS, US_EPA_CASTNET, US_NADP_AIRMoN, US_NADP_AMNet, US_NADP_AMoN, 
#                US_NADP_MDN, US_NADP_NTN, WMO_WDCGG, WMO_WDCPC
#networks = 'default'
# networks with wet dep data
#networks = ['US_NADP_NTN','US_NADP_MDN','US_NADP_AIRMoN','US_EPA_AQS','UK_AIR','EBAS','EEA_AQ_eReporting','EANET','INDAAF','CANADA_NAPS','CAPMoN','WMO_WDCPC']
networks = ['US_NADP_NTN']
# MERGE NETWORKS (list of networks to merge other network data)
# default = all networks
# VALID OPTIONS: GHOST, GHOST-PUBLIC
#merge_networks = 'default'
merge_networks = ['GHOST']

# PARAMETERS (list of parameters that you wish to process, or default per network)
# default = all parameters
# Gases:                                      O3, NO, NO2, SO2, CO, CH4,
#                                            ACETYLENE, GLYOXAL, ETHENE, ACETALDEHYDE, ETHANE, ETHANOL, PROPENE, PROPANE, 1,3-BUTADIENE, 1-BUTENE, 
#                                            ISOBUTANE, ISOPRENE, N-PENTANE, BENZENE, N-HEXANE, TOLUENE, XYELENE, M-P-XYLENE, M-XYLENE, P-XYLENE, O-XYLENE, 
#                                            1,2,4-TRIMETHYLBENZENE, MONOTERPENES, BENZO[a]PYRENE, NMVOC, VOC, NMHC, HC, 
#                                            NH3, HNO3, PAN, HCHO, HCl, HF, H2S
# Particulate Matter - size differentiated : PM10, PM2.5, PM1
# Particulate Matter - speciated:             Al, As, BaP, BC, C, Ca++, Cd, Cl-, Co, Cr, Cu, DUST, EC, Fe, Hg, K+, Mg++, Mn, MSA, Na+, NH4+, 
#                                            NH4NO3, Ni, NO3-, OC, Pb, Se, SO4--, SO4--_NSS, SO4--_SS, V, Zn
# Particulate Matter - aerosol coefficients:  AERO_ABS_COEFF_370nm, AERO_ABS_COEFF_467nm, AERO_ABS_COEFF_470nm, AERO_ABS_COEFF_520nm,
#                                            AERO_ABS_COEFF_522nm, AERO_ABS_COEFF_525nm, AERO_ABS_COEFF_528nm, AERO_ABS_COEFF_530nm, 
#                                            AERO_ABS_COEFF_550nm, AERO_ABS_COEFF_565nm, AERO_ABS_COEFF_590nm, AERO_ABS_COEFF_637nm, 
#                                            AERO_ABS_COEFF_652nm, AERO_ABS_COEFF_660nm, AERO_ABS_COEFF_670nm, AERO_ABS_COEFF_880nm, AERO_ABS_COEFF_950nm,
#                                            ABSAE_370-880nm, ABSAE_370-950nm, ABSAE_467-652nm, ABSAE_467-660nm
#                                            AERO_LIGHT_BACKSCAT_COEFF_450nm, AERO_LIGHT_BACKSCAT_COEFF_520nm, AERO_LIGHT_BACKSCAT_COEFF_525nm,
#                                            AERO_LIGHT_BACKSCAT_COEFF_530nm, AERO_LIGHT_BACKSCAT_COEFF_532nm, AERO_LIGHT_BACKSCAT_COEFF_550nm, 
#                                            AERO_LIGHT_BACKSCAT_COEFF_635nm, AERO_LIGHT_BACKSCAT_COEFF_700nm, AERO_LIGHT_BACKSCAT_COEFF_850nm, 
#                                            AERO_LIGHT_SCAT_COEFF_450nm, AERO_LIGHT_SCAT_COEFF_520nm, AERO_LIGHT_SCAT_COEFF_525nm, 
#                                            AERO_LIGHT_SCAT_COEFF_530nm, AERO_LIGHT_SCAT_COEFF_532nm, AERO_LIGHT_SCAT_COEFF_550nm, 
#                                            AERO_LIGHT_SCAT_COEFF_635nm, AERO_LIGHT_SCAT_COEFF_700nm, AERO_LIGHT_SCAT_COEFF_850nm,
#                                            SCATAE_450-635nm, SCATAE_450-700nm,
#                                            BACKSCAT_FRAC_450nm, BACKSCAT_FRAC_520nm, BACKSCAT_FRAC_525nm, BACKSCAT_FRAC_530nm, BACKSCAT_FRAC_532nm, 
#                                            BACKSCAT_FRAC_550nm, BACKSCAT_FRAC_635nm, BACKSCAT_FRAC_700nm, BACKSCAT_FRAC_850nm, ASY_525nm, ASY_550nm,
#                                            BC_MAC_637nm, BC_MAC_880nm, SSA_470nm, SSA_520nm, SSA_590nm, SSA_637nm, SSA_660nm
# PM10 - speciated:                           PM10_Al, PM10_As, PM10_BaP, PM10_BC, PM10_C, PM10_Ca++, PM10_Cd, PM10_Cl-, PM10_Co,
#                                            PM10_Cr, PM10_Cu, PM10_DUST, PM10_EC, PM10_Fe, PM10_Hg, PM10_K+, PM10_Mg++, PM10_Mn, PM10_MSA, PM10_Na+,
#                                            PM10_NH4+, PM10_NH4NO3, PM10_Ni, PM10_NO3-, PM10_OC, PM10_Pb, PM10_Se, PM10_SO4--,
#                                            PM10_SO4--_NSS, PM10_SO4--_SS, PM10_V, PM10_Zn
# PM10 - aerosol coefficients:                PM10_AERO_ABS_COEFF_370nm, PM10_AERO_ABS_COEFF_467nm, PM10_AERO_ABS_COEFF_470nm, PM10_AERO_ABS_COEFF_520nm,
#                                            PM10_AERO_ABS_COEFF_522nm, PM10_AERO_ABS_COEFF_525nm, PM10_AERO_ABS_COEFF_528nm, PM10_AERO_ABS_COEFF_530nm, 
#                                            PM10_AERO_ABS_COEFF_550nm, PM10_AERO_ABS_COEFF_565nm, PM10_AERO_ABS_COEFF_590nm, PM10_AERO_ABS_COEFF_637nm, 
#                                            PM10_AERO_ABS_COEFF_652nm, PM10_AERO_ABS_COEFF_660nm, PM10_AERO_ABS_COEFF_670nm, PM10_AERO_ABS_COEFF_880nm, PM10_AERO_ABS_COEFF_950nm,
#                                            PM10_ABSAE_370-880nm, PM10_ABSAE_370-950nm, PM10_ABSAE_467-652nm, PM10_ABSAE_467-660nm,
#                                            PM10_AERO_LIGHT_BACKSCAT_COEFF_450nm, PM10_AERO_LIGHT_BACKSCAT_COEFF_520nm, PM10_AERO_LIGHT_BACKSCAT_COEFF_525nm,
#                                            PM10_AERO_LIGHT_BACKSCAT_COEFF_530nm, PM10_AERO_LIGHT_BACKSCAT_COEFF_532nm, PM10_AERO_LIGHT_BACKSCAT_COEFF_550nm, 
#                                            PM10_AERO_LIGHT_BACKSCAT_COEFF_635nm, PM10_AERO_LIGHT_BACKSCAT_COEFF_700nm, PM10_AERO_LIGHT_BACKSCAT_COEFF_850nm, 
#                                            PM10_AERO_LIGHT_SCAT_COEFF_450nm, PM10_AERO_LIGHT_SCAT_COEFF_520nm, PM10_AERO_LIGHT_SCAT_COEFF_525nm, 
#                                            PM10_AERO_LIGHT_SCAT_COEFF_530nm, PM10_AERO_LIGHT_SCAT_COEFF_532nm, PM10_AERO_LIGHT_SCAT_COEFF_550nm, 
#                                            PM10_AERO_LIGHT_SCAT_COEFF_635nm, PM10_AERO_LIGHT_SCAT_COEFF_700nm, PM10_AERO_LIGHT_SCAT_COEFF_850nm,
#                                            PM10_SCATAE_450-635nm, PM10_SCATAE_450-700nm
#                                            PM10_BACKSCAT_FRAC_450nm, PM10_BACKSCAT_FRAC_520nm, PM10_BACKSCAT_FRAC_525nm, PM10_BACKSCAT_FRAC_530nm, PM10_BACKSCAT_FRAC_532nm, 
#                                            PM10_BACKSCAT_FRAC_550nm, PM10_BACKSCAT_FRAC_635nm, PM10_BACKSCAT_FRAC_700nm, PM10_BACKSCAT_FRAC_850nm, PM10_ASY_525nm, PM10_ASY_550nm, 
#                                            PM10_BC_MAC_637nm, PM10_BC_MAC_880nm, PM10_SSA_470nm, PM10_SSA_520nm, PM10_SSA_590nm, PM10_SSA_637nm, PM10_SSA_660nm  
# PM2.5 - speciated:                          PM2.5_Al, PM2.5_As, PM2.5_BaP, PM2.5_BC, PM2.5_C, PM2.5_Ca++, PM2.5_Cd, PM2.5_Cl-,
#                                            PM2.5_Co, PM2.5_Cr, PM2.5_Cu, PM2.5_DUST, PM2.5_EC, PM2.5_Fe, PM2.5_Hg, PM2.5_K+, PM2.5_Mg++, PM2.5_Mn,
#                                            PM2.5_MSA, PM2.5_Na+, PM2.5_NH4+, PM2.5_NH4NO3, PM2.5_Ni, PM2.5_NO3-, PM2.5_OC,
#                                            PM2.5_Pb, PM2.5_Se, PM2.5_SO4--, PM2.5_SO4--_NSS, PM2.5_SO4--_SS, PM2.5_V, PM2.5_Zn
# PM2.5 - aerosol coefficients:               PM2.5_AERO_ABS_COEFF_370nm, PM2.5_AERO_ABS_COEFF_467nm, PM2.5_AERO_ABS_COEFF_470nm, PM2.5_AERO_ABS_COEFF_520nm,
#                                            PM2.5_AERO_ABS_COEFF_522nm, PM2.5_AERO_ABS_COEFF_525nm, PM2.5_AERO_ABS_COEFF_528nm, PM2.5_AERO_ABS_COEFF_530nm, 
#                                            PM2.5_AERO_ABS_COEFF_550nm, PM2.5_AERO_ABS_COEFF_565nm, PM2.5_AERO_ABS_COEFF_590nm, PM2.5_AERO_ABS_COEFF_637nm,  
#                                            PM2.5_AERO_ABS_COEFF_652nm, PM2.5_AERO_ABS_COEFF_660nm, PM2.5_AERO_ABS_COEFF_670nm, PM2.5_AERO_ABS_COEFF_880nm, PM2.5_AERO_ABS_COEFF_950nm,
#                                            PM2.5_ABSAE_370-880nm, PM2.5_ABSAE_370-950nm, PM2.5_ABSAE_467-652nm, PM2.5_ABSAE_467-660nm,
#                                            PM2.5_AERO_LIGHT_BACKSCAT_COEFF_450nm, PM2.5_AERO_LIGHT_BACKSCAT_COEFF_520nm, PM2.5_AERO_LIGHT_BACKSCAT_COEFF_525nm,
#                                            PM2.5_AERO_LIGHT_BACKSCAT_COEFF_530nm, PM2.5_AERO_LIGHT_BACKSCAT_COEFF_532nm, PM2.5_AERO_LIGHT_BACKSCAT_COEFF_550nm, 
#                                            PM2.5_AERO_LIGHT_BACKSCAT_COEFF_635nm, PM2.5_AERO_LIGHT_BACKSCAT_COEFF_700nm, PM2.5_AERO_LIGHT_BACKSCAT_COEFF_850nm, 
#                                            PM2.5_AERO_LIGHT_SCAT_COEFF_450nm, PM2.5_AERO_LIGHT_SCAT_COEFF_520nm, PM2.5_AERO_LIGHT_SCAT_COEFF_525nm, 
#                                            PM2.5_AERO_LIGHT_SCAT_COEFF_530nm, PM2.5_AERO_LIGHT_SCAT_COEFF_532nm, PM2.5_AERO_LIGHT_SCAT_COEFF_550nm, 
#                                            PM2.5_AERO_LIGHT_SCAT_COEFF_635nm, PM2.5_AERO_LIGHT_SCAT_COEFF_700nm, PM2.5_AERO_LIGHT_SCAT_COEFF_850nm,
#                                            PM2.5_SCATAE_450-635nm, PM2.5_SCATAE_450-700nm
#                                            PM2.5_BACKSCAT_FRAC_450nm, PM2.5_BACKSCAT_FRAC_520nm, PM2.5_BACKSCAT_FRAC_525nm, PM2.5_BACKSCAT_FRAC_530nm, PM2.5_BACKSCAT_FRAC_532nm, 
#                                            PM2.5_BACKSCAT_FRAC_550nm, PM2.5_BACKSCAT_FRAC_635nm, PM2.5_BACKSCAT_FRAC_700nm, PM2.5_BACKSCAT_FRAC_850nm, PM2.5_ASY_525nm, PM2.5_ASY_550nm, 
#                                            PM2.5_BC_MAC_637nm, PM2.5_BC_MAC_880nm, PM2.5_SSA_470nm, PM2.5_SSA_520nm, PM2.5_SSA_590nm, PM2.5_SSA_637nm, PM2.5_SSA_660nm    
# PM1 - speciated:                            PM1_Al, PM1_As, PM1_BaP, PM1_BC, PM1_C, PM1_Ca++, PM1_Cd, PM1_Cl-, PM1_Co, PM1_Cr,
#                                            PM1_Cu, PM1_DUST, PM1_EC, PM1_Fe, PM1_Hg, PM1_K+, PM1_Mg++, PM1_Mn, PM1_MSA, PM1_Na+, PM1_NH4+, PM1_NH4NO3, 
#                                            PM1_Ni, PM1_NO3-, PM1_OC, PM1_Pb, PM1_Se, PM1_SO4--, PM1_SO4--_NSS, PM1_SO4--_SS, PM1_V, PM1_Zn
# PM1 - aerosol coefficients:                 PM1_AERO_ABS_COEFF_370nm, PM1_AERO_ABS_COEFF_467nm, PM1_AERO_ABS_COEFF_470nm, PM1_AERO_ABS_COEFF_520nm,
#                                            PM1_AERO_ABS_COEFF_522nm, PM1_AERO_ABS_COEFF_525nm, PM1_AERO_ABS_COEFF_528nm, PM1_AERO_ABS_COEFF_530nm, 
#                                            PM1_AERO_ABS_COEFF_550nm, PM1_AERO_ABS_COEFF_565nm, PM1_AERO_ABS_COEFF_590nm, PM1_AERO_ABS_COEFF_637nm,  
#                                            PM1_AERO_ABS_COEFF_652nm, PM1_AERO_ABS_COEFF_660nm, PM1_AERO_ABS_COEFF_670nm, PM1_AERO_ABS_COEFF_880nm, PM1_AERO_ABS_COEFF_950nm,
#                                            PM1_ABSAE_370-880nm, PM1_ABSAE_370-950nm, PM1_ABSAE_467-652nm, PM1_ABSAE_467-660nm,
#                                            PM1_AERO_LIGHT_BACKSCAT_COEFF_450nm, PM1_AERO_LIGHT_BACKSCAT_COEFF_520nm, PM1_AERO_LIGHT_BACKSCAT_COEFF_525nm,
#                                            PM1_AERO_LIGHT_BACKSCAT_COEFF_530nm, PM1_AERO_LIGHT_BACKSCAT_COEFF_532nm, PM1_AERO_LIGHT_BACKSCAT_COEFF_550nm, 
#                                            PM1_AERO_LIGHT_BACKSCAT_COEFF_635nm, PM1_AERO_LIGHT_BACKSCAT_COEFF_700nm, PM1_AERO_LIGHT_BACKSCAT_COEFF_850nm, 
#                                            PM1_AERO_LIGHT_SCAT_COEFF_450nm, PM1_AERO_LIGHT_SCAT_COEFF_520nm, PM1_AERO_LIGHT_SCAT_COEFF_525nm, 
#                                            PM1_AERO_LIGHT_SCAT_COEFF_530nm, PM1_AERO_LIGHT_SCAT_COEFF_532nm, PM1_AERO_LIGHT_SCAT_COEFF_550nm, 
#                                            PM1_AERO_LIGHT_SCAT_COEFF_635nm, PM1_AERO_LIGHT_SCAT_COEFF_700nm, PM1_AERO_LIGHT_SCAT_COEFF_850nm,
#                                            PM1_SCATAE_450-635nm, PM1_SCATAE_450-700nm
#                                            PM1_BACKSCAT_FRAC_450nm, PM1_BACKSCAT_FRAC_520nm, PM1_BACKSCAT_FRAC_525nm, PM1_BACKSCAT_FRAC_530nm, PM1_BACKSCAT_FRAC_532nm, 
#                                            PM1_BACKSCAT_FRAC_550nm, PM1_BACKSCAT_FRAC_635nm, PM1_BACKSCAT_FRAC_700nm, PM1_BACKSCAT_FRAC_850nm, PM1_ASY_525nm, PM1_ASY_550nm, 
#                                            PM1_BC_MAC_637nm, PM1_BC_MAC_880nm, PM1_SSA_470nm, PM1_SSA_520nm, PM1_SSA_590nm, PM1_SSA_637nm, PM1_SSA_660nm   
# Total AOD                                   AOD_380nm, AOD_440nm, AOD_500nm, AOD_500nm_COARSE, AOD_500nm_FINE, FINE_MODE_FRAC_500nm, AOD_550nm,
#                                            AOD_675nm, AOD_870nm, AOD_1020nm, AE_440-870nm
# Extinction AOD                              EXTAOD_440nm, EXTAOD_440nm_COARSE, EXTAOD_440nm_FINE, EXTAOD_550nm, EXTAOD_675nm, EXTAOD_675nm_COARSE, EXTAOD_675nm_FINE,
#                                            EXTAOD_870nm, EXTAOD_870nm_COARSE, EXTAOD_870nm_FINE, EXTAOD_1020nm, EXTAOD_1020nm_COARSE, EXTAOD_1020nm_FINE,
#                                            EXTAE_440-675nm, EXTAE_440-870nm
# Absorption AOD                             ABSAOD_440nm, ABSAOD_550nm, ABSAOD_675nm, ABSAOD_870nm, ABSAOD_1020nm, ABSAE_440-675nm, ABSAE_440-870nm, ABSAE_675-870nm, ABSWDAE
# Absorption AOD - Black Carbon              BC_ABSAOD_440nm, BC_ABSAE_440-870nm
# Absorption AOD - Brown Carbon              BrC_ABSAOD_440nm
# Scattering AOD                             SCATAE_440-675nm
# Single Scattering Albedo:                  SSA_440nm, SSA_550nm, SSA_675nm, SSA_870nm, SSA_1020nm, SSAAE_440-675nm
# Asymmetry/Sphericity factors                ASY_440nm, ASY_440nm_COARSE, ASY_440nm_FINE, ASY_675nm, ASY_675nm_COARSE, ASY_675nm_FINE, 
#                                            ASY_870nm, ASY_870nm_COARSE, ASY_870nm_FINE, ASY_1020nm, ASY_1020nm_COARSE, ASY_1020nm_FINE, SPH             
# Refractive Indices                          RIN_REAL_440nm, RIN_REAL_675nm, RIN_REAL_870nm, RIN_REAL_1020nm, 
#                                            RIN_IMAG_440nm, RIN_IMAG_675nm, RIN_IMAG_870nm, RIN_IMAG_1020nm
# Volume Concentration                       VCONC, VCONC_FINE, VCONC_COARSE
# Size Distribution                           VCONC_BIN1, VCONC_BIN2, VCONC_BIN3, VCONC_BIN4, VCONC_BIN5, VCONC_BIN6, VCONC_BIN7, VCONC_BIN8, VCONC_BIN9,
#                                            VCONC_BIN10, VCONC_BIN11, VCONC_BIN12, VCONC_BIN13, VCONC_BIN14, VCONC_BIN15, VCONC_BIN16, VCONC_BIN17, VCONC_BIN18, 
#                                            VCONC_BIN19, VCONC_BIN20, VCONC_BIN21, VCONC_BIN22                     
# Schuster Dust Filters                      SCHUSTER_FILTER_BC_BrC, SCHUSTER_FILTER_SS
# Precipitation speciated:                    Al_PREC, As_PREC, BaP_PREC, Ca++_PREC, Cd_PREC, Cl-_PREC, Co_PREC, Cr_PREC, Cu_PREC, Fe_PREC, Hg_PREC,
#                                            K+_PREC, Mg++_PREC, Mn_PREC, MSA_PREC, Na+_PREC, NH4+_PREC, NH4NO3_PREC, Ni_PREC,
#                                            NO3-_PREC, Pb_PREC, Se_PREC, SO4--_PREC, SO4--_NSS_PREC, SO4--_SS_PREC, V_PREC, Zn_PREC
# Wet Deposition speciated:                   Al_WETDEP, As_WETDEP, BaP_WETDEP, Ca++_WETDEP, Cd_WETDEP, Cl-_WETDEP, Co_WETDEP, Cr_WETDEP, Cu_WETDEP, Fe_WETDEP, 
#                                            Hg_WETDEP, K+_WETDEP, Mg++_WETDEP, Mn_WETDEP, MSA_WETDEP, Na+_WETDEP, NH4+_WETDEP, NH4NO3_WETDEP, Ni_WETDEP,
#                                            NO3-_WETDEP, Pb_WETDEP, Se_WETDEP, SO4--_WETDEP, SO4--_NSS_WETDEP, SO4--_SS_WETDEP, V_WETDEP, Zn_WETDEP
# Meteorology:                                WND_DIR_10M, WND_SPD_10M, TEMP_2M, RH_2M, SST, DEW_PT_2M, PRES_2M, SLP,
#                                            PREC_ACCUM, SNOW_ACCUM, SNOW_DEPTH, CEILING_HEIGHT, VIS_DIST, CLOUD_CVG_FRAC
#parameters = 'default'
#parameters = ['EXTAOD_440nm','EXTAOD_675nm','EXTAOD_870nm','EXTAE_440-870nm','ABSAOD_440nm','ABSAOD_675nm','ABSAOD_870nm','ABSAE_440-870nm','SSA_440nm','SSA_675nm','SSA_870nm']
#parameters = ['VCONC_BIN1', 'VCONC_BIN2', 'VCONC_BIN3', 'VCONC_BIN4', 'VCONC_BIN5', 'VCONC_BIN6', 'VCONC_BIN7', 'VCONC_BIN8', 'VCONC_BIN9', 'VCONC_BIN10', 'VCONC_BIN11', 'VCONC_BIN12', 'VCONC_BIN13', 'VCONC_BIN14', 'VCONC_BIN15', 'VCONC_BIN16', 'VCONC_BIN17', 'VCONC_BIN18', 'VCONC_BIN19', 'VCONC_BIN20', 'VCONC_BIN21', 'VCONC_BIN22']
#parameters = ['AOD_440nm','AOD_675nm','AOD_870nm','AE_440-870nm','FINE_MODE_FRAC_500nm','RIN_IMAG_675nm','RIN_IMAG_870nm','RIN_IMAG_1020nm']
#parameters = ['AOD_550nm','EXTAOD_550nm','ABSAOD_550nm','SCHUSTER_FILTER_BC_BrC','SCHUSTER_FILTER_SS']
#parameters = ['SSA_550nm']
#parameters = ['M-P-XYLENE']
#parameters = ['XYLENE','M-P-XYLENE','M-XYLENE','P-XYLENE','O-XYLENE']
#parameters = ['EXTAE_440-675nm','ABSAE_440-675nm','ABSAE_675-870nm','SSAAE_440-675nm']
#parameters = ['SCATAE_440-675nm','ABSWDAE']
#parameters = ['BC_ABSAE_440-870nm']
#parameters = ['BC_ABSAOD_440nm']
#parameters = ['BrC_ABSAOD_440nm']
# CANADA NAPS PREC metals
#parameters = ['Al_PREC', 'As_PREC', 'Cd_PREC', 'Co_PREC', 'Cr_PREC', 'Cu_PREC', 'Fe_PREC', 'Mn_PREC', 'Ni_PREC', 'Pb_PREC', 'Se_PREC', 'V_PREC', 'Zn_PREC']
#parameters = ['PREC_ACCUM']
# INDAAF precip
#parameters = ['SO4--_PREC', 'NO3-_PREC', 'NH4+_PREC', 'Cl-_PREC','Na+_PREC', 'K+_PREC', 'Mg++_PREC', 'Ca++_PREC', 'H+_PREC']
# parameters precip US EPA AQS
#parameters = ['SO4--_PREC']
#parameters = ['H+_PREC','Mg++_PREC','NO3-_PREC','Ca++_PREC','Cl-_PREC','K+_PREC','NH4+_PREC','Na+_PREC','SO4--_PREC']
#parameters = ['H+_PREC']
#parameters =  ['Ca++_WETDEP', 'Mg++_WETDEP', 'K+_WETDEP', 'Na+_WETDEP', 'NH4+_WETDEP', 'NO3-_WETDEP', 'Cl-_WETDEP', 'SO4--_WETDEP', 'PO4---_WETDEP', 'F-_WETDEP'] 
# CASTNET precip
#parameters= ['SO4--_PREC', 'NO3-_PREC', 'NH4+_PREC', 'Cl-_PREC','Na+_PREC', 'K+_PREC', 'Mg++_PREC', 'Ca++_PREC']
#parameters = ['SO4--_PREC']
#parameters = ['H+_PREC', 'PREC_ACCUM', 'SO4--_PREC', 'NH4+_PREC', 'NO3-_PREC']
# UK AIR precip
#parameters = ['SO4--_PREC', 'NO3-_PREC', 'NH4+_PREC', 'Cl-_PREC','Na+_PREC', 'K+_PREC', 'Mg++_PREC', 'Ca++_PREC', 'PREC_ACCUM']
#parameters = ['Cl-_PREC'] 
# EANET precip
#parameters = ['SO4--_PREC', 'NO3-_PREC', 'NH4+_PREC', 'Cl-_PREC', 'F-_PREC', 'PO4---_PREC', 'Na+_PREC', 'K+_PREC', 'Mg++_PREC', 'Ca++_PREC', 'H+_PREC']
#parameters = ['SO4--_WETDEP', 'NO3-_WETDEP', 'NH4+_WETDEP', 'Cl-_WETDEP', 'F-_WETDEP', 'PO4---_WETDEP', 'Na+_WETDEP', 'K+_WETDEP', 'Mg++_WETDEP', 'Ca++_WETDEP']
# prec params of EBAS
#parameters = ['Ca++_WETDEP', 'Na+_WETDEP', 'Mg++_WETDEP']
#parameters = ['Al_WETDEP', 'As_WETDEP', 'Ca++_WETDEP', 'Cd_WETDEP', 'Cl-_WETDEP', 'Co_WETDEP', 'Cr_WETDEP', 'Cu_WETDEP', 'Fe_WETDEP', 'Hg_WETDEP', 'K+_WETDEP', 'Mg++_WETDEP', 'Mn_WETDEP','MSA_WETDEP', 'NO3-_WETDEP','NH4+_WETDEP', 'Na+_WETDEP', 'Ni_WETDEP', 'Pb_WETDEP', 'SO4--_WETDEP', 'SO4--_NSS_WETDEP', 'Se_WETDEP', 'V_WETDEP', 'Zn_WETDEP']
#parameters = ['Al_PREC', 'As_PREC', 'BaP_PREC', 'Ca++_PREC', 'Cd_PREC', 'Cl-_PREC', 'Co_PREC', 'Cr_PREC', 'Cu_PREC', 'Fe_PREC', 'Hg_PREC', 'K+_PREC', 'Mg++_PREC', 'Mn_PREC','MSA_PREC', 'NO3-_PREC','NH4+_PREC', 'Na+_PREC', 'Ni_PREC', 'Pb_PREC', 'SO4--_PREC', 'SO4--_NSS_PREC', 'Se_PREC', 'V_PREC', 'Zn_PREC']
# prec parameters EEA
#parameters = ['As_PREC','Ca++_PREC','Cd_PREC','Cl-_PREC','Co_PREC','Cr_PREC','Cu_PREC','Hg_PREC','K+_PREC','Mg++_PREC','Mn_PREC','NH4+_PREC','NO3-_PREC','Na+_PREC','Ni_PREC','Pb_PREC','SO4--_PREC','V_PREC','Zn_PREC', 'H+_PREC'] 
# precip parameters CANADA NAPS
#parameters = ['SO4--_PREC', 'SO4--_NSS_PREC', 'NO3-_PREC', 'PO4---_PREC', 'NH4+_PREC', 'Cl-_PREC', 'Na+_PREC', 'K+_PREC', 'Mg++_PREC', 'Ca++_PREC', 'H+_PREC']
# precip CAPMoN
#parameters = ['SO4--_PREC', 'NO3-_PREC', 'NH4+_PREC', 'Cl-_PREC', 'Na+_PREC', 'K+_PREC', 'Mg++_PREC', 'Ca++_PREC', 'H+_PREC']
#parameters = ['SO4--_WETDEP', 'SO4--_NSS_WETDEP', 'NO3-_WETDEP', 'PO4---_WETDEP', 'NH4+_WETDEP', 'Cl-_WETDEP', 'Na+_WETDEP', 'K+_WETDEP', 'Mg++_WETDEP', 'Ca++_WETDEP']
#parameters = ['SO4--_PREC']
#parameters = ['SO4--_PREC', 'NO3-_PREC', 'NH4+_PREC', 'Cl-_PREC', 'Na+_PREC', 'K+_PREC', 'Mg++_PREC', 'Ca++_PREC', 'PO4---_PREC', 'F-_PREC', 'PREC_ACCUM']
# US NADP NTN
#parameters = ['Ca++_PREC', 'Mg++_PREC', 'K+_PREC', 'Na+_PREC', 'NH4+_PREC', 'NO3-_PREC', 'Cl-_PREC','H+_PREC']
# US NADP AIRMoN
#parameters = ['Ca++_PREC', 'Mg++_PREC', 'K+_PREC', 'Na+_PREC', 'NH4+_PREC', 'NO3-_PREC', 'Cl-_PREC', 'SO4--_PREC', 'PO4---_PREC', 'H+_PREC']
# US NADP MDN
#parameters = ['Hg_PREC']
# WMO WDCPC
#parameters = ['Ca++_PREC', 'Mg++_PREC', 'K+_PREC', 'Na+_PREC', 'NH4+_PREC', 'NO3-_PREC', 'Cl-_PREC', 'SO4--_PREC', 'PO4---_PREC', 'F-_PREC', 'Zn_PREC', 'H+_PREC']
#parameters = ['H+_PREC','NO3-_PREC','PO4---_PREC', 'F-_PREC', 'Zn_PREC']
#parameters = ['SO4--_PREC']
#parameters = ['PM10', 'PM2.5']
#parameters = ['BC_MAC_637nm', 'SSA_470nm']
#parameters = ['BC_MAC_637nm', 'BC_MAC_880nm', 'SSA_470nm', 'SSA_520nm', 'SSA_590nm', 'SSA_637nm', 'SSA_660nm', 'PM10_BC_MAC_637nm', 'PM10_BC_MAC_880nm', 'PM10_SSA_470nm', 'PM10_SSA_520nm', 'PM10_SSA_590nm', 'PM10_SSA_637nm', 'PM10_SSA_660nm', 'PM2.5_BC_MAC_637nm', 'PM2.5_BC_MAC_880nm', 'PM2.5_SSA_470nm', 'PM2.5_SSA_520nm', 'PM2.5_SSA_590nm', 'PM2.5_SSA_637nm', 'PM2.5_SSA_660nm', 'PM1_BC_MAC_637nm', 'PM1_BC_MAC_880nm', 'PM1_SSA_470nm', 'PM1_SSA_520nm', 'PM1_SSA_590nm', 'PM1_SSA_637nm', 'PM1_SSA_660nm']
#parameters = ['EC','AERO_ABS_COEFF_637nm','BC_MAC_637nm','AERO_ABS_COEFF_880nm','BC_MAC_880nm','PM10_EC','PM10_AERO_ABS_COEFF_637nm','PM10_BC_MAC_637nm','PM10_AERO_ABS_COEFF_880nm','PM10_BC_MAC_880nm','PM2.5_EC','PM2.5_AERO_ABS_COEFF_637nm','PM2.5_BC_MAC_637nm','PM2.5_AERO_ABS_COEFF_880nm','PM2.5_BC_MAC_880nm']
#parameters = ['AERO_LIGHT_SCAT_COEFF_450nm','AERO_LIGHT_SCAT_COEFF_525nm','AERO_LIGHT_SCAT_COEFF_550nm','AERO_LIGHT_SCAT_COEFF_635nm','AERO_LIGHT_SCAT_COEFF_700nm','AERO_ABS_COEFF_470nm','AERO_ABS_COEFF_520nm','AERO_ABS_COEFF_590nm','AERO_ABS_COEFF_637nm','AERO_ABS_COEFF_660nm','SCATAE_450-635nm','SCATAE_450-700nm',
#              'PM10_AERO_LIGHT_SCAT_COEFF_450nm','PM10_AERO_LIGHT_SCAT_COEFF_525nm','PM10_AERO_LIGHT_SCAT_COEFF_550nm','PM10_AERO_LIGHT_SCAT_COEFF_635nm','PM10_AERO_LIGHT_SCAT_COEFF_700nm','PM10_AERO_ABS_COEFF_470nm','PM10_AERO_ABS_COEFF_520nm','PM10_AERO_ABS_COEFF_590nm','PM10_AERO_ABS_COEFF_637nm','PM10_AERO_ABS_COEFF_660nm','PM10_SCATAE_450-635nm','PM10_SCATAE_450-700nm',
#              'PM2.5_AERO_LIGHT_SCAT_COEFF_450nm','PM2.5_AERO_LIGHT_SCAT_COEFF_525nm','PM2.5_AERO_LIGHT_SCAT_COEFF_550nm','PM2.5_AERO_LIGHT_SCAT_COEFF_635nm','PM2.5_AERO_LIGHT_SCAT_COEFF_700nm','PM2.5_AERO_ABS_COEFF_470nm','PM2.5_AERO_ABS_COEFF_520nm','PM2.5_AERO_ABS_COEFF_590nm','PM2.5_AERO_ABS_COEFF_637nm','PM2.5_AERO_ABS_COEFF_660nm','PM2.5_SCATAE_450-635nm','PM2.5_SCATAE_450-700nm',
#              'PM1_AERO_LIGHT_SCAT_COEFF_450nm','PM1_AERO_LIGHT_SCAT_COEFF_525nm','PM1_AERO_LIGHT_SCAT_COEFF_550nm','PM1_AERO_LIGHT_SCAT_COEFF_635nm','PM1_AERO_LIGHT_SCAT_COEFF_700nm','PM1_AERO_ABS_COEFF_470nm','PM1_AERO_ABS_COEFF_520nm','PM1_AERO_ABS_COEFF_590nm','PM1_AERO_ABS_COEFF_637nm','PM1_AERO_ABS_COEFF_660nm','PM1_SCATAE_450-635nm','PM1_SCATAE_450-700nm']
#parameters = ['PM10_EC', 'PM2.5_EC', 'PM1_AERO_ABS_COEFF_470nm', 'PM1_AERO_ABS_COEFF_520nm', 'PM1_AERO_ABS_COEFF_590nm', 'PM1_AERO_ABS_COEFF_637nm', 'PM1_AERO_ABS_COEFF_660nm']
#parameters = ['SCATAE_450-635nm','SCATAE_450-700nm','PM10_SCATAE_450-635nm','PM10_SCATAE_450-700nm','PM2.5_SCATAE_450-635nm','PM2.5_SCATAE_450-700nm','PM1_SCATAE_450-635nm','PM1_SCATAE_450-700nm',
#              'BC_MAC_637nm', 'BC_MAC_880nm', 'SSA_470nm', 'SSA_520nm', 'SSA_590nm', 'SSA_637nm', 'SSA_660nm', 'PM10_BC_MAC_637nm', 'PM10_BC_MAC_880nm', 'PM10_SSA_470nm', 'PM10_SSA_520nm', 'PM10_SSA_590nm', 'PM10_SSA_637nm', 'PM10_SSA_660nm', 'PM2.5_BC_MAC_637nm', 'PM2.5_BC_MAC_880nm', 'PM2.5_SSA_470nm', 'PM2.5_SSA_520nm', 'PM2.5_SSA_590nm', 'PM2.5_SSA_637nm', 'PM2.5_SSA_660nm', 'PM1_BC_MAC_637nm', 'PM1_BC_MAC_880nm', 'PM1_SSA_470nm', 'PM1_SSA_520nm', 'PM1_SSA_590nm', 'PM1_SSA_637nm', 'PM1_SSA_660nm']
#parameters = ['PM1_AERO_ABS_COEFF_637nm','PM1_AERO_ABS_COEFF_880nm','PM1_EC']
#parameters = ['PM1_BC_MAC_637nm', 'PM1_BC_MAC_880nm', 'SSA_470nm', 'SSA_520nm', 'SSA_590nm', 'SSA_637nm', 'SSA_660nm']
# INDAAF aerosol composition
#parameters = ['PM10_Cl-','PM10_NO3-','PM10_SO4--','PM10_Na+','PM10_NH4+','PM10_K+','PM10_Mg++','PM10_Ca++', 'PM10_C', 'PM10_EC', 'PM10_OC', 'PM10_Al','PM10_Fe','PM10_Mn','PM10_Pb','PM10_Cr','PM10_Cu','PM10_Zn','PM10_Cd','PM10_Ni', 'PM2.5_Cl-','PM2.5_NO3-','PM2.5_SO4--','PM2.5_Na+','PM2.5_NH4+','PM2.5_K+','PM2.5_Mg++','PM2.5_Ca++', 'PM2.5_C', 'PM2.5_EC', 'PM2.5_OC', 'PM2.5_Al','PM2.5_Fe','PM2.5_Mn','PM2.5_Pb','PM2.5_Cr','PM2.5_Cu','PM2.5_Zn','PM2.5_Cd','PM2.5_Ni', 'PM_Cl-','PM_NO3-','PM_SO4--','PM_Na+','PM_NH4+','PM_K+','PM_Mg++','PM_Ca++', 'PM_C', 'PM_EC', 'PM_OC', 'PM_Al','PM_Fe','PM_Mn','PM_Pb','PM_Cr','PM_Cu','PM_Zn','PM_Cd','PM_Ni']
#parameters = ['PM10_Al', 'PM10_Fe', 'PM10_Mn', 'PM10_Pb', 'PM10_Cr', 'PM10_Cu', 'PM10_Zn', 'PM10_Cd', 'PM10_Ni', 'PM2.5_Al', 'PM2.5_Fe', 'PM2.5_Mn', 'PM2.5_Pb', 'PM2.5_Cr', 'PM2.5_Cu', 'PM2.5_Zn', 'PM2.5_Cd', 'PM2.5_Ni', 'PM10_C', 'PM10_EC', 'PM10_OC']
parameters = ['PREC_ACCUM']
#parameters = ['SO4--_PREC']

#TEMPORAL_RESOLUTIONS = H, D, M, HI (temporal resolution desired of merged output netCDF files: hourly, daily, monthly, hourly instantaneous)       
#default = all temporal resolutions for instantaneous measured parameters, and H, D, M for all others
temporal_resolutions = 'default'

#REGIONS = GLOBAL, EU, IP, NA (this works by selectively keeping data within a defined rectangle per region)
#default = all regions for NOAA_ISD network, GLOBAL for all others
regions = 'default'
#regions = ['GLOBAL','EU','IP','NA']
#regions = ['GLOBAL']

#STAGES_TO_PROCESS (list of GHOST stages that you wish to process)
#default = if all default networks are being processed, this is all stages.
#          if a selection of networks is being processed, this is all stages except M1

#PARSING OF NETWORK DATA FILES PARSING TO STATION SPECIFIC .npz FILES
#  S1 = Raw network file parse
#  S2 = Concatenate parsed station data files / Split stations by significantly changing locational information
#  S3 = Add globally gridded metadata / Quality control / Add temporal classification codes

#TEMPORAL AVERAGING / AGGREGATION OF STATIONS TO netCDF
#  T1 = Temporally averaging by station
#  T2 = Aggregate temporally averaged station data in monthly netCDFs

#DATA PRODUCT
#  P1 = Create data product from other processed components

#SPLIT EBAS
#  E1 = Split out EBAS data into individual native networks

#MERGE 
#  M1 = Merge data across networks

#stages_to_process = 'default'
#stages_to_process = ['S1','S2','S3','T1','T2','P1','E1','M1']
#stages_to_process = ['S1','S2','S3','T1','T2']
#stages_to_process = ['S3']
#stages_to_process = ['S1']
stages_to_process = ['T1', 'T2']
#stages_to_process = ['E1']
#stages_to_process = ['M1']
###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###
