import os

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###

#GHOST Processing Configuration File 
ghost_version = '1.5.1'

#define the data root directory (i.e. where the observations are stored)
data_root = '/gpfs/projects/bsc32/AC_cache/obs/ghost/'
#data_root = '/gpfs/scratch/bsc32/bsc032715/GHOST_data/'

#define the processing root directory (i.e. where the processing is being run from)
process_root = os.getcwd().split('/run')[0]

#define the compress root directory (i.e. where the data is compressed per network)
compress_root = '/gpfs/projects/bsc32/AC_cache/obs/ghost/delivered_data/PUBLIC/'

#define the BSC account to do the processing on
#OPTIONS: bsc32, cns101 -> this is for prace account
bsc_account = 'cns109' 

#define the QOS (Quality of Service) used to manage jobs on the SLURM system
#it is possible to define the qos of the pipeline/stage levels of submission
#OPTIONS: gp_bsces (max walltime of 2 days), gp_resa (max walltime of 3 days), gp_debug (max walltime of 2 hours)
#         gp_hbm (max walltime of 3 days)
qos_pipeline = 'gp_resa'
qos_stage = 'gp_resa'

#define the chunk size to process the GHOST pipeline tasks (i.e. how many pipelines can be processed simultaneously)
pipeline_chunk_size = 15

#define the chunk size to process GHOST stage tasks (i.e. the size of CPU chunks used to process stage tasks)
#this should not exceed the number of CPUs per node on machine (112 physical cores on Marenostrum5)
stage_chunk_size = 55

#define the maximum walltime to process the pipeline/stages
#if 'default' is set the maximum allowed walltime for the qos is taken
#pipeline_max_walltime = 'default'
#stage_max_walltime = 'default'
pipeline_max_walltime = '06:00:00'
stage_max_walltime = '06:00:00'

#define job array limit to process GHOST stage tasks (i.e. the number of packed chunks allowed to be submitted simultaneously)
job_array_limit = 100

#define the maximum number of stations than can be should be processed for network
max_N_stations = 10000

#define if previous stage temporary files should be removed after stage completion
remove_temporary_data_files_after_stage = False

#define compression type to compress netCDF files
#OPTIONS: zlib, szip, zstd, bzip2, blosc_lz, blosc_lz4, blosc_lz4hc, blosc_zlib, blosc_zstd
compression_type = 'zlib'

#define compression level to compress netCDF files (0-9)
#1 being fastest and lowest compression ratio, 9 being slowest and highest compression ratio
#0 turns off compression
#should only be set for 'zlib', 'bzip' and 'zstd' (otherwise set as None)
compression_level = 1

#turn netCDF shuffle on or off
#this will de-interlace a block of data before compression by reordering the bytes
#should only be 'True' if compression type is 'zlib'
compression_shuffle = True

#define if need to use high memory nodes to process S1 tasks (where memory bottleneck can occur). Only valid on MN5
high_memory_S1 = False

#define if need to use high memory nodes to process T2 tasks (where memory bottleneck can occur). Only valid on MN5
high_memory_T2 = False

#define if there is an error .out file created for submitted process
error_out_file = True

#define if to turn on debug mode, where just one job within each stage is processed (e.g. one station in S1) 
debug_GHOST = False 

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###
#START_DATE = YYYYMM (start year of processed data file - should be fixed as '197001')
start_date = '197001'

#END_DATE   = YYYYMM (end year of processed data file - should be first month of current year)
end_date = '202501'

# NETWORKS (list of networks whose data you wish to process)
# default = all networks
# VALID OPTIONS: AERONET_v3_lev1.5, AERONET_v3_lev2.0, BJMEMC, CANADA_NAPS, CAPMoN, CHILE_SINCA, CNEMC, EANET, EBAS, 
#                EEA, INDAAF, JAPAN_NIES, MEXICO_CDMX, MITECO, NAPMD, NOAA_ISD, NZ_ECAN, NZ_LAWA, UK_AIR, 
#                US_EPA_AirNow_DOS, US_EPA_AQS, US_EPA_CASTNET, US_NADP_AIRMoN, US_NADP_AMNet, US_NADP_AMoN, 
#                US_NADP_MDN, US_NADP_NTN, WMO_WDCGG, WMO_WDCPC
#networks = 'default'
#FOCI networks
#networks = ['AERONET_v3_lev1.5', 'AERONET_v3_lev2.0', 'EBAS']
networks = ['EEA']

# MERGE NETWORKS (list of networks to merge other network data)
# default = all networks
# VALID OPTIONS: GHOST, GHOST-PUBLIC
merge_networks = 'default'

# PARAMETERS (list of parameters (GHOST or BSC name), or matrices that you wish to process, or default per network)
# default = all relevant parameters per stage
#parameters = 'default'
#FOCI and precip parameters
#parameters = ['acprec','precip','wetdep','AERO_ABS_COEFF_370nm','AERO_ABS_COEFF_467nm','AERO_ABS_COEFF_470nm','AERO_ABS_COEFF_520nm','AERO_ABS_COEFF_522nm','AERO_ABS_COEFF_525nm','AERO_ABS_COEFF_528nm','AERO_ABS_COEFF_530nm','AERO_ABS_COEFF_550nm','AERO_ABS_COEFF_565nm','AERO_ABS_COEFF_590nm','AERO_ABS_COEFF_637nm','AERO_ABS_COEFF_652nm','AERO_ABS_COEFF_660nm','AERO_ABS_COEFF_670nm','AERO_ABS_COEFF_880nm','AERO_ABS_COEFF_950nm','ABSAE_370-880nm','ABSAE_370-950nm','ABSAE_467-652nm','ABSAE_467-660nm','AERO_LIGHT_BACKSCAT_COEFF_450nm','AERO_LIGHT_BACKSCAT_COEFF_520nm','AERO_LIGHT_BACKSCAT_COEFF_525nm','AERO_LIGHT_BACKSCAT_COEFF_530nm','AERO_LIGHT_BACKSCAT_COEFF_532nm','AERO_LIGHT_BACKSCAT_COEFF_550nm','AERO_LIGHT_BACKSCAT_COEFF_635nm','AERO_LIGHT_BACKSCAT_COEFF_700nm','AERO_LIGHT_BACKSCAT_COEFF_850nm','AERO_LIGHT_SCAT_COEFF_450nm','AERO_LIGHT_SCAT_COEFF_520nm','AERO_LIGHT_SCAT_COEFF_525nm','AERO_LIGHT_SCAT_COEFF_530nm','AERO_LIGHT_SCAT_COEFF_532nm','AERO_LIGHT_SCAT_COEFF_550nm','AERO_LIGHT_SCAT_COEFF_635nm','AERO_LIGHT_SCAT_COEFF_700nm','AERO_LIGHT_SCAT_COEFF_850nm','SCATAE_450-635nm','SCATAE_450-700nm','BACKSCAT_FRAC_450nm','BACKSCAT_FRAC_520nm','BACKSCAT_FRAC_525nm','BACKSCAT_FRAC_530nm','BACKSCAT_FRAC_532nm','BACKSCAT_FRAC_550nm','BACKSCAT_FRAC_635nm','BACKSCAT_FRAC_700nm','BACKSCAT_FRAC_850nm','ASY_525nm','ASY_550nm','BC_MAC_637nm','BC_MAC_880nm','SSA_470nm','SSA_520nm','SSA_590nm','SSA_637nm','SSA_660nm',                      
#              'PM10_AERO_ABS_COEFF_370nm','PM10_AERO_ABS_COEFF_467nm','PM10_AERO_ABS_COEFF_470nm','PM10_AERO_ABS_COEFF_520nm','PM10_AERO_ABS_COEFF_522nm','PM10_AERO_ABS_COEFF_525nm','PM10_AERO_ABS_COEFF_528nm','PM10_AERO_ABS_COEFF_530nm','PM10_AERO_ABS_COEFF_550nm','PM10_AERO_ABS_COEFF_565nm','PM10_AERO_ABS_COEFF_590nm','PM10_AERO_ABS_COEFF_637nm','PM10_AERO_ABS_COEFF_652nm','PM10_AERO_ABS_COEFF_660nm','PM10_AERO_ABS_COEFF_670nm','PM10_AERO_ABS_COEFF_880nm','PM10_AERO_ABS_COEFF_950nm','PM10_ABSAE_370-880nm','PM10_ABSAE_370-950nm','PM10_ABSAE_467-652nm','PM10_ABSAE_467-660nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM10_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM10_AERO_LIGHT_SCAT_COEFF_450nm','PM10_AERO_LIGHT_SCAT_COEFF_520nm','PM10_AERO_LIGHT_SCAT_COEFF_525nm','PM10_AERO_LIGHT_SCAT_COEFF_530nm','PM10_AERO_LIGHT_SCAT_COEFF_532nm','PM10_AERO_LIGHT_SCAT_COEFF_550nm','PM10_AERO_LIGHT_SCAT_COEFF_635nm','PM10_AERO_LIGHT_SCAT_COEFF_700nm','PM10_AERO_LIGHT_SCAT_COEFF_850nm','PM10_SCATAE_450-635nm','PM10_SCATAE_450-700nm','PM10_BACKSCAT_FRAC_450nm','PM10_BACKSCAT_FRAC_520nm','PM10_BACKSCAT_FRAC_525nm','PM10_BACKSCAT_FRAC_530nm','PM10_BACKSCAT_FRAC_532nm','PM10_BACKSCAT_FRAC_550nm','PM10_BACKSCAT_FRAC_635nm','PM10_BACKSCAT_FRAC_700nm','PM10_BACKSCAT_FRAC_850nm','PM10_ASY_525nm','PM10_ASY_550nm','PM10_BC_MAC_637nm','PM10_BC_MAC_880nm','PM10_SSA_470nm','PM10_SSA_520nm','PM10_SSA_590nm','PM10_SSA_637nm','PM10_SSA_660nm',                      
#              'PM2.5_AERO_ABS_COEFF_370nm','PM2.5_AERO_ABS_COEFF_467nm','PM2.5_AERO_ABS_COEFF_470nm','PM2.5_AERO_ABS_COEFF_520nm','PM2.5_AERO_ABS_COEFF_522nm','PM2.5_AERO_ABS_COEFF_525nm','PM2.5_AERO_ABS_COEFF_528nm','PM2.5_AERO_ABS_COEFF_530nm','PM2.5_AERO_ABS_COEFF_550nm','PM2.5_AERO_ABS_COEFF_565nm','PM2.5_AERO_ABS_COEFF_590nm','PM2.5_AERO_ABS_COEFF_637nm','PM2.5_AERO_ABS_COEFF_652nm','PM2.5_AERO_ABS_COEFF_660nm','PM2.5_AERO_ABS_COEFF_670nm','PM2.5_AERO_ABS_COEFF_880nm','PM2.5_AERO_ABS_COEFF_950nm','PM2.5_ABSAE_370-880nm','PM2.5_ABSAE_370-950nm','PM2.5_ABSAE_467-652nm','PM2.5_ABSAE_467-660nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM2.5_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM2.5_AERO_LIGHT_SCAT_COEFF_450nm','PM2.5_AERO_LIGHT_SCAT_COEFF_520nm','PM2.5_AERO_LIGHT_SCAT_COEFF_525nm','PM2.5_AERO_LIGHT_SCAT_COEFF_530nm','PM2.5_AERO_LIGHT_SCAT_COEFF_532nm','PM2.5_AERO_LIGHT_SCAT_COEFF_550nm','PM2.5_AERO_LIGHT_SCAT_COEFF_635nm','PM2.5_AERO_LIGHT_SCAT_COEFF_700nm','PM2.5_AERO_LIGHT_SCAT_COEFF_850nm','PM2.5_SCATAE_450-635nm','PM2.5_SCATAE_450-700nm','PM2.5_BACKSCAT_FRAC_450nm','PM2.5_BACKSCAT_FRAC_520nm','PM2.5_BACKSCAT_FRAC_525nm','PM2.5_BACKSCAT_FRAC_530nm','PM2.5_BACKSCAT_FRAC_532nm','PM2.5_BACKSCAT_FRAC_550nm','PM2.5_BACKSCAT_FRAC_635nm','PM2.5_BACKSCAT_FRAC_700nm','PM2.5_BACKSCAT_FRAC_850nm','PM2.5_ASY_525nm','PM2.5_ASY_550nm','PM2.5_BC_MAC_637nm','PM2.5_BC_MAC_880nm','PM2.5_SSA_470nm','PM2.5_SSA_520nm','PM2.5_SSA_590nm','PM2.5_SSA_637nm','PM2.5_SSA_660nm',                      
#              'PM1_AERO_ABS_COEFF_370nm','PM1_AERO_ABS_COEFF_467nm','PM1_AERO_ABS_COEFF_470nm','PM1_AERO_ABS_COEFF_520nm','PM1_AERO_ABS_COEFF_522nm','PM1_AERO_ABS_COEFF_525nm','PM1_AERO_ABS_COEFF_528nm','PM1_AERO_ABS_COEFF_530nm','PM1_AERO_ABS_COEFF_550nm','PM1_AERO_ABS_COEFF_565nm','PM1_AERO_ABS_COEFF_590nm','PM1_AERO_ABS_COEFF_637nm','PM1_AERO_ABS_COEFF_652nm','PM1_AERO_ABS_COEFF_660nm','PM1_AERO_ABS_COEFF_670nm','PM1_AERO_ABS_COEFF_880nm','PM1_AERO_ABS_COEFF_950nm','PM1_ABSAE_370-880nm','PM1_ABSAE_370-950nm','PM1_ABSAE_467-652nm','PM1_ABSAE_467-660nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_450nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_520nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_525nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_530nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_532nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_550nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_635nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_700nm','PM1_AERO_LIGHT_BACKSCAT_COEFF_850nm','PM1_AERO_LIGHT_SCAT_COEFF_450nm','PM1_AERO_LIGHT_SCAT_COEFF_520nm','PM1_AERO_LIGHT_SCAT_COEFF_525nm','PM1_AERO_LIGHT_SCAT_COEFF_530nm','PM1_AERO_LIGHT_SCAT_COEFF_532nm','PM1_AERO_LIGHT_SCAT_COEFF_550nm','PM1_AERO_LIGHT_SCAT_COEFF_635nm','PM1_AERO_LIGHT_SCAT_COEFF_700nm','PM1_AERO_LIGHT_SCAT_COEFF_850nm','PM1_SCATAE_450-635nm','PM1_SCATAE_450-700nm','PM1_BACKSCAT_FRAC_450nm','PM1_BACKSCAT_FRAC_520nm','PM1_BACKSCAT_FRAC_525nm','PM1_BACKSCAT_FRAC_530nm','PM1_BACKSCAT_FRAC_532nm','PM1_BACKSCAT_FRAC_550nm','PM1_BACKSCAT_FRAC_635nm','PM1_BACKSCAT_FRAC_700nm','PM1_BACKSCAT_FRAC_850nm','PM1_ASY_525nm','PM1_ASY_550nm','PM1_BC_MAC_637nm','PM1_BC_MAC_880nm','PM1_SSA_470nm','PM1_SSA_520nm','PM1_SSA_590nm','PM1_SSA_637nm','PM1_SSA_660nm',                      
#              'AOD_550nm','EXTAOD_550nm','ABSAOD_550nm','ABSAE_440-675nm','ABSAE_675-870nm','EXTAE_440-675nm','SCATAE_440-675nm','SSA_550nm','SSAAE_440-675nm','ABSWDAE','BC_ABSAE_440-870nm','BC_ABSAOD_440nm','BrC_ABSAOD_440nm','SCHUSTER_FILTER_BC_BrC','SCHUSTER_FILTER_SS',
#              'PM10_AOD_550nm','PM10_EXTAOD_550nm','PM10_ABSAOD_550nm','PM10_ABSAE_440-675nm','PM10_ABSAE_675-870nm','PM10_EXTAE_440-675nm','PM10_SCATAE_440-675nm','PM10_SSA_550nm','PM10_SSAAE_440-675nm','PM10_ABSWDAE','PM10_BC_ABSAE_440-870nm','PM10_BC_ABSAOD_440nm','PM10_BrC_ABSAOD_440nm','PM10_SCHUSTER_FILTER_BC_BrC','PM10_SCHUSTER_FILTER_SS',
#              'PM2.5_AOD_550nm','PM2.5_EXTAOD_550nm','PM2.5_ABSAOD_550nm','PM2.5_ABSAE_440-675nm','PM2.5_ABSAE_675-870nm','PM2.5_EXTAE_440-675nm','PM2.5_SCATAE_440-675nm','PM2.5_SSA_550nm','PM2.5_SSAAE_440-675nm','PM2.5_ABSWDAE','PM2.5_BC_ABSAE_440-870nm','PM2.5_BC_ABSAOD_440nm','PM2.5_BrC_ABSAOD_440nm','PM2.5_SCHUSTER_FILTER_BC_BrC','PM2.5_SCHUSTER_FILTER_SS',
#              'PM1_AOD_550nm','PM1_EXTAOD_550nm','PM1_ABSAOD_550nm','PM1_ABSAE_440-675nm','PM1_ABSAE_675-870nm','PM1_EXTAE_440-675nm','PM1_SCATAE_440-675nm','PM1_SSA_550nm','PM1_SSAAE_440-675nm','PM1_ABSWDAE','PM1_BC_ABSAE_440-870nm','PM1_BC_ABSAOD_440nm','PM1_BrC_ABSAOD_440nm','PM1_SCHUSTER_FILTER_BC_BrC','PM1_SCHUSTER_FILTER_SS']
parameters = ['acprec','precip','wetdep']

#TEMPORAL_RESOLUTIONS = H, D, M, HI (temporal resolution desired of merged output netCDF files: hourly, daily, monthly, hourly instantaneous)       
#default = all temporal resolutions for instantaneous measured parameters, and H, D, M for all others
temporal_resolutions = 'default'

#REGIONS = GLOBAL, EU, IP, NA (this works by selectively keeping data within a defined rectangle per region)
#default = all regions for NOAA_ISD network, GLOBAL for all others
regions = 'default'

#STAGES_TO_PROCESS (list of GHOST stages that you wish to process)
#default = if all default networks are being processed, this is all stages except C1 and Z1
#          if a selection of networks is being processed, this is all stages except M1, C1 and Z1

#PARSING OF NETWORK DATA FILES PARSING TO STATION SPECIFIC .npz FILES
#  S1 = Raw network file parse
#  S2 = Concatenate parsed station data files / Split stations by significantly changing locational information
#  S3 = Add globally gridded metadata / Quality control / Add temporal classification codes

#TEMPORAL AVERAGING / AGGREGATION OF STATIONS TO netCDF
#  T1 = Temporally averaging by station
#  T2 = Aggregate temporally averaged station data in monthly netCDFs

#DATA PRODUCT
#  P1 = Create data product from other processed components

#SPLIT EBAS
#  E1 = Split out EBAS data into individual native networks

#MERGE 
#  M1 = Merge data across networks

#COMPRESS
#  C1 = Compress processed GHOST netCDFs

#ZIP
#  Z1 = Zip compressed netCDFs per network

#stages_to_process = 'default'
#stages_to_process = ['M1','C1','Z1']
stages_to_process = ['S1']
#stages_to_process = ['S1','S2','S3','T1','T2']
#stages_to_process = ['S1','S2','S3','T1','T2','P1','M1','C1','Z1']
###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###
