import os

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###

#GHOST Processing Configuration File 
ghost_version = '1.5.1'

#define the data root directory (i.e. where the observations are stored)
data_root = '/gpfs/projects/bsc32/AC_cache/obs/ghost/'
#data_root = '/gpfs/scratch/bsc32/bsc032715/GHOST_data/'

#define the processing root directory (i.e. where the processing is being run from)
process_root = os.getcwd().split('/run')[0]

#define the compress root directory (i.e. where the data is compressed per network)
compress_root = '/gpfs/projects/bsc32/AC_cache/obs/ghost/delivered_data/PUBLIC/'

#define the BSC account to do the processing on
#OPTIONS: bsc32, cns101 -> this is for prace account
bsc_account = 'cns109' 

#define the QOS (Quality of Service) used to manage jobs on the SLURM system
#it is possible to define the qos of the pipeline/stage levels of submission
#OPTIONS: gp_bsces (max walltime of 2 days), gp_resa (max walltime of 3 days), gp_debug (max walltime of 2 hours)
#         gp_hbm (max walltime of 3 days)
qos_pipeline = 'gp_resa'
qos_stage = 'gp_resa'

#define the chunk size to process the GHOST pipeline tasks (i.e. how many pipelines can be processed simultaneously)
pipeline_chunk_size = 25

#define the chunk size to process GHOST stage tasks (i.e. the size of CPU chunks used to process stage tasks)
#this should not exceed the number of CPUs per node on machine (112 physical cores on Marenostrum5)
stage_chunk_size = 55

#define the maximum walltime to process the pipeline/stages
#if 'default' is set the maximum allowed walltime for the qos is taken
pipeline_max_walltime = 'default'
stage_max_walltime = 'default'
#pipeline_max_walltime = '02:00:00'
#stage_max_walltime = '06:00:00'

#define job array limit to process GHOST stage tasks (i.e. the number of packed chunks allowed to be submitted simultaneously)
job_array_limit = 100

#define the maximum number of stations than can be should be processed for network
max_N_stations = 10000

#define if previous stage temporary files should be removed after stage completion
remove_temporary_data_files_after_stage = False

#define compression type to compress netCDF files
#OPTIONS: zlib, szip, zstd, bzip2, blosc_lz, blosc_lz4, blosc_lz4hc, blosc_zlib, blosc_zstd
compression_type = 'zlib'

#define compression level to compress netCDF files (0-9)
#1 being fastest and lowest compression ratio, 9 being slowest and highest compression ratio
#0 turns off compression
#should only be set for 'zlib', 'bzip' and 'zstd' (otherwise set as None)
compression_level = 1

#turn netCDF shuffle on or off
#this will de-interlace a block of data before compression by reordering the bytes
#should only be 'True' if compression type is 'zlib'
compression_shuffle = True

#define if need to use high memory nodes to process S1 tasks (where memory bottleneck can occur). Only valid on MN5
high_memory_S1 = False

#define if need to use high memory nodes to process T2 tasks (where memory bottleneck can occur). Only valid on MN5
high_memory_T2 = False

#define if there is an error .out file created for submitted process
error_out_file = True

#define if to turn on debug mode, where just one job within each stage is processed (e.g. one station in S1) 
debug_GHOST = False 

###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###
#START_DATE = YYYYMM (start year of processed data file - should be fixed as '197001')
start_date = '197001'

#END_DATE   = YYYYMM (end year of processed data file - should be first month of current year)
end_date = '202501'

# NETWORKS (list of networks whose data you wish to process)
# default = all networks
# VALID OPTIONS: AERONET_v3_lev1.5, AERONET_v3_lev2.0, BJMEMC, CANADA_NAPS, CAPMoN, CHILE_SINCA, CNEMC, EANET, EBAS, 
#                EEA_AIRBASE, EEA_AQ_eReporting, INDAAF, JAPAN_NIES, MEXICO_CDMX, MITECO, NAPMD, NOAA_ISD, NZ_ECAN, 
#                NZ_LAWA, UK_AIR, US_EPA_AirNow_DOS, US_EPA_AQS, US_EPA_CASTNET, US_NADP_AIRMoN, US_NADP_AMNet, 
#                US_NADP_AMoN, US_NADP_MDN, US_NADP_NTN, WMO_WDCGG, WMO_WDCPC
#networks = 'default'
# precip networks
networks = ['CANADA_NAPS', 'CAPMoN', 'EANET', 'EBAS','EEA_AQ_eReporting', 'INDAAF', 'UK_AIR','US_EPA_AQS', 'US_EPA_CASTNET', 'US_NADP_AIRMoN', 'US_NADP_MDN', 'US_NADP_NTN', 'WMO_WDCPC']
#networks = ['CANADA_NAPS']
# MERGE NETWORKS (list of networks to merge other network data)
# default = all networks
# VALID OPTIONS: GHOST, GHOST-PUBLIC
merge_networks = 'default'

# PARAMETERS (list of parameters (GHOST or BSC name), or matrices that you wish to process, or default per network)
# default = all relevant parameters per stage
#parameters = 'default'
parameters = ['acprec', 'precip', 'wetdep']

#TEMPORAL_RESOLUTIONS = H, D, M, HI (temporal resolution desired of merged output netCDF files: hourly, daily, monthly, hourly instantaneous)       
#default = all temporal resolutions for instantaneous measured parameters, and H, D, M for all others
temporal_resolutions = 'default'

#REGIONS = GLOBAL, EU, IP, NA (this works by selectively keeping data within a defined rectangle per region)
#default = all regions for NOAA_ISD network, GLOBAL for all others
regions = 'default'

#STAGES_TO_PROCESS (list of GHOST stages that you wish to process)
#default = if all default networks are being processed, this is all stages except C1 and Z1
#          if a selection of networks is being processed, this is all stages except M1, C1 and Z1

#PARSING OF NETWORK DATA FILES PARSING TO STATION SPECIFIC .npz FILES
#  S1 = Raw network file parse
#  S2 = Concatenate parsed station data files / Split stations by significantly changing locational information
#  S3 = Add globally gridded metadata / Quality control / Add temporal classification codes

#TEMPORAL AVERAGING / AGGREGATION OF STATIONS TO netCDF
#  T1 = Temporally averaging by station
#  T2 = Aggregate temporally averaged station data in monthly netCDFs

#DATA PRODUCT
#  P1 = Create data product from other processed components

#SPLIT EBAS
#  E1 = Split out EBAS data into individual native networks

#MERGE 
#  M1 = Merge data across networks

#COMPRESS
#  C1 = Compress processed GHOST netCDFs

#ZIP
#  Z1 = Zip compressed netCDFs per network

#stages_to_process = 'default'
stages_to_process = ['S1', 'S2', 'S3', 'T1', 'T2']
#stages_to_process = ['S1']


stages_to_process = 'default'
###--------------------------------------------------------------------------------------------------###
###--------------------------------------------------------------------------------------------------###
