import datetime

# global download configuration ==============================================================================

# modes: 'all': download all the available files
#        'nrt': download data that includes last x days
mode = 'nrt'

# when nrt mode, provide time period in which data should be downloaded; if mode == 'all' these dates will be overwritten
latency_days = 7
start_date = datetime.date.today() - datetime.timedelta(days = latency_days)
stop_date = datetime.date.today()

# ghost version
version = '1.6'

# want to download data?    
dl_data = True

# want to download meta data? 
dl_metadata = False 

# meta data parsing configuration, only applies if dl_metadata is True
# True: re creates json basefile from given input network provided metafile and adds current downloaded metadata to basefile; Careful, all intermediate meta timestamps will be gone
# False: does not recreate base file and only adds current metadata file to existing basefile
meta_basefile = False

# do you want to parse all meta files ever downloaded to get full meta history (True) or only base file and latest meta file (False)?
meta_history = False

# number of cpus
num_cpu = 2

# download location
root = '/esarchive/obs/ghost'

# path to save log files
path_logfiles = '/esarchive/scratch/rgrodofz/software/ghost/GHOST/download_scripts/log_files'


# download parameters per network =============================================================================
# times in minutes
download_params = {'AERONET_v3_lev1.5':        {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 10,   'meta_params': {}},
                   'AERONET_v3_lev2.0':        {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 10,   'meta_params': {}},
                   'BJMEMC':                   {'max_time_dl': 3,      'max_number_tries': 3,      'total_timeout': 15,   'meta_params': {'delimiter_meta_csv': ';', 'identifier': 'ghost_station_reference',             'base_file_id': '20131201',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitude'], 'Longitude': ['longitude']}},
                   'CANADA_NAPS':              {'max_time_dl': 5,      'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'NAPS_ID',                             'base_file_id': '20240423',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['Latitude'], 'Longitude': ['Longitude'], 'Altitude': ['Elevation'], 'Height': ['Inlet_Height']}},
                   'CAPMoN':                   {'max_time_dl': 5,      'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'create_CAPMoN_identifier',            'base_file_id': '20240424',    'network_provided_update_time': False,  'encoding': 'ISO-8859-1',  'Latitude': ['Latitude '], 'Longitude': ['Longitude '], 'Altitude': ['Elevation_Altitude_m']}},
                   'CHILE_SINCA':              {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 90,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'station_reference',                   'base_file_id': '20240725',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['UTM_lat'], 'Longitude': ['UTM_lon']}},
                   'CNEMC':                    {'max_time_dl': 3,      'max_number_tries': 3,      'total_timeout': 15,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'station_reference',                   'base_file_id': '2016',        'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitude'], 'Longitude': ['longitude']}},                  
                   'EANET':                    {'max_time_dl': 5,      'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'Code',                                'base_file_id': '20240507',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['Latitude_degree','Latitude_minute','Latitude_second'], 'Longitude': ['Longitude_degree','Longitude_minute','Longitude_second'], 'Altitude': ['Altitude']}},                     
                   'EBAS':                     {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {}},             
                   'EEA_AIRBASE':              {'max_time_dl': 20,     'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'create_EEA_AIRBASE_identifier',       'base_file_id': '20240825',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['station_latitude_decimal_degrees'], 'Longitude': ['station_longitude_decimal_degrees'], 'Altitude': ['station_altitude'], 'Height': ['height_sampling_point']}},     
                   'EEA_AQ_eReporting':        {'max_time_dl': 20,     'max_number_tries': 3,      'total_timeout': 15,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'create_EEA_AQ_eReporting_identifier', 'base_file_id': '20240620',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['Latitude'], 'Longitude': ['Longitude'], 'Altitude': ['Altitude'], 'Height': ['Inlet Height']}},
                   'JAPAN_NIES':               {'max_time_dl': 60,     'max_number_tries': 3,      'total_timeout': 15,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'NIES_area_code',                      'base_file_id': '1972',        'network_provided_update_time': False,  'encoding': 'ISO-8859-1',  'Latitude': ['latitude_degree','latitude_minute','latitude_second'], 'Longitude': ['longitude_degree','longitude_minute','longitude_second'], 'Altitude': ['altitude(m)'] }},
                   'MEXICO_CDMX':              {'max_time_dl': 240,    'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'cve_estac',                           'base_file_id': '20240425',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitud'], 'Longitude': ['longitud'], 'Altitude': ['alt']}},
                   'MITECO':                   {'max_time_dl': 10,     'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ';', 'identifier': 'COD_ESTACION_DEM',                    'base_file_id': '2001',        'network_provided_update_time': False,  'encoding': 'ISO-8859-1',  'Latitude': ['LATITUD_G'], 'Longitude': ['LONGITUD_G'], 'Altitude': ['ALTITUD']}},
                   'NAPMD':                    {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 180,  'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'station_id',                          'base_file_id': '20240919',    'network_provided_update_time': True,   'encoding': 'utf-8',       'Latitude': ['Latitude'], 'Longitude': ['Longitude'], 'Altitude': ['Altitude']}},               
                   'NOAA_ISD':                 {'max_time_dl': 15,     'max_number_tries': 3,      'total_timeout': 60,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'create_NOAA_ISD_identifier',          'base_file_id': '20240426',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['LAT'], 'Longitude': ['LON'], 'Altitude': ['ELEV(M)']}},                   
                   'NZ_ECAN':                  {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 10,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'create_NZ_ECAN_identifier',           'base_file_id': '20240617',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['StationLatitude'], 'Longitude': ['StationLongitude']}},
                   'NZ_LAWA':                  {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {}},
                   'UK_AIR':                   {'max_time_dl': 60,     'max_number_tries': 3,      'total_timeout': 30,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'Sampling Point ID',                   'base_file_id': '20240527',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['Latitude'], 'Longitude': ['Longitude'], 'Altitude': ['Altitude'], 'Height': ['Inlet Height (m)']}},
                   'US_EPA_AirNow_DOS':        {'max_time_dl': 10,     'max_number_tries': 3,      'total_timeout': 10,   'meta_params': {}},
                   'US_EPA_AQS':               {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'create_US_EPA_AQS_identifier',        'base_file_id': '20240729',    'network_provided_update_time': False, 'encoding': 'utf-8',       'Latitude': ['Latitude'], 'Longitude': ['Longitude'], 'Altitude': ['Elevation']}},
                   'US_EPA_CASTNET':           {'max_time_dl': 10,     'max_number_tries': 3,      'total_timeout': 10,   'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'SITE_ID',                             'base_file_id': '20240419',    'network_provided_update_time': True,  'encoding': 'utf-8',       'Latitude': ['LATITUDE'], 'Longitude': ['LONGITUDE'], 'Altitude': ['ELEVATION']}},
                   'US_NADP_AIRMoN':           {'max_time_dl': 7,      'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'siteid',                              'base_file_id': '20240614',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitude'], 'Longitude': ['longitude'], 'Altitude': ['elevation']}},
                   'US_NADP_AMNet':            {'max_time_dl': 10,     'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'siteId',                              'base_file_id': '20240424',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitude'], 'Longitude': ['longitude'], 'Altitude': ['elevation']}},
                   'US_NADP_AMoN':             {'max_time_dl': 7,      'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'siteId',                              'base_file_id': '20240425',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitude'], 'Longitude': ['longitude'], 'Altitude': ['elevation']}},
                   'US_NADP_MDN':              {'max_time_dl': 7,      'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'siteId',                              'base_file_id': '20240613',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitude'], 'Longitude': ['longitude'], 'Altitude': ['elevation']}},              
                   'US_NADP_NTN':              {'max_time_dl': 7,      'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': 'siteId',                              'base_file_id': '20240613',    'network_provided_update_time': False,  'encoding': 'utf-8',       'Latitude': ['latitude'], 'Longitude': ['longitude'], 'Altitude': ['elevation']}},
                   'WMO_WDCGG':                {'max_time_dl': 60,     'max_number_tries': 3,      'total_timeout': 45,   'meta_params': {}},
                   'WMO_WDCPC':                {'max_time_dl': 30,     'max_number_tries': 3,      'total_timeout': 1,    'meta_params': {'delimiter_meta_csv': ',', 'identifier': None,                                  'base_file_id': '',            'network_provided_update_time': True,   'encoding': 'utf-8',       'Latitude': ['Latitude'], 'Longitude': ['Longitude'], 'Altitude': ['Altitude']}}
                  }

# networks you want to download
networks_to_download = ['all']
#networks_to_download = ['WMO_WDCGG']

all_networks = ['AERONET_v3_lev1.5', # ok
                'AERONET_v3_lev2.0', 
                'BJMEMC', # ok
                'CANADA_NAPS', # ok
                'CAPMoN', # ok
                'CHILE_SINCA', # ok
                'CNEMC', # ok
                'EANET', # ok
                'EBAS', # ok
                'EEA_AQ_eReporting', 
                'JAPAN_NIES',
                'MEXICO_CDMX', # many tries needed before no ConnectionError, now ok
                'MITECO', # ok
                'NAPDM', # ok
                'NOAA_ISD', # ok, takes very long
                'NZ_ECAN', # ok
                'NZ_LAWA',
                'UK_AIR', 
                'US_EPA_AirNow_DOS', # downloads files double with request of random years
                'US_EPA_AQS',  
                'US_EPA_CASTNET', # ok
                'US_NADP_AIRMoN', # ok
                'US_NADP_AMNet', # ok
                'US_NADP_AMoN', # ok
                'US_NADP_MDN', # ok
                'US_NADP_NTN', # ok
                'WMO_WDCGG', # ok
                'WMO_WDCPC'
                ] 

all_nrt_networks = ['AERONET_v3_lev1.5', # went fine, download only last day
                    'AERONET_v3_lev2.0', # went fine, download only last day
                    'BJMEMC', # went fine, download only last day
                    'CHILE_SINCA', # ???, download last 30 days
                    'CNEMC', # went fine, download only last day
                    'EEA_AQ_eReporting', # went fine, E2a stream from 2023-01-01 until now ### new download: only last 48 hours available, always download those
                    'JAPAN_NIES', # went fine, download last 2 months to always cover 30 days
                    'NOAA_ISD', # went fine, download 2024
                    'NZ_ECAN',
                    'UK_AIR', # went fine, download 2024; only one stream 'auto' -> corrected, now all streams
                    'US_EPA_AirNow_DOS', # download last 2 months to alwyas cover 30 days; monthly files
                    'US_EPA_CASTNET', # download 2024; error occured randomly: requests.exceptions.ConnectionError: HTTPSConnectionPool(host='gaftp.epa.gov', port=443): Max retries exceeded with url: /castnet/CASTNET_Outgoing/data/ozone_2024.zip (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f54f0e7f3d0>: Failed to establish a new connection: [Errno -2] Name or service not known'))           
                    'WMO_WDCGG' # downloads all data in nrt
                    ] 