# script by Calum


import pandas as pd
import os
import sys
import requests
from EIONET import FormatEIONET
from datetime import datetime


date = datetime.today()
print(date)

datesdf = [(date - pd.Timedelta(days=2)).strftime("%Y%m%d"),
                (date - pd.Timedelta(days=1)).strftime("%Y%m%d"),
                (date).strftime("%Y%m%d")]

# Set download url
ServiceUrl = "http://discomap.eea.europa.eu/map/fme/latest"

# Destination directory
destdir = "/esarchive/obs/eea/eionet/original_files"
tempdir = destdir + '/tmp/'
print(destdir)

# Countries to download
countries = ['AD', 'AT', 'BA', 'BE', 'BG', 'CH', 'CY', 'CZ', 'DE', 'DK','EE', 'ES', 'FI', 'FR', 'GE', 'GB', 'GI', 'GR', 
             'HR', 'HU', 'IE', 'IS', 'IT', 'LT', 'LU', 'LV', 'ME', 'MK', 'MT', 'NL', 'NO', 
             'PL', 'PT', 'RO', 'RS', 'SE', 'SI', 'SK', 'XK', 'TR', 'LI', 'AL', 'UA'] #Station list updated calum Feb 2023 https://www.eea.europa.eu/data-and-maps/daviz/eionet/data/eionet-data/list-eurostat-eea-and-fao-country-codes-side-by-side/daviz.json
#countries = ['AD']

# Pollutant to be downloaded
pollutants = ['NO','PM10','CO','NO2','SO2','O3','PM2.5']
#pollutants = ['O3', 'NO2', 'PM2.5']

dfs = []

# Download each pollutant for each country and add them to a list of dataframes
for country in countries:
    for pollutant in pollutants:
        downloadFile = '%s/%s_%s.csv' % (ServiceUrl, country, pollutant)
        print('Downloading: %s' % downloadFile )
        try:
            df=pd.read_csv(downloadFile)
            print("Worked")
            print ('Download finished')
        except:
            print("Didn't work - file corrupted")
        print("------")
        dfs.append(df)
        

# Combine all dfs into one big csv for all the latest data
print ('Combining files')
finalfile = tempdir + '%s-%s.csv' % (datesdf[0], datesdf[2] )
concat_df = pd.concat(dfs)

concat_df.to_csv(finalfile, index=False)


columns_rep = ['network_countrycode', 'pollutant', 'samplingpoint_x', 
                        'samplingpoint_y', 'station_code', 'station_name',
                        'value_datetime_begin', 'value_datetime_end',
                        'station_altitude', 'value_unit']

df = pd.read_csv(finalfile, engine='python', header=0,index_col=False).set_index('value_datetime_begin', drop = False)

df.index = pd.to_datetime(df.index).to_pydatetime()


# After reloading that csv (Not sure if we need to do that, might be good to create a tmp file anyway)
for date in datesdf:
    thefile = destdir +'/' + str(date) + '.csv'
    print(thefile)
    # If the individual date csv files exist already in esarchive then append extra data
    # And remove duplicates
    if os.path.exists(thefile):
        print("Updating existing file")
        print(date)
        dfold = pd.read_csv(thefile, engine='python', header=0,index_col=False )
        dfnew = dfold.append(df[str(date)], ignore_index = True)

        dfnew = dfnew.drop_duplicates(subset=columns_rep, keep="first")
    # If the date file doesn't exist then creat a new one from df
    else:
        print("Creating new file")
        dfnew = df[str(date)]
        print(dfnew)

    
    dfnew.to_csv(thefile, index = False)

###Formatting
cl = FormatEIONET(datesdf)
cl.retrieve_stations()
cl.retrieve_eionet_data()
cl.write_netcdf()
        
print('Finish!')