# use API
import requests
import pandas as pd

import os
import sys
import logging
import urllib
import datetime as dt
import json
import csv
from pathlib import Path
from datetime import date
from datetime import timedelta
import zipfile
import time
import shutil
import metadata_parsing_functions

from selenium import webdriver
from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException
from bs4 import BeautifulSoup


class eea_ereporting_api(object):
    """defines and configures API to query the eea database"""

    def __init__(self, ):
        self.logger = logging.getLogger(__file__)
        self.url_api = "https://eeadmz1-downloads-api-appservice.azurewebsites.net/"
        self.headers = {'content-type': 'application/json', 'accept': 'application/json'}

        self.parquet_files = 'ParquetFile'
        self.get_countries = 'Country'
        self.get_cities = 'City'
        self.components = 'Property'

        return

    def get_parquet_files(self, max_time_per_dl, n_max_tries, ccode, city, pcode, dataset):
        """build a query to return data in parquet files"""
        query = urllib.parse.urljoin(self.url_api, self.parquet_files)
        request_body = {'countries': [ccode], 'cities': [city], 'properties': [pcode], 'datasets':[dataset], 'source': 'Api'}
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            response = requests.post(url=query, json=request_body, timeout=max_time_per_dl)

            if response.status_code == 200:
                return response.content
                errcode = response.status_code
            else:
                # try again
                print('Response error {}, attempt {}'.format(response.status_code, n_tries))
                errcode = response.status_code
                n_tries += 1

        if n_tries == n_max_tries:
            print('Failed downloading {}_{}_{}_{} {} times in {} seconds, error code {}'.format(ccode, city, pcode, dataset, n_tries, max_time_per_dl, errcode))
            return None
        

    def get_countrycodes_in_list(self, ):
        """build a query to return country codes"""
        query = urllib.parse.urljoin(self.url_api, self.get_countries)
        response = requests.get(url=query, data='')
        if response.status_code == 200:
            return response.text
        else:
            return print('Response error {}'.format(response.status_code))

    def get_components(self, ):
        """build a query to return components"""
        query = urllib.parse.urljoin(self.url_api, self.components)
        response = requests.get(url=query, data='')
        if response.status_code == 200:
            return response.text
        else:
            return print('Response error {}'.format(response.status_code))
        max_time_per_dl, n_max_tries

    def get_cities_in_list(self, ccode):
        """build a query to return cities"""
        query = urllib.parse.urljoin(self.url_api, self.get_cities)
        request_body = [ccode]
        response = requests.post(url=query, json=request_body)
        if response.status_code == 200:
            return response.content
        else:
            return print('Response error {}'.format(response.status_code))


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):
    # from October 2024 the API will have option to choose desired time period; until then, we will use https://discomap.eea.europa.eu/map/fme/AirQualityUTDExport.htm for nrt

    if mode == 'all':
        streams = ['E1a', 'E2a', 'Airbase'] 
        
    elif mode == 'nrt':
        version = mode
        streams = ['E2a'] # this means E2a: data from 2023-01-01 until present
    else:
        print('time mode inapplicable')


    if mode == 'all': # for now API not used for nrt; from October 2024 it will be 
    
        countries = json.loads(eea_ereporting_api().get_countrycodes_in_list())
        #countries = [{'countryCode': 'AT'}]
        components = json.loads(eea_ereporting_api().get_components())

        for stream in streams: 
            # choose the dataset
            if stream == 'E1a':
                dataset = 2 # this means E1a: data from 2013 until 2022
            elif stream == 'E2a':
                dataset = 1 # this means E2a: data from 2023-01-01 until present
            elif stream == 'Airbase':
                dataset = 3 # this means Airbase, old data until 2013
            else:
                print("Invalid data stream")
                continue
            print("Download started from {}".format(stream))

            for country in countries:
                ccode = country['countryCode']

                cities = json.loads(eea_ereporting_api().get_cities_in_list(ccode))
                for city in cities:
                    cityname = city['cityName']

                    for component in components: # downloading ALL components available, not needed for GHOST
                        pcode = component['notation'].replace(' ', '_')

                        data = eea_ereporting_api().get_parquet_files(max_time_per_dl, n_max_tries, ccode, cityname, pcode, dataset)
                        if data == None:
                            continue # move to next component
                        else:
                            os.makedirs('{}/EEA_AQ_eReporting/original_files/{}/{}/{}/'.format(dl_root,version, stream, pcode), exist_ok=True)
                            download_location = '{}/EEA_AQ_eReporting/original_files/{}/{}/{}/'.format(dl_root,version, stream, pcode)
                            filename = '{}_{}_{}.zip'.format(ccode, cityname, pcode)

                            with open(download_location+filename, 'wb') as outfile:
                                outfile.write(data)
                            print("Downloaded {}".format(filename))

                            with zipfile.ZipFile(download_location+filename, 'r') as zip_ref:
                                zip_ref.extractall(download_location)
                            os.remove(download_location+filename)

                            # list all parquetfiles
                            parquets = os.listdir(download_location+'/{}/'.format(stream))
                            for parquet in parquets:
                                df = pd.read_parquet(download_location+'/{}/{}'.format(stream, parquet))
                                df.to_csv(download_location+'/{}.csv'.format(parquet[:-8]), index=None)
                            shutil.rmtree(download_location+'/{}'.format(stream))
                            print("Conversion to csv done!")

    elif mode == 'nrt':
        # download last 48 hours of available data from https://discomap.eea.europa.eu/map/fme/AirQualityUTDExport.htm
        #Python sample for download of air quality up-to-date files
        #The Pyton script shows a simple sample how the pre-processed CSV files can be downloaded.
        #EEA takes no responsibility of the script and the code is provided 'as is', without warranty of any kind.
        today = date.today().strftime('%Y%m%d')
        os.makedirs('{}/EEA_AQ_eReporting/original_files/nrt/discomap/{}/'.format(dl_root,today), exist_ok=True)

        # Set download url (updated 02/05/2024)
        ServiceUrl = "https://eeadmz1batchservice02.blob.core.windows.net/airquality-e-extract"

        inventory_url = ' https://eeadmz1batchservice02.blob.core.windows.net/airquality-e-extract/Inventory.csv'
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            r = requests.get(inventory_url, timeout=max_time_per_dl)
            if r.status_code == 200:
                urllib.request.urlretrieve(inventory_url, '{}/EEA_AQ_eReporting/original_files/nrt/inventory_disco.csv'.format(dl_root))
                errcode = r.status_code

            elif r.status_code == 404:
                print('No {}'.format(inventory_url))
                errcode = 200 #skip
            else:
                # try again
                print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                errcode = r.status_code
                n_tries += 1
                time.sleep(n_tries ** 2) # wait a lil more every time 

        if n_tries == n_max_tries:
            print('Failed downloading {} {} times in {} seconds, error code {}'.format(inventory_url, n_tries, max_time_per_dl, errcode))

        inventory = pd.read_csv( '{}/EEA_AQ_eReporting/original_files/nrt/inventory_disco.csv'.format(dl_root), header=0)

        # Countries to download
        # Note: List is not complete
        #countries = ['SE','DE','ES']

        # Pollutant to be downloaded
        #pollutants = ['C6H6','PM10','CO','NO2','SO2','O3','PM2.5']

        for index, row in inventory.iterrows():
            country = row['country']
            pollutant = row['pollutant'].replace(' ', '')

            fileName = '{}/EEA_AQ_eReporting/original_files/nrt/discomap/{}/{}_{}.csv'.format(dl_root,today, country, pollutant)
            downloadFile = '%s/%s_%s.csv' % (ServiceUrl, country, pollutant)
            n_tries = 0
            errcode = 999

            while (n_tries < n_max_tries) and (errcode != 200):
                r = requests.get(downloadFile, timeout=max_time_per_dl)
                if r.status_code == 200:
                    urllib.request.urlretrieve(downloadFile, fileName)
                    print("Downloaded {}".format(downloadFile))
                    errcode = r.status_code

                elif r.status_code == 404:
                    print('No {}'.format(downloadFile))
                    errcode = 200 #skip
                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    time.sleep(n_tries ** 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(downloadFile, n_tries, max_time_per_dl, errcode))

                # print('Downloading: %s' % downloadFile )
                # file = requests.get(downloadFile).content
                # with open(fileName, 'wb') as output:
                #     output.write(file)
                

      		


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root,  meta_basefile, meta_history, meta_params):

    baseurl = 'https://discomap.eea.europa.eu/App/AQViewer/index.html?fqn=Airquality_Dissem.b2g.measurements'
    download_location = '{}/EEA_AQ_eReporting/metadata/{}/network_provided/'.format(dl_root,version)
    today = date.today()
    
    n_tries = 0
    errcode = 999
    metadata_new = {}
    while (n_tries < n_max_tries) and (errcode != 200):
        try:
            # set up driver
            options = Options()
            prefs = {'download.default_directory' : download_location}
            options.add_experimental_option('prefs', prefs)
            options.add_argument("--no-sandbox")
            options.add_argument("--headless")
            svc = webdriver.ChromeService(executable_path=binary_path)
            driver = webdriver.Chrome(service=svc, options=options)

            # open url
            driver.get(baseurl)
            #WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_element_located((By.XPATH, '//span[@class="nomobile"][text()="Download CSV"]/..'))) # wait till loaded
            time.sleep(10)

            button = driver.find_element(By.XPATH, '//span[@class="nomobile"][text()="Download CSV"]/..') # find parent of span element "Download CSV"
            button.click() #download

            while not os.path.exists(download_location+'DataExtract.csv.zip'):
                time.sleep(1)

                if os.path.isfile(download_location+'DataExtract.csv.zip'):
                    print('metadata download successful')

                    # unzip
                    with zipfile.ZipFile(download_location+'DataExtract.csv.zip', 'r') as zip_ref:
                        zip_ref.extractall(download_location)
            
            os.remove(download_location+'DataExtract.csv.zip')
            os.rename(download_location+'DataExtract.csv', download_location+'EEA_AQ_eReporting_META_{}.csv'.format(today.strftime('%Y%m%d')))

            driver.close()
            errcode = 200
    
        except WebDriverException as e:
            print(e) 
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue

    if n_tries == n_max_tries:
        print('Failed downloading EEA_AQ_eReporting metadata {} times in {} seconds'.format(n_tries, max_time_per_dl))

    print('done')

    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)

    """
    # create json from original metadata file =====================================================================================
    json_metadata = {}
    with open('{}/EEA_AQ_eReporting/metadata/{}/network_provided/DataExtract.csv'.format(dl_root,version), 'r') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['Air Quality Station EoI Code']+'_'+row['Sampling Point Id']+'_'+row['Process Id']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata[key] = row

    with open('{}/EEA_AQ_eReporting/metadata/{}/processed/EEA_AQ_eReporting_META.json'.format(dl_root,version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))
    
    
    # create json in desired shape from current metadata file
    json_metadata_now = {}  
    with open(download_location+"DataExtract.csv", encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)
        print('hola')
        for row in csv_filedata:  
            key = row['Air Quality Station EoI Code']+'_'+row['Sampling Point Id']+'_'+row['Process Id']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata_now[key] = row
        print('hola2')
    
    with open('{}/EEA_AQ_eReporting/metadata/{}/network_provided/EEA_AQ_eReporting_META_{}.json'.format(dl_root,version, today.strftime('%Y%m%d')), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata_now, indent=4, ensure_ascii=False))
        

    # read standardised file to compare!
    with open('{}/EEA_AQ_eReporting/metadata/{}/processed/EEA_AQ_eReporting_META.json'.format(dl_root,version), 'r', encoding='utf-8') as f:
        json_metadata = json.loads(f.read())

        for station in json_metadata: # loop through all the old stations
            if station in json_metadata_now.keys(): # if station is in current meta data, go on
                for parameter in json_metadata[station]:
                    if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                        # if different value, append the standardised metadeta file
                        print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                        json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                        json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                    else:
                        pass
            else:
                print('Station {} was abolished'.format(station))

        for station in json_metadata_now: # loop through all the new stations
            if station in json_metadata.keys(): # if station is in old meta data
                pass # comparison was done before
            else: # new station appeared!
                print('New station {}'.format(station))
                json_metadata.update({station: json_metadata_now[station]})

    # safe
    with open('{}/EEA_AQ_eReporting/metadata/{}/processed/EEA_AQ_eReporting_META.json'.format(dl_root,version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))"""
        
