# use API
import requests
import pandas as pd

import os
import sys
import logging
import urllib
import datetime as dt
import json
import csv
from pathlib import Path
from datetime import date
from datetime import timedelta
import zipfile
import time
import shutil

from selenium import webdriver
from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException
from bs4 import BeautifulSoup


class eea_ereporting_api(object):
    """defines and configures API to query the eea database"""

    def __init__(self, ):
        self.logger = logging.getLogger(__file__)
        self.url_api = "https://eeadmz1-downloads-api-appservice.azurewebsites.net/"
        self.headers = {'content-type': 'application/json', 'accept': 'application/json'}

        self.parquet_files = 'ParquetFile'
        self.get_countries = 'Country'
        self.get_cities = 'City'
        self.components = 'Property'

        return

    def get_parquet_files(self, max_time_per_dl, n_max_tries, ccode, city, pcode, dataset):
        """build a query to return data in parquet files"""
        query = urllib.parse.urljoin(self.url_api, self.parquet_files)
        request_body = {'countries': [ccode], 'cities': [city], 'properties': [pcode], 'datasets':[dataset], 'source': 'Api'}
        n_tries = 0
        errcode = 999

        while (n_tries < n_max_tries) and (errcode != 200):
            response = requests.post(url=query, json=request_body, timeout=max_time_per_dl)

            if response.status_code == 200:
                return response.content
                errcode = response.status_code
            else:
                # try again
                print('Response error {}, attempt {}'.format(response.status_code, n_tries))
                errcode = response.status_code
                n_tries += 1

        if n_tries == n_max_tries:
            print('Failed downloading {}_{}_{}_{} {} times in {} seconds, error code {}'.format(ccode, city, pcode, dataset, n_tries, max_time_per_dl, errcode))
            return None
        

    def get_countrycodes_in_list(self, ):
        """build a query to return country codes"""
        query = urllib.parse.urljoin(self.url_api, self.get_countries)
        response = requests.get(url=query, data='')
        if response.status_code == 200:
            return response.text
        else:
            return print('Response error {}'.format(response.status_code))

    def get_components(self, ):
        """build a query to return components"""
        query = urllib.parse.urljoin(self.url_api, self.components)
        response = requests.get(url=query, data='')
        if response.status_code == 200:
            return response.text
        else:
            return print('Response error {}'.format(response.status_code))
        max_time_per_dl, n_max_tries

    def get_cities_in_list(self, ccode):
        """build a query to return cities"""
        query = urllib.parse.urljoin(self.url_api, self.get_cities)
        request_body = [ccode]
        response = requests.post(url=query, json=request_body)
        if response.status_code == 200:
            return response.content
        else:
            return print('Response error {}'.format(response.status_code))


def download_data(mode, version, n_max_tries, max_time_per_dl):

    if mode == 'all':
        streams = ['E1a', 'E2a', 'Airbase'] 
        
    elif mode == 'nrt':
        version = mode
        streams = 'E2a' # this means E2a: data from 2023-01-01 until present
    else:
        print('time mode inapplicable')
    
    countries = json.loads(eea_ereporting_api().get_countrycodes_in_list())
    #countries = [{'countryCode': 'AT'}]
    components = json.loads(eea_ereporting_api().get_components())

    for stream in streams: 
        # choose the dataset
        if stream == 'E1a':
            dataset = 2 # this means E1a: data from 2013 until 2022
        elif stream == 'E2a':
            dataset = 1 # this means E2a: data from 2023-01-01 until present
        elif stream == 'Airbase':
            dataset = 3 # this means Airbase, old data until 2013
        else:
            print("Invalid data stream")
            continue
        print("Download started from {}".format(stream))

        for country in countries:
            ccode = country['countryCode']

            cities = json.loads(eea_ereporting_api().get_cities_in_list(ccode))
            for city in cities:
                cityname = city['cityName']

                for component in components:
                    pcode = component['notation'].replace(' ', '_')

                    data = eea_ereporting_api().get_parquet_files(max_time_per_dl, n_max_tries, ccode, cityname, pcode, dataset)
                    if data == None:
                        continue # move to next component
                    else:
                        os.makedirs('/esarchive/obs/ghost/EEA_AQ_eReporting/original_files/{}/{}/{}/'.format(version, stream, pcode), exist_ok=True)
                        download_location = '/esarchive/obs/ghost/EEA_AQ_eReporting/original_files/{}/{}/{}/'.format(version, stream, pcode)
                        filename = '{}_{}_{}.zip'.format(ccode, cityname, pcode)

                        with open(download_location+filename, 'wb') as outfile:
                            outfile.write(data)
                        print("Downloaded {}".format(filename))

                        with zipfile.ZipFile(download_location+filename, 'r') as zip_ref:
                            zip_ref.extractall(download_location)
                        os.remove(download_location+filename)

                        # list all parquetfiles
                        parquets = os.listdir(download_location+'/{}/'.format(stream))
                        for parquet in parquets:
                            df = pd.read_parquet(download_location+'/{}/{}'.format(stream, parquet))
                            df.to_csv(download_location+'/{}.csv'.format(parquet[:-8]), index=None)
                        shutil.rmtree(download_location+'/{}'.format(stream))
                        print("Conversion to csv done!")


def download_metadata(n_max_tries, max_time_per_dl):

    baseurl = 'https://discomap.eea.europa.eu/App/AQViewer/index.html?fqn=Airquality_Dissem.b2g.measurements'
    download_location = '/esarchive/obs/ghost/EEA_AQ_eReporting/metadata/network_provided/'
    today = date.today()
    
    n_tries = 0
    errcode = 999
    metadata_new = {}
    while (n_tries < n_max_tries) and (errcode != 200):
        try:
            # set up driver
            options = Options()
            prefs = {'download.default_directory' : download_location}
            options.add_experimental_option('prefs', prefs)
            options.add_argument("--no-sandbox")
            options.add_argument("--headless")
            svc = webdriver.ChromeService(executable_path=binary_path)
            driver = webdriver.Chrome(service=svc, options=options)

            # open url
            driver.get(baseurl)
            #WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_element_located((By.XPATH, '//span[@class="nomobile"][text()="Download CSV"]/..'))) # wait till loaded
            time.sleep(10)

            button = driver.find_element(By.XPATH, '//span[@class="nomobile"][text()="Download CSV"]/..') # find parent of span element "Download CSV"
            button.click() #download

            while not os.path.exists(download_location+'DataExtract.csv.zip'):
                time.sleep(1)

                if os.path.isfile(download_location+'DataExtract.csv.zip'):
                    print('metadata download successful')

                    # unzip
                    with zipfile.ZipFile(download_location+'DataExtract.csv.zip', 'r') as zip_ref:
                        zip_ref.extractall(download_location)
            
            os.remove(download_location+'DataExtract.csv.zip')

            driver.close()
            errcode = 200
    
        except WebDriverException as e:
            print(e) 
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue

    if n_tries == n_max_tries:
        print('Failed downloading EEA_AQ_eReporting metadata {} times in {} seconds'.format(n_tries, max_time_per_dl))

    """
    # create json from original metadata file =====================================================================================
    json_metadata = {}
    with open('/esarchive/obs/ghost/EEA_AQ_eReporting/metadata/network_provided/DataExtract.csv', 'r') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['Air Quality Station EoI Code']+'_'+row['Sampling Point Id']+'_'+row['Process Id']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata[key] = row

    with open('/esarchive/obs/ghost/EEA_AQ_eReporting/metadata/processed/EEA_AQ_eReporting_META.json', 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))
    
    """
    # create json in desired shape from current metadata file
    json_metadata_now = {}  
    with open(download_location+"DataExtract.csv", encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['Air Quality Station EoI Code']+'_'+row['Sampling Point Id']+'_'+row['Process Id']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata_now[key] = row
    
    with open('/esarchive/obs/ghost/EEA_AQ_eReporting/metadata/network_provided/EEA_AQ_eReporting_META_{}.json'.format(today.strftime('%Y%m%d')), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata_now, indent=4, ensure_ascii=False))
        

    # read standardised file to compare!
    with open('/esarchive/obs/ghost/EEA_AQ_eReporting/metadata/processed/EEA_AQ_eReporting_META.json', 'r', encoding='utf-8') as f:
        json_metadata = json.loads(f.read())

        for station in json_metadata: # loop through all the old stations
            if station in json_metadata_now.keys(): # if station is in current meta data, go on
                for parameter in json_metadata[station]:
                    if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                        # if different value, append the standardised metadeta file
                        print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                        json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                        json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                    else:
                        pass
            else:
                print('Station {} was abolished'.format(station))

        for station in json_metadata_now: # loop through all the new stations
            if station in json_metadata.keys(): # if station is in old meta data
                pass # comparison was done before
            else: # new station appeared!
                print('New station {}'.format(station))
                json_metadata.update({station: json_metadata_now[station]})

    # safe
    with open('/esarchive/obs/ghost/EEA_AQ_eReporting/metadata/processed/EEA_AQ_eReporting_META.json', 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))
        
