from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
import requests
import time
from datetime import date
from datetime import timedelta
import zipfile

import os.path
import os
import pandas as pd
import csv
import json

def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):
    
    url = 'https://monitoring.eanet.asia/document/menu/index#publicData'
    today = date.today().strftime('%Y%m%d') #+ timedelta(days = 1)).strftime('%Y%m%d') # problem with timezones???
    
    if mode == 'all':
        bdate = date(1980, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today()

        os.makedirs('{}/EANET/original_files/{}/'.format(dl_root, version), exist_ok=True)
        download_location = "{}/EANET/original_files/{}/".format(dl_root, version)


    elif mode == 'nrt':
        print("EANET no nrt")
        exit()

    else:
        print('time mode inapplicable')
        exit()
    

    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    driver.get(url)
    time.sleep(max_time_per_dl)

    # login
    email = driver.find_element(By.ID, "email") 
    email.send_keys("raphael.grodofzig@bsc.es")
    passwd = driver.find_element(By.ID, "passwd") 
    passwd.send_keys("274s9QZ5")
    time.sleep(max_time_per_dl)
    driver.find_element(By.NAME, "submitBtn").click()

    time.sleep(max_time_per_dl)

    # find countries
    dropdown_element = driver.find_element(By.ID, 'countryCd')
    select = Select(dropdown_element)
    countries = []
    COUNTRIES = []
    for country in select.options:
        if country.text == "-":
            pass
        else:
            countries.append(str(country.text))

            if country.text == "Korea, Republic of":
                COUNTRIES.append("SOUTH_KOREA")
            elif country.text == "Lao People's Democratic Republic":
                COUNTRIES.append("LAOS")
            elif country.text == "Viet Nam":
                COUNTRIES.append("VIETNAM")
            else:
                COUNTRIES.append(str(country.text).upper())

    print(countries)
    print(COUNTRIES)
 
    #countries = ["Cambodia", "China"]
    #COUNTRIES = ["CAMBODIA", "CHINA"]

    # download
    i=0
    for country in countries:
        # create download directory
        os.makedirs(download_location+COUNTRIES[i]+'/', exist_ok=True)

        dropdown_element = driver.find_element(By.ID, 'countryCd')
        select = Select(dropdown_element)
        select.select_by_visible_text(country)
        
        print("successfully selected {}".format(country))

        # start download
        driver.find_element(By.CSS_SELECTOR, "input[id='downloadBtn'][type='button']").click()

        # wait until download finished
        while not os.path.exists("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today)):
            time.sleep(1)

            if os.path.isfile("{}/detail_{}.zip".format(download_location, today)):
                print('{} download successful'.format(country))

                # move to country directory
                os.rename("{}/detail_{}.zip".format(download_location, today), "{}/detail_{}.zip".format(download_location+COUNTRIES[i], today))

                # unzip
                with zipfile.ZipFile("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today), 'r') as zip_ref:
                    zip_ref.extractall(os.path.dirname("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today)))


        os.remove("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today))

        i=i+1

    driver.close()


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root):

    url_metadata = 'https://www.eanet.asia/wp-content/uploads/2024/01/Site_Information_Acid_Deposition_Monitoring_NMP2023_1117.xlsm'
    download_location = dl_root+"/EANET/metadata/"+version+"/network_provided/EANET_META_{}.xlsm"
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    n_tries = 0
    errcode = 999
    today = date.today()
    
    while (n_tries < n_max_tries) and (errcode != 200):
        r = requests.get(url_metadata, timeout=max_time_per_dl, headers=Headers)
        if r.status_code == 200:
            with open(download_location.format(today.strftime('%Y%m%d')), 'wb') as outfile:
                outfile.write(r.content)
            print('Downloaded metadata')
            errcode = r.status_code
        elif r.status_code == 404:
            print("No metadata found, error 404")
            errcode = 200
        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            max_time_per_dl = max_time_per_dl*2 # increase waiting time 
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_metadata, n_tries, max_time_per_dl, errcode))
    time.sleep(1)

    #convert to csv
    metadata = pd.read_excel(download_location.format(today.strftime('%Y%m%d'))).fillna('')
    metadata.to_csv('{}/EANET/metadata/{}/network_provided/EANET_META_{}.csv'.format(dl_root, version, today.strftime('%Y%m%d')), index=None)
    os.remove(download_location.format(today.strftime('%Y%m%d')))

    """
    # create json from original metadata file
    json_metadata = {}
    with open('{}/EANET/metadata/{}/network_provided/EANET_META_{}.csv'.format(dl_root, version, today.strftime('%Y%m%d')), 'r', encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['Code']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata[key] = row

    with open('{}/EANET/metadata/{}/processed/EANET_META.json'.format(dl_root, version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4))
        
    """
    # create json in desired shape from current metadata file
    json_metadata_now = {}  
    with open('{}/EANET/metadata/{}/network_provided/EANET_META_{}.csv'.format(dl_root, version, today.strftime('%Y%m%d')), encoding='utf-8') as file:
        csv_filedata = csv.DictReader(file)

        for row in csv_filedata:  
            key = row['Code']
            update_date = today.strftime('%Y-%m-%d')
            for parameter in row:
                row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
            json_metadata_now[key] = row

    
    # read standardised file to compare!
    with open('{}/EANET/metadata/{}/processed/EANET_META.json'.format(dl_root, version), 'r', encoding='ISO-8859-1') as f:
        json_metadata = json.loads(f.read())

        for station in json_metadata: # loop through all the old stations
            if station in json_metadata_now.keys(): # if station is in current meta data, go on
                for parameter in json_metadata[station]:
                    if parameter in json_metadata_now[station].keys(): # check if column of csv exists in new file
                        if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                            # if different value, append the standardised metadeta file
                            print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                            json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                            json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                        else:
                            pass
                    else:
                        print('{} not in new metadata file'.format(parameter))
            else:
                print('Station {} was abolished'.format(station))

        for station in json_metadata_now: # loop through all the new stations
            for parameter in json_metadata_now[station]: # loop through all the parameters
                if station in json_metadata.keys(): # if station is in old meta data
                    pass # comparison was done before
                else: # new station appeared!
                    print('New station {}'.format(station))
                    json_metadata.update({station: json_metadata_now[station]})
                # is there a new parameter that wasn't in the old file?
                if parameter in json_metadata[station].keys():
                    pass # parameter (column) is already there
                else:
                    print('{} is new'.format(parameter))
                    json_metadata[station].update({parameter: json_metadata_now[station][parameter]})


    # safe
    with open('{}/EANET/metadata/{}/processed/EANET_META.json'.format(dl_root, version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4))