from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
import requests
import time
from datetime import date
from datetime import timedelta
import zipfile
import importlib

import os.path
import os
import sys
import pandas as pd
import csv
import json
import metadata_parsing_functions

def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):
    
    url = 'https://monitoring.eanet.asia/document/menu/index#publicData'
    today = date.today().strftime('%Y%m%d') #+ timedelta(days = 1)).strftime('%Y%m%d') # problem with timezones???
    
    if mode == 'all':
        bdate = date(1980, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today()

        os.makedirs('{}/EANET/original_files/{}/'.format(dl_root, version), exist_ok=True)
        download_location = "{}/EANET/original_files/{}/".format(dl_root, version)


    elif mode == 'nrt':
        print("EANET no nrt")
        exit()

    else:
        print('time mode inapplicable')
        exit()
    

    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    driver.get(url)
    time.sleep(max_time_per_dl)

    # login
    email = driver.find_element(By.ID, "email") 
    email.send_keys("raphael.grodofzig@bsc.es")
    passwd = driver.find_element(By.ID, "passwd") 
    passwd.send_keys("274s9QZ5")
    time.sleep(max_time_per_dl)
    driver.find_element(By.NAME, "submitBtn").click()

    time.sleep(max_time_per_dl)

    # find countries
    dropdown_element = driver.find_element(By.ID, 'countryCd')
    select = Select(dropdown_element)
    countries = []
    COUNTRIES = []
    for country in select.options:
        if country.text == "-":
            pass
        else:
            countries.append(str(country.text))

            if country.text == "Korea, Republic of":
                COUNTRIES.append("SOUTH_KOREA")
            elif country.text == "Lao People's Democratic Republic":
                COUNTRIES.append("LAOS")
            elif country.text == "Viet Nam":
                COUNTRIES.append("VIETNAM")
            else:
                COUNTRIES.append(str(country.text).upper())

    print(countries)
    print(COUNTRIES)
 
    #countries = ["Cambodia", "China"]
    #COUNTRIES = ["CAMBODIA", "CHINA"]

    # download
    i=0
    for country in countries:
        # create download directory
        os.makedirs(download_location+COUNTRIES[i]+'/', exist_ok=True)

        dropdown_element = driver.find_element(By.ID, 'countryCd')
        select = Select(dropdown_element)
        select.select_by_visible_text(country)
        
        print("successfully selected {}".format(country))

        # start download
        driver.find_element(By.CSS_SELECTOR, "input[id='downloadBtn'][type='button']").click()

        # wait until download finished
        while not os.path.exists("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today)):
            time.sleep(1)

            if os.path.isfile("{}/detail_{}.zip".format(download_location, today)):
                print('{} download successful'.format(country))

                # move to country directory
                os.rename("{}/detail_{}.zip".format(download_location, today), "{}/detail_{}.zip".format(download_location+COUNTRIES[i], today))

                # unzip
                with zipfile.ZipFile("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today), 'r') as zip_ref:
                    zip_ref.extractall(os.path.dirname("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today)))


        os.remove("{}/detail_{}.zip".format(download_location+COUNTRIES[i], today))

        i=i+1

    driver.close()


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    
    url_metadata = 'https://www.eanet.asia/wp-content/uploads/2024/01/Site_Information_Acid_Deposition_Monitoring_NMP2023_1117.xlsm'
    download_location = dl_root+"/EANET/metadata/"+version+"/network_provided/EANET_META_{}.xlsm"
    Headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}

    n_tries = 0
    errcode = 999
    today = date.today()
    """
    while (n_tries < n_max_tries) and (errcode != 200):
        r = requests.get(url_metadata, timeout=max_time_per_dl, headers=Headers)
        if r.status_code == 200:
            with open(download_location.format(today.strftime('%Y%m%d')), 'wb') as outfile:
                outfile.write(r.content)
            print('Downloaded metadata')
            errcode = r.status_code
        elif r.status_code == 404:
            print("No metadata found, error 404")
            errcode = 200
        else:
            # try again
            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
            errcode = r.status_code
            n_tries += 1
            max_time_per_dl = max_time_per_dl*2 # increase waiting time 
            time.sleep(n_tries ** 2) # wait a lil more every time 

    if n_tries == n_max_tries:
        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url_metadata, n_tries, max_time_per_dl, errcode))
    time.sleep(1)

    #convert to csv
    metadata = pd.read_excel(download_location.format(today.strftime('%Y%m%d'))).fillna('')
    metadata.to_csv('{}/EANET/metadata/{}/network_provided/EANET_META_{}.csv'.format(dl_root, version, today.strftime('%Y%m%d')), index=None)
    os.remove(download_location.format(today.strftime('%Y%m%d')))"""

    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)


    # there are missing stations in provided meta data file. Add these stations from manually compiled meta file from GHOST1.5 
    # ==========================================================================
    # open metadata json file
    with open('{}/{}/metadata/{}/processed/{}_META.json'.format(dl_root, network, version, network), 'r', encoding=meta_params['encoding']) as f:
        metadata_v16 = json.loads(f.read())

    # open metafile from 1.5
    sys.path.append(dl_root+'/EANET/metadata/1.5/processed/')
    meta_load = importlib.import_module('EANET_META')
    metadata_v15 = meta_load.EANET_meta

    meta_parameters_v15 = list(metadata_v15[list(metadata_v15.keys())[0]].keys())
    meta_parameters_v16 = list(metadata_v16[list(metadata_v16.keys())[0]].keys())

    for station_ref in metadata_v15:
        if station_ref in list(metadata_v16.keys()):
            pass # station is already there
        else:
            # make structure
            metadata_v16[station_ref] = {meta_parameter: {'values':[], 'update_time':[]} for meta_parameter in meta_parameters_v16}

            # fill data
            metadata_v16[station_ref]['Code']['values'].append(station_ref)
            metadata_v16[station_ref]['Code']['update_time'].append(today.strftime('%Y-%m-%d'))

            metadata_v16[station_ref]['Site']['values'].append(metadata_v15[station_ref]['station_name'])
            metadata_v16[station_ref]['Site']['update_time'].append(today.strftime('%Y-%m-%d'))

            metadata_v16[station_ref]['Country']['values'].append(metadata_v15[station_ref]['country'])
            metadata_v16[station_ref]['Country']['update_time'].append(today.strftime('%Y-%m-%d'))

            metadata_v16[station_ref]['Classification']['values'].append(metadata_v15[station_ref]['classification'])
            metadata_v16[station_ref]['Classification']['update_time'].append(today.strftime('%Y-%m-%d'))

            metadata_v16[station_ref]['Altitude']['values'].append(metadata_v15[station_ref]['altitude'])
            metadata_v16[station_ref]['Altitude']['update_time'].append(today.strftime('%Y-%m-%d'))

            if 'comments' in list(metadata_v15[station_ref].keys()):
                metadata_v16[station_ref]['Note']['values'].append(metadata_v15[station_ref]['comments'])
                metadata_v16[station_ref]['Note']['update_time'].append(today.strftime('%Y-%m-%d'))

            # latitude!
            metadata_v16[station_ref]['Latitude_degree']['values'].append(metadata_v15[station_ref]['latitude'].split(',')[0].replace('S', '').replace('N', ''))
            metadata_v16[station_ref]['Latitude_degree']['update_time'].append(today.strftime('%Y-%m-%d'))

            metadata_v16[station_ref]['Latitude_minute']['values'].append(metadata_v15[station_ref]['latitude'].split(',')[1].replace('S', '').replace('N', ''))
            metadata_v16[station_ref]['Latitude_minute']['update_time'].append(today.strftime('%Y-%m-%d'))

            if len(metadata_v15[station_ref]['latitude']) > 6:
                metadata_v16[station_ref]['Latitude_second']['values'].append(metadata_v15[station_ref]['latitude'].split(',')[2].replace('S', '').replace('N', ''))
            else:
                metadata_v16[station_ref]['Latitude_second']['values'].append('00')
            metadata_v16[station_ref]['Latitude_second']['update_time'].append(today.strftime('%Y-%m-%d'))

            # longitude!
            metadata_v16[station_ref]['Longitude_degree']['values'].append(metadata_v15[station_ref]['longitude'].split(',')[0].replace('E', '').replace('W', ''))
            metadata_v16[station_ref]['Longitude_degree']['update_time'].append(today.strftime('%Y-%m-%d'))

            print(metadata_v15[station_ref]['longitude'].split(','))
            metadata_v16[station_ref]['Longitude_minute']['values'].append(metadata_v15[station_ref]['longitude'].split(',')[1].replace('E', '').replace('W', ''))
            metadata_v16[station_ref]['Longitude_minute']['update_time'].append(today.strftime('%Y-%m-%d'))

            if len(metadata_v15[station_ref]['longitude']) > 7:
                metadata_v16[station_ref]['Longitude_second']['values'].append(metadata_v15[station_ref]['longitude'].split(',')[2].replace('E', '').replace('W', ''))
            else:
                metadata_v16[station_ref]['Longitude_second']['values'].append('00')
            metadata_v16[station_ref]['Longitude_second']['update_time'].append(today.strftime('%Y-%m-%d'))
                
        # add methods as new parameter to v16
        metadata_v16[station_ref]['methods'] = {'values':[], 'update_time':[]}
        metadata_v16[station_ref]['methods']['values'].append(metadata_v15[station_ref]['methods'])
        metadata_v16[station_ref]['methods']['update_time'].append(today.strftime('%Y-%m-%d'))

        # add orientation (N-S to v16)
        metadata_v16[station_ref]['orientation'] = {'values':[], 'update_time':[]}
        metadata_v16[station_ref]['orientation']['values'].append(metadata_v15[station_ref]['latitude'][-1])
        metadata_v16[station_ref]['orientation']['update_time'].append(today.strftime('%Y-%m-%d'))


    with open('{}/{}/metadata/{}/processed/{}_META.json'.format(dl_root, network, version, network), 'w', encoding=meta_params['encoding']) as f:
        f.write(json.dumps(metadata_v16, indent=4))
