from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
import requests
import time
from datetime import date
from datetime import timedelta
import zipfile

import os.path
import os
import pathlib
import pandas as pd
import numpy as np
import shutil
import csv
import json
import urllib


def switch_panels(driver, tab_index, max_time_wait):

    if tab_index == 0:
        mon_year = '"tab-pane fade active show"'
        hour = '"tab-pane fade"'
        bur = '"tab-pane fade"'
        meta = '"tab-pane fade"'

        search_btn = 'my_search_button'
        create_btn = 'my_create_archive'
        download_btn = 'my_download_archive'

    elif tab_index == 1:
        mon_year = '"tab-pane fade"'
        hour = '"tab-pane fade active show"'
        bur = '"tab-pane fade"'
        meta = '"tab-pane fade"'

        search_btn = 't_search_button'
        create_btn = 't_create_archive'
        download_btn = 't_download_archive'

    elif tab_index == 2:
        mon_year = '"tab-pane fade"'
        hour = '"tab-pane fade"'
        bur = '"tab-pane fade active show"'
        meta = '"tab-pane fade"'

        search_btn = 'tg_search_button'
        create_btn = 'tg_create_archive'
        download_btn = 'tg_download_archive'

    elif tab_index == 3:
        mon_year = '"tab-pane fade"'
        hour = '"tab-pane fade"'
        bur = '"tab-pane fade"'
        meta = '"tab-pane fade active show"'

        search_btn = 'ms_search_button'
        create_btn = 'ms_create_archive'
        download_btn = 'ms_download_archive'

    else:
        print("Error when switching tabs")

    # switch panels
    tab = driver.find_element(By.ID, 'monthly-yearly-panel')
    driver.execute_script("arguments[0].removeAttribute('class')", tab)
    driver.execute_script("arguments[0].setAttribute('class', {})".format(mon_year), tab)

    tab = driver.find_element(By.ID, 'time-gov-panel')
    driver.execute_script("arguments[0].removeAttribute('class')", tab)
    driver.execute_script("arguments[0].setAttribute('class', {})".format(bur), tab)

    tab = driver.find_element(By.ID, 'time-panel')
    driver.execute_script("arguments[0].removeAttribute('class')", tab)
    driver.execute_script("arguments[0].setAttribute('class', {})".format(hour), tab)

    tab = driver.find_element(By.ID, 'measuring-station-panel')
    driver.execute_script("arguments[0].removeAttribute('class')", tab)
    driver.execute_script("arguments[0].setAttribute('class', {})".format(meta), tab)


    # click buttons
    WebDriverWait(driver, max_time_wait).until(EC.element_to_be_clickable((By.ID, search_btn))) # wait till loaded
    search_button = driver.find_element(By.ID, search_btn)
    driver.execute_script("arguments[0].click();", search_button)
    
    WebDriverWait(driver, max_time_wait).until(EC.element_to_be_clickable((By.ID, create_btn))) # wait till loaded
    create_archive = driver.find_element(By.ID, create_btn)
    driver.execute_script("arguments[0].click();", create_archive)
    
    WebDriverWait(driver, max_time_wait*2).until(EC.element_to_be_clickable((By.ID, download_btn))) # wait till loaded
    download_button = driver.find_element(By.ID, download_btn)
    driver.execute_script("arguments[0].click();", download_button)



def download_data(mode, version, n_max_tries, max_time_per_dl):

    cities = {1: "Hokkaido", 2: "Aomori", 3: "Iwate", 4: "Miyagi", 5: "Akita", 6: "Yamagata", 7: "Fukushima", 8: "Ibaraki", 9: "Tochigi", 10: "Gunma", 11: "Saitama", 12: "Chiba", 13: "Tokyo",
        14: "Kanagawa", 15: "Niigata", 16: "Toyama", 17: "Ishikawa", 18: "Fukui", 19: "Yamanashi", 20: "Nagano", 21: "Gifu", 22: "Shizuoka", 23: "Aichi", 24: "Mie", 25: "Shiga", 26: "Kyoto",
        27: "Osaka", 28: "Hyogo", 29: "Nara", 30: "Wakayama", 31: "Tottori", 32: "Shimane", 33: "Okayama", 34: "Hiroshima", 35: "Yamaguchi", 36:"Tokushima", 37:"Kagawa", 38:"Ehime",
        39:"Kochi", 40:"Fukuoka", 41:"Saga", 42:"Nagasaki", 43:"Kumamoto", 44:"Oita", 45: "Miyazaki", 46:"Kagoshima", 47:"Okinawa"}


    if mode == 'all':
        baseurl = 'https://tenbou.nies.go.jp/download/'
        bdate = date(1971, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today()

        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version), exist_ok=True)
        download_location = '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)

        # create date array, daily; format to YYYYMMDD
        years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()

        options = Options()
        prefs = {'download.default_directory' : download_location}
        options.add_experimental_option('prefs', prefs)
        options.add_argument("--no-sandbox")
        #options.add_argument("--headless")
        svc = webdriver.ChromeService(executable_path=binary_path)
        driver = webdriver.Chrome(service=svc, options=options)

        driver.get(baseurl)
        time.sleep(2)

        # monthly tab ==========================================================================================================
        """
        # switch to monthly/yearly button
        switch_panels(driver, 0, max_time_per_dl) # 0 stands for monthly tab

        filename = [file for file in os.listdir(download_location.format(version)) if os.path.isfile(os.path.join(download_location.format(version), file))][0]
        if 'crdownload' in filename:
            filename = str(filename[:-11]) # cut out crdownload
        else:
            pass

        # create monthly folder
        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'monthly/', exist_ok=True)

        while not os.path.exists(download_location.format(version)+filename):
            time.sleep(1)

            if os.path.isfile(download_location.format(version)+filename):
                print('{} download successful'.format("monthly/yearly data"))

                # unzip
                with zipfile.ZipFile(download_location.format(version)+filename, 'r') as zip_ref:
                    zip_ref.extractall(download_location.format(version)+'monthly/')
        
        os.remove(download_location.format(version)+filename)

        # sort files
        monthly_files = os.listdir(download_location.format(version)+'monthly/')

        for year in years:
            os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'monthly/'+year+'/', exist_ok=True)
            for file in monthly_files:
                if year in file[:6]:
                    # unzip again
                    with zipfile.ZipFile(download_location.format(version)+'monthly/{}'.format(file), 'r') as zip_ref:
                        zip_ref.extractall(download_location.format(version)+'monthly/{}/{}.txt'.format(year, file[:-4]))
                    os.remove(download_location.format(version)+'monthly/{}'.format(file))

        time.sleep(2)
    """
        # time tab ==========================================================================================================
        switch_panels(driver, 1, max_time_per_dl) # 1 stands for hourly tab

        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'hourly/', exist_ok=True)

        # wait until download finished
        time.sleep(10) # time for the file fo appear in directory
        filename = [file for file in os.listdir(download_location.format(version)) if os.path.isfile(os.path.join(download_location.format(version), file))][0]
        if 'crdownload' in filename:
            filename = str(filename[:-11]) # cut out crdownload
        else:
            pass
        print(filename)

        while not os.path.exists(download_location.format(version)+filename):
            time.sleep(1)

            if os.path.isfile(download_location.format(version)+filename):
                print('{} download successful'.format("hourly data"))

                # unzip
                with zipfile.ZipFile(download_location.format(version)+filename, 'r') as zip_ref:
                    zip_ref.extractall(download_location.format(version)+'hourly/')
        
        #os.remove(download_location.format(version)+filename)

        # sort files
        hourly_files = os.listdir(download_location.format(version)+'hourly/')

        for file in hourly_files:
            # unzip again
            with zipfile.ZipFile(download_location.format(version)+'hourly/{}'.format(file), 'r') as zip_ref:
                zip_ref.extractall(download_location.format(version)+'hourly/')
            os.remove(download_location.format(version)+'hourly/{}'.format(file))

        hourly_folders = list(pathlib.Path(download_location.format(version)+'hourly/').rglob('*.zip')) #os.listdir(download_location.format(version)+'hourly/')

        for folder in hourly_folders:
            # unzip again
            with zipfile.ZipFile(folder, 'r') as zip_ref:
                zip_ref.extractall(download_location.format(version)+'hourly/') # takes 10 mins!
            os.remove(folder)

        # put all files in single new directory, delete everything else
        hourly_files_all = list(pathlib.Path(download_location.format(version)+'hourly/').rglob('*.txt'))
        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/'.format(version), exist_ok=True)

        for file in hourly_files_all:
            filename = os.path.split(file)[-1]
            shutil.move(file, '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/{}'.format(version, filename))

        shutil.rmtree(download_location.format(version)+'hourly/')

        hourly_files_all = os.listdir('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/'.format(version))
        for city in cities:
            for year in range(2009, 2022):
                for file in hourly_files_all:
                    if ((int(file[1:3]) == city) and (int(file[3:7]) == year)):
                        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/hourly/{}/{}/'.format(version, cities[city], year), exist_ok=True)
                        shutil.move('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/'.format(version)+file, '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/hourly/{}/{}/{}'.format(version, cities[city], year, file))


        shutil.rmtree(download_location.format(version)+'temp/')
        time.sleep(2)

        # bureau tab ==========================================================================================================
        switch_panels(driver, 2, max_time_per_dl) # 1 stands for bureau tab

        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'bureau/', exist_ok=True)

        # wait until download finished
        time.sleep(10) # time for the file fo appear in directory
        filename = [file for file in os.listdir(download_location.format(version)) if os.path.isfile(os.path.join(download_location.format(version), file))][0]
        if 'crdownload' in filename:
            filename = str(filename[:-11]) # cut out crdownload
        else:
            pass
        print(filename)

        while not os.path.exists(download_location.format(version)+filename):
            time.sleep(1)

            if os.path.isfile(download_location.format(version)+filename):
                print('{} download successful'.format("bureau data"))

                # unzip
                with zipfile.ZipFile(download_location.format(version)+filename, 'r') as zip_ref:
                    zip_ref.extractall(download_location.format(version)+'bureau/')
        
        os.remove(download_location.format(version)+filename)

        # sort files
        bureau_files = os.listdir(download_location.format(version)+'bureau/')

        for file in bureau_files:
            with zipfile.ZipFile(download_location.format(version)+'bureau/{}'.format(file), 'r') as zip_ref:
                zip_ref.extractall(download_location.format(version)+'bureau/')
            os.remove(download_location.format(version)+'bureau/{}'.format(file))

        time.sleep(2)

        # check
        bureau_files = os.listdir(download_location.format(version)+'bureau/')
        bureau_files_remaining = [filename for filename in bureau_files if filename.endswith('.csv')]
        for remaining_file in bureau_files_remaining:
            year = os.path.split(remaining_file)[-1][:4]
            os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/bureau/{}/'.format(version, year), exist_ok=True)
            shutil.move(remaining_file, '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/bureau/{}/'.format(version, year))

        driver.close()


    elif mode == 'nrt':
        bdate = (date.today() - timedelta(days=30)).strftime('%Y%m')
        edate = date.today().strftime('%Y%m')

        download_location = '/esarchive/obs/ghost/JAPAN_NIES/original_files/nrt/{}_{}/nrt_{}.json'
        baseurl = 'https://soramame.env.go.jp/soramame/api/data_search?Start_YM={}&End_YM={}&TDFKN_CD={}'

        regions = [str(x) for x in list(range(1, 48))] # region codes
        
        for region in regions:

            os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/nrt/{}_{}/'.format(region.zfill(2), cities[int(region)]), exist_ok=True)
            url = baseurl.format(bdate, edate, region.zfill(2))
            n_tries = 0
            errcode = 999

            while (n_tries < n_max_tries) and (errcode != 200):
                r = requests.get(url, timeout=max_time_per_dl)
                if r.status_code == 200:
                    with open(download_location.format(region.zfill(2), cities[int(region)], region.zfill(2)), 'wb') as outfile:
                        outfile.write(r.content)
                    #urllib.request.urlretrieve(url, download_location.format(region.zfill(2), cities[int(region)], region.zfill(2), bdate))
                    print('Downloaded {}'.format(url))
                    errcode = r.status_code

                elif r.status_code == 404:
                    print('No {}'.format(url))
                    errcode = 200 #skip
                else:
                    # try again
                    print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                    errcode = r.status_code
                    n_tries += 1
                    max_time_per_dl = max_time_per_dl * 2
                    time.sleep(n_tries ** 2) # wait a lil more every time 

            if n_tries == n_max_tries:
                print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))

            
    else:
        print('time mode inapplicable')




def download_metadata(n_max_tries, max_time_per_dl, version):

    baseurl = 'https://tenbou.nies.go.jp/download/'
    last_year_meta_av = 2021
    today = date.today()
    
    os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/'.format(version), exist_ok=True)
    download_location = '/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/'.format(version)
    
    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    options.add_argument("--headless")
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    driver.get(baseurl)
    time.sleep(2)

    # station tab ==========================================================================================================
    # switch to station button
    switch_panels(driver, 3, max_time_per_dl) # 3 stands for station tab
    time.sleep(5)

    filename = [file for file in os.listdir(download_location) if not file.endswith('.txt')][0] #os.path.isfile(os.path.join(download_location, file))]
    print(filename)
    if 'crdownload' in filename:
        filename = str(filename[:-11]) # cut out crdownload
    else:
        pass

    while not os.path.exists(download_location+filename):
        time.sleep(1)

        if os.path.isfile(download_location+filename):
            print('metadata download successful')

            # unzip
            with zipfile.ZipFile(download_location+filename, 'r') as zip_ref:
                zip_ref.extractall(download_location)
    
    os.remove(download_location+filename)

    # list all files
    meta_files = os.listdir(download_location)

    for file in meta_files:
        year = file[2:6]
        # unzip again
        with zipfile.ZipFile(download_location+file, 'r') as zip_ref:
            zip_ref.extractall(download_location)
        os.remove(download_location+file)

    driver.close()
    
    # JSON ==================================================================================
    # find english names of parameter
    data1970 = pd.read_csv('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/TM19700000.txt'.format(version), sep=',', encoding='ISO-8859-1')
    parameter_eng = list(data1970)

    # create json from original metadata files
    years = pd.date_range(date(1970, 1, 1), date(last_year_meta_av+1, 1, 1), freq='Y')
    json_metadata = {}
    
    for year in years:
        print(year)
        
        data = pd.read_csv('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/TM{}0000.txt'.format(version, year.strftime('%Y')), encoding='ISO-8859-1')
        if len(data.columns) == len(parameter_eng):
            data.columns = parameter_eng
        else:
            diff = len(data.columns) - len(parameter_eng)
            if diff < 0:
                data.columns = parameter_eng[:diff]
            else:
                parameter_eng += list(range(diff))
                data.columns = parameter_eng
        print('Number of columns: ', len(data.columns))

        data.to_csv('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/TM{}0000.txt'.format(version, year.strftime('%Y')), encoding='ISO-8859-1', index=False, header=True)

        with open('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/TM{}0000.txt'.format(version, year.strftime('%Y')), 'r', encoding='ISO-8859-1') as file:
            csv_filedata = csv.DictReader(file, delimiter=',')

            for row in csv_filedata:
                key = row['NIES_area_code']
                update_date = date(year.year, 1, 1).strftime('%Y-%m-%d')
                for parameter in row:
                    row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
                json_metadata[key] = row

        with open('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/JAPAN_NIES_META_{}.json'.format(version, year.strftime('%Y')), 'w', encoding='ISO-8859-1') as f:
            f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))
            
    
    # read standardised file to compare!
    with open('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/JAPAN_NIES_META_1970.json'.format(version), 'r', encoding='ISO-8859-1') as f:
        json_metadata = json.loads(f.read())

        for year in years[1:]:
            print(year)

            with open('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/network_provided/temp/JAPAN_NIES_META_{}.json'.format(version, year.strftime('%Y')), 'r', encoding='ISO-8859-1') as f:
                json_metadata_now = json.loads(f.read())

            for station in json_metadata: # loop through all the old stations
                if station in json_metadata_now.keys(): # if station is in current meta data, go on
                    for parameter in json_metadata[station]:
                        if parameter in json_metadata_now[station]: # check if column exists in new file
                            if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                                # if different value, append the standardised metadeta file
                                print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                                json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                                json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                            else:
                                pass
                        else:
                            print("Metadata parameter {} disappeared".format(parameter))
                else:
                    print('Station {} was abolished'.format(station))

            for station in json_metadata_now: # loop through all the new stations
                if station in json_metadata.keys(): # if station is in old meta data
                    pass # comparison was done before
                else: # new station appeared!
                    print('New station {}'.format(station))
                    json_metadata.update({station: json_metadata_now[station]})


        # safe
        with open('/esarchive/obs/ghost/JAPAN_NIES/metadata/{}/processed/JAPAN_NIES_META.json'.format(version), 'w', encoding='ISO-8859-1') as f:
            f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))
