from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
import requests
import time
from datetime import date
from datetime import timedelta
import zipfile

import os.path
import os
import pathlib
import pandas as pd
import shutil



def switch_panels(driver, tab_index):

    if tab_index == 0:
        mon_year = '"tab-pane fade active show"'
        hour = '"tab-pane fade"'
        bur = '"tab-pane fade"'

        search_btn = 'my_search_button'
        create_btn = 'my_create_archive'
        download_btn = 'my_download_archive'

    elif tab_index == 1:
        mon_year = '"tab-pane fade"'
        hour = '"tab-pane fade active show"'
        bur = '"tab-pane fade"'

        search_btn = 't_search_button'
        create_btn = 't_create_archive'
        download_btn = 't_download_archive'

    elif tab_index == 2:
        mon_year = '"tab-pane fade"'
        hour = '"tab-pane fade"'
        bur = '"tab-pane fade active show"'

        search_btn = 'tg_search_button'
        create_btn = 'tg_create_archive'
        download_btn = 'tg_download_archive'

    else:
        print("Error when switching tabs")

    # switch panels
    tab = driver.find_element(By.ID, 'monthly-yearly-panel')
    driver.execute_script("arguments[0].removeAttribute('class')", tab)
    driver.execute_script("arguments[0].setAttribute('class', {})".format(mon_year), tab)

    tab = driver.find_element(By.ID, 'time-gov-panel')
    driver.execute_script("arguments[0].removeAttribute('class')", tab)
    driver.execute_script("arguments[0].setAttribute('class', {})".format(bur), tab)

    tab = driver.find_element(By.ID, 'time-panel')
    driver.execute_script("arguments[0].removeAttribute('class')", tab)
    driver.execute_script("arguments[0].setAttribute('class', {})".format(hour), tab)


    # click buttons
    WebDriverWait(driver, 60).until(EC.element_to_be_clickable((By.ID, search_btn))) # wait till loaded
    search_button = driver.find_element(By.ID, search_btn)
    driver.execute_script("arguments[0].click();", search_button)
    
    WebDriverWait(driver, 60).until(EC.element_to_be_clickable((By.ID, create_btn))) # wait till loaded
    create_archive = driver.find_element(By.ID, create_btn)
    driver.execute_script("arguments[0].click();", create_archive)
    
    WebDriverWait(driver, 120).until(EC.element_to_be_clickable((By.ID, download_btn))) # wait till loaded
    download_button = driver.find_element(By.ID, download_btn)
    driver.execute_script("arguments[0].click();", download_button)



def scraper(mode, version):
    
    baseurl = 'https://tenbou.nies.go.jp/download/'

    if mode == 'all':
        bdate = date(1971, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today()

        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version), exist_ok=True)

        download_location = '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)

    elif mode == 'nrt':
        bdate = date(2024, 3, 2) #date.today() - timedelta(days = 1) # if code is run after 2 am, data from previous day will be available
        edate = date(2024, 3, 3) #date.today() - timedelta(days = 1)
        download_location = '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)

    else:
        print('time mode inapplicable')


    # create date array, daily; format to YYYYMMDD
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()

    cities = {1: "Hokkaido", 2: "Aomori", 3: "Iwate", 4: "Miyagi", 5: "Akita", 6: "Yamagata", 7: "Fukushima", 8: "Ibaraki", 9: "Tochigi", 10: "Gunma", 11: "Saitama", 12: "Chiba", 13: "Tokyo",
            14: "Kanagawa", 15: "Niigata", 16: "Toyama", 17: "Ishikawa", 18: "Fukui", 19: "Yamanashi", 20: "Nagano", 21: "Gifu", 22: "Shizuoka", 23: "Aichi", 24: "Mie", 25: "Shiga", 26: "Kyoto",
            27: "Osaka", 28: "Hyogo", 29: "Nara", 30: "Wakayama", 31: "Tottori", 32: "Shimane", 33: "Okayama", 34: "Hiroshima", 35: "Yamaguchi", 36:"Tokushima", 37:"Kagawa", 38:"Ehime",
            39:"Kochi", 40:"Fukuoka", 41:"Saga", 42:"Nagasaki", 43:"Kumamoto", 44:"Oita", 45: "Miyazaki", 46:"Kagoshima", 47:"Okinawa"}

    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    #options.add_argument("--headless")
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    driver.get(baseurl)
    time.sleep(2)

    # monthly tab ==========================================================================================================
    """
    # switch to monthly/yearly button
    switch_panels(driver, 0) # 0 stands for monthly tab

    filename = [file for file in os.listdir(download_location.format(version)) if os.path.isfile(os.path.join(download_location.format(version), file))][0]
    if 'crdownload' in filename:
        filename = str(filename[:-11]) # cut out crdownload
    else:
        pass

    # create monthly folder
    os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'monthly/', exist_ok=True)

    while not os.path.exists(download_location.format(version)+filename):
        time.sleep(1)

        if os.path.isfile(download_location.format(version)+filename):
            print('{} download successful'.format("monthly/yearly data"))

            # unzip
            with zipfile.ZipFile(download_location.format(version)+filename, 'r') as zip_ref:
                zip_ref.extractall(download_location.format(version)+'monthly/')
    
    os.remove(download_location.format(version)+filename)

    # sort files
    monthly_files = os.listdir(download_location.format(version)+'monthly/')

    for year in years:
        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'monthly/'+year+'/', exist_ok=True)
        for file in monthly_files:
            if year in file[:6]:
                # unzip again
                with zipfile.ZipFile(download_location.format(version)+'monthly/{}'.format(file), 'r') as zip_ref:
                    zip_ref.extractall(download_location.format(version)+'monthly/{}/{}.txt'.format(year, file[:-4]))
                os.remove(download_location.format(version)+'monthly/{}'.format(file))

    time.sleep(2)
"""
    # time tab ==========================================================================================================
    switch_panels(driver, 1) # 1 stands for hourly tab

    os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'hourly/', exist_ok=True)

    # wait until download finished
    time.sleep(10) # time for the file fo appear in directory
    filename = [file for file in os.listdir(download_location.format(version)) if os.path.isfile(os.path.join(download_location.format(version), file))][0]
    if 'crdownload' in filename:
        filename = str(filename[:-11]) # cut out crdownload
    else:
        pass
    print(filename)

    while not os.path.exists(download_location.format(version)+filename):
        time.sleep(1)

        if os.path.isfile(download_location.format(version)+filename):
            print('{} download successful'.format("hourly data"))

            # unzip
            with zipfile.ZipFile(download_location.format(version)+filename, 'r') as zip_ref:
                zip_ref.extractall(download_location.format(version)+'hourly/')
    
    #os.remove(download_location.format(version)+filename)

    # sort files
    hourly_files = os.listdir(download_location.format(version)+'hourly/')

    for file in hourly_files:
        # unzip again
        with zipfile.ZipFile(download_location.format(version)+'hourly/{}'.format(file), 'r') as zip_ref:
            zip_ref.extractall(download_location.format(version)+'hourly/')
        os.remove(download_location.format(version)+'hourly/{}'.format(file))

    hourly_folders = list(pathlib.Path(download_location.format(version)+'hourly/').rglob('*.zip')) #os.listdir(download_location.format(version)+'hourly/')

    for folder in hourly_folders:
        # unzip again
        with zipfile.ZipFile(folder, 'r') as zip_ref:
            zip_ref.extractall(download_location.format(version)+'hourly/') # takes 10 mins!
        os.remove(folder)

    # put all files in single new directory, delete everything else
    hourly_files_all = list(pathlib.Path(download_location.format(version)+'hourly/').rglob('*.txt'))
    os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/'.format(version), exist_ok=True)

    for file in hourly_files_all:
        filename = os.path.split(file)[-1]
        shutil.move(file, '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/{}'.format(version, filename))

    shutil.rmtree(download_location.format(version)+'hourly/')

    hourly_files_all = os.listdir('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/'.format(version))
    for city in cities:
        for year in range(2009, 2022):
            for file in hourly_files_all:
                if ((int(file[1:3]) == city) and (int(file[3:7]) == year)):
                    os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/hourly/{}/{}/'.format(version, cities[city], year), exist_ok=True)
                    shutil.move('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/temp/'.format(version)+file, '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/hourly/{}/{}/{}'.format(version, cities[city], year, file))


    shutil.rmtree(download_location.format(version)+'temp/')
    time.sleep(2)

    # bureau tab ==========================================================================================================
    switch_panels(driver, 2) # 1 stands for bureau tab

    os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/'.format(version)+'bureau/', exist_ok=True)

    # wait until download finished
    time.sleep(10) # time for the file fo appear in directory
    filename = [file for file in os.listdir(download_location.format(version)) if os.path.isfile(os.path.join(download_location.format(version), file))][0]
    if 'crdownload' in filename:
        filename = str(filename[:-11]) # cut out crdownload
    else:
        pass
    print(filename)

    while not os.path.exists(download_location.format(version)+filename):
        time.sleep(1)

        if os.path.isfile(download_location.format(version)+filename):
            print('{} download successful'.format("bureau data"))

            # unzip
            with zipfile.ZipFile(download_location.format(version)+filename, 'r') as zip_ref:
                zip_ref.extractall(download_location.format(version)+'bureau/')
    
    os.remove(download_location.format(version)+filename)

    # sort files
    bureau_files = os.listdir(download_location.format(version)+'bureau/')

    for file in bureau_files:
        with zipfile.ZipFile(download_location.format(version)+'bureau/{}'.format(file), 'r') as zip_ref:
            zip_ref.extractall(download_location.format(version)+'bureau/')
        os.remove(download_location.format(version)+'bureau/{}'.format(file))

    time.sleep(2)

    # check
    bureau_files = os.listdir(download_location.format(version)+'bureau/')
    bureau_files_remaining = [filename for filename in bureau_files if filename.endswith('.csv')]
    for remaining_file in bureau_files_remaining:
        year = os.path.split(remaining_file)[-1][:4]
        os.makedirs('/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/bureau/{}/'.format(version, year), exist_ok=True)
        shutil.move(remaining_file, '/esarchive/obs/ghost/JAPAN_NIES/original_files/{}/bureau/{}/'.format(version, year))

        

    driver.close()