from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from bs4 import BeautifulSoup
from chromedriver_py import binary_path

import tarfile 
import requests
import time
import os.path
import os

from compare_two_files import compare_files


def scraper(mode, version):

    # paths and variables
    variables = ["co", "ch4"]
    var = ["CO", "CH4"] # name of directories

    login_url = 'https://gaw.kishou.go.jp/login/user'
    url = 'https://gaw.kishou.go.jp/search/gas_species/{}/latest'

    
    if mode == "all":

        os.makedirs('/esarchive/obs/ghost/WMO_WDCGG/original_files/{}/'.format(version), exist_ok=True)
        download_location = "/esarchive/obs/ghost/WMO_WDCGG/original_files/{}/".format(version)

    elif mode == "nrt":

        os.makedirs('/esarchive/obs/ghost/WMO_WDCGG/original_files/nrt/temp/', exist_ok=True)
        download_location = "/esarchive/obs/ghost/WMO_WDCGG/original_files/nrt/temp/"

    # set up driver
    options = Options()
    prefs = {'download.default_directory' : download_location}
    options.add_experimental_option('prefs', prefs)
    options.add_argument("--no-sandbox")
    svc = webdriver.ChromeService(executable_path=binary_path)
    driver = webdriver.Chrome(service=svc, options=options)

    # open url
    driver.get(login_url)
    time.sleep(2)

    # login
    email = driver.find_element(By.ID, "uid") 
    email.send_keys("raphael.grodofzig@bsc.es")
    passwd = driver.find_element(By.ID, "pwd") 
    passwd.send_keys("IbRGaH66")
    time.sleep(1)
    driver.find_element(By.ID, "loginbt").click()
    print("Successfully logged in")
    time.sleep(2)


    for variable in variables:
    
        driver.get(url.format(variable))

        WebDriverWait(driver, 60).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "button[class='btn'][type='submit']"))) # wait till loaded

        # start download
        driver.find_element(By.CSS_SELECTOR, "button[class='btn'][type='submit']").click()

        # wait until download finished
        while not os.path.exists("{}{}_latest.txt.tar.gz".format(download_location, variable)):
            time.sleep(1)

            if os.path.isfile("{}{}_latest.txt.tar.gz".format(download_location, variable)):
                print('{} download successful'.format(variable))

    driver.close()

    if mode == "nrt":
        i=0
        for variable in variables:

            # compare old and new files after download
            same_files = compare_files("{}{}_latest.txt.tar.gz".format(download_location, variable), "{}{}_latest.txt.tar.gz".format(download_location[:-5], variable))

            if same_files == True:
                # delete file in temp
                os.remove("{}{}_latest.txt.tar.gz".format(download_location, variable))
            else:
                # delete old file, move new file
                os.remove("{}{}_latest.txt.tar.gz".format(download_location[:-5], variable))
                os.rename("{}{}_latest.txt.tar.gz".format(download_location, variable), "{}{}_latest.txt.tar.gz".format(download_location[:-5], variable))

                # untar
                file = tarfile.open("{}{}_latest.txt.tar.gz".format(download_location[:-5], variable)) 
                file.extractall(download_location[:-5])
                file.close()

                # move files
                os.rename(download_location[:-5]+"txt/"+variable, download_location[:-5]+var[i])
                os.rmdir(download_location[:-5]+"txt/")
                i = i+1
    

    elif mode == "all":
        i=0
        for variable in variables:
            # untar
            file = tarfile.open("{}{}_latest.txt.tar.gz".format(download_location, variable)) 
            file.extractall(download_location)
            file.close()

            # remove zips
            os.remove("{}{}_latest.txt.tar.gz".format(download_location, variable))

            # move files
            os.rename(download_location+"txt/"+variable, download_location+var[i])
            os.rmdir(download_location+"txt/")
            i = i+1
    


