from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests

import time
import pandas
from datetime import date
from datetime import timedelta
import os.path
import urllib
import pandas as pd


def scraper(mode, version):

    base_url = 'https://thredds.nilu.no/thredds/fileServer/ebas_doi/{}/{}/{}/{}.nc'

    if mode == 'all':
        bdate = date(2013, 12, 1)
        edate = date.today() 

        os.makedirs('/esarchive/obs/ghost/EBAS/original_files/{}/'.format(version), exist_ok=True)
        download_location = '/esarchive/obs/ghost/EBAS/original_files/'+version+'/{}.nc'

    elif mode == 'nrt':
        bdate = date(2024, 3, 1) #date.today() - timedelta(days = 1) # if code is run after 2 am, data from previous day will be available
        edate = date(2024, 3, 3) #date.today() - timedelta(days = 1)
        download_location = '/esarchive/obs/ghost/EBAS/original_files/nrt/{}.nc'

    else:
        print('time mode inapplicable')


    # import DOIs of datasets
    dois = pd.read_csv('EBAS_doilist.txt', header=None, names=["DOI"])

    # download
    for index, doi in dois.iterrows():
        doi_string = doi.loc["DOI"][-9:]

        url = base_url.format(doi_string[0:2], doi_string[2:4], doi_string[5:7], doi_string)
        r = requests.get(url, timeout=60)
        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location.format(doi_string))
            print('Downloaded DOI {}'.format(doi_string))
        else:
            print('No {}'.format(url))

        time.sleep(1)