from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import requests

import time
import pandas
from datetime import date
from datetime import timedelta
import os.path
import urllib
import pandas as pd


def scraper(mode, version):

    base_url = 'https://quotsoft.net/air/data'

    if mode == 'all':
        bdate = date(2013, 12, 1) #date(1960, 1, 1) # date before record starts
        edate = date(2013, 12, 31) #date.today() - timedelta(days = 1)

        os.makedirs('/esarchive/obs/ghost/BJMEMC/original_files/{}/'.format(version), exist_ok=True)

        download_location = '/esarchive/obs/ghost/BJMEMC/original_files/'+version+'/{}/{}.csv'

    elif mode == 'nrt':
        bdate = date(2024, 3, 1) #date.today() - timedelta(days = 1) # if code is run after 2 am, data from previous day will be available
        edate = date(2024, 3, 3) #date.today() - timedelta(days = 1)
        download_location = '/esarchive/obs/ghost/BJMEMC/original_files/nrt/{}/{}.csv'

    else:
        print('time mode inapplicable')


    # create date array, daily; format YYYYMMDD
    dates = pandas.date_range(bdate, edate, freq='d').strftime('%Y%m%d').tolist()

    # download
    for day in dates:
        url = '{}/beijing_all_{}.csv'.format(base_url, day)
        r = requests.get(url, timeout=120)
        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location.format('all', day))
            print('Downloaded {}'.format(url))
        else:
            print('No {}'.format(url))

        url = '{}/beijing_extra_{}.csv'.format(base_url, day)
        r = requests.get(url, timeout=120)
        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location.format('extra', day))
            print('Downloaded {}'.format(url))
        else:
            print('No {}'.format(url))


        time.sleep(1)

        # reorganize files per component
        components_all = ['AQI', 'PM2.5', 'PM2.5_24h', 'PM10', 'PM10_24h']
        components_extra = ['SO2', 'SO2_24h', 'NO2', 'NO2_24h','O3', 'O3_24h', 'CO', 'CO_24h']

        if mode == "nrt":
            for component in components_all:
                # all
                og_file = pd.read_csv(download_location.format('all', day))
                comp = og_file[og_file["type"] == component]

                output_path = download_location.format('all', component+"_{}".format(dates[0][0:4]))
                comp.to_csv(output_path, mode='a', header=not os.path.exists(output_path), index=False)

                # check if there is data doubled
                new_file = pd.read_csv(output_path).drop_duplicates()
                new_file.to_csv(output_path, index=False)

            os.remove(download_location.format('all', day))


            for component in components_extra:
                # extra
                og_file = pd.read_csv(download_location.format('extra', day))
                comp = og_file[og_file["type"] == component]

                output_path = download_location.format('extra', component+"_{}".format(dates[0][0:4]))
                comp.to_csv(output_path, mode='a', header=not os.path.exists(output_path), index=False)

                # check if there is data doubled
                new_file = pd.read_csv(output_path).drop_duplicates()
                new_file.to_csv(output_path, index=False)


            # delete original files
            os.remove(download_location.format('extra', day))
