import requests
import time
import pandas as pd
from datetime import date
from datetime import timedelta
import os.path
import urllib



def scraper(mode, version):

    base_url = 'https://quotsoft.net/air/data'


    if mode == 'all':
        bdate = date(2014, 5, 1) #date(1960, 1, 1) # date before record starts
        edate = date(2014, 5, 31) #date.today() - timedelta(days = 1)

        os.makedirs('/esarchive/obs/ghost/CNEMC/original_files/{}/'.format(version), exist_ok=True)

        download_location = '/esarchive/obs/ghost/CNEMC/original_files/'+version+'/{}/{}.csv'

    elif mode == 'nrt':
        bdate = date(2024, 3, 2) #date.today() - timedelta(days = 1) # if code is run after 2 am, data from previous day will be available
        edate = date(2024, 3, 3) #date.today() - timedelta(days = 1)
        download_location = '/esarchive/obs/ghost/CNEMC/original_files/nrt/{}/{}.csv'

    else:
        print('time mode inapplicable')


    # create date array, daily; format to YYYYMMDD
    dates = pd.date_range(bdate, edate, freq='d').strftime('%Y%m%d').tolist()
    print(dates)

    # download
    for day in dates:
        url = '{}/china_sites_{}.csv'.format(base_url, day)
        r = requests.get(url, timeout=120)
        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location.format('sites', day))
            print('Downloaded {}'.format(url))
        else:
            print('No {}'.format(url))

        url = '{}/china_cities_{}.csv'.format(base_url, day)
        r = requests.get(url, timeout=120)
        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location.format('cities', day))
            print('Downloaded {}'.format(url))
        else:
            print('No {}'.format(url))


        time.sleep(1)

        # reorganize files per component
        components = ['AQI', 'PM2.5', 'PM2.5_24h', 'PM10', 'PM10_24h','SO2', 'SO2_24h', 'NO2', 'NO2_24h','O3', 'O3_24h', 'O3_8h', 'O3_8h_24h', 'CO', 'CO_24h']

        if mode == "nrt":
            for component in components:
                # sites
                og_file = pd.read_csv(download_location.format('sites', day))
                comp = og_file[og_file["type"] == component]

                output_path = download_location.format('sites', component+"_{}".format(dates[0][0:4]))
                comp.to_csv(output_path, mode='a', header=not os.path.exists(output_path), index=False)

                # check if there is data doubled
                new_file = pd.read_csv(output_path).drop_duplicates()
                new_file.to_csv(output_path, index=False)


                # cities
                og_file = pd.read_csv(download_location.format('cities', day))
                comp = og_file[og_file["type"] == component]

                output_path = download_location.format('cities', component+"_{}".format(dates[0][0:4]))
                comp.to_csv(output_path, mode='a', header=not os.path.exists(output_path), index=False)

                # check if there is data doubled
                new_file = pd.read_csv(output_path).drop_duplicates()
                new_file.to_csv(output_path, index=False)


            # delete original files
            os.remove(download_location.format('sites', day))
            os.remove(download_location.format('cities', day))
