import requests
import time
import pandas as pd
from datetime import date
from datetime import timedelta
import os.path
import urllib
import tarfile
import shutil
import gzip



def scraper(mode, version):

    base_url = 'http://www.aire.cdmx.gob.mx/opendata/anuales_horarios_gz/contaminantes_{}.csv.gz'


    if mode == 'all':
        bdate = date(1980, 1, 1) #date(1960, 1, 1) # date before record starts
        edate = date.today()

        os.makedirs('/esarchive/obs/ghost/MEXICO_CDMX/original_files/{}/'.format(version), exist_ok=True)

        download_location = '/esarchive/obs/ghost/MEXICO_CDMX/original_files/'+version+'/contaminantes_{}.csv.gz'

    elif mode == 'nrt':
        bdate = date(2024, 3, 2) #date.today() - timedelta(days = 1) # if code is run after 2 am, data from previous day will be available
        edate = date(2024, 3, 3) #date.today() - timedelta(days = 1)
        download_location = '/esarchive/obs/ghost/MEXICO_CDMX/original_files/nrt/contaminantes_{}.csv.gz'

    else:
        print('time mode inapplicable')


    # create date array, daily; format to YYYYMMDD
    years = pd.date_range(bdate, edate, freq='Y').strftime('%Y').tolist()

    # download
    for year in years:
        url = base_url.format(year)
        r = requests.get(url, timeout=120)
        if r.status_code == 200:
            urllib.request.urlretrieve(url, download_location.format(year))
            print('Downloaded {}'.format(url))

            # unzip
            with gzip.open(download_location.format(year), 'rb') as f_in:
                with open(download_location.format(year)[:-3], 'wb') as f_out:
                    shutil.copyfileobj(f_in, f_out)
                

            # remove files
            os.remove(download_location.format(year))

        else:
            print('No {}'.format(url))

        time.sleep(1)
