from selenium import webdriver
from chromedriver_py import binary_path
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait # type: ignore
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException
from bs4 import BeautifulSoup


import time
import datetime
from datetime import date
from datetime import timedelta
import pandas as pd
import os.path
import time
import shutil
import os
import re
import csv
import json
import metadata_parsing_functions
import requests


def download_data(mode, version, n_max_tries, max_time_per_dl, start, stop, dl_root):

    if mode == 'all':
        # create download directory
        bdate = date(1990, 1, 1).strftime('%Y%m%d')[2:] # first date SINCA was reporting
        edate = date.today().strftime('%Y%m%d')[2:]

    elif mode == 'nrt':
        bdate = start.strftime('%Y%m%d')[2:]
        edate = stop.strftime('%Y%m%d')[2:]
        print(bdate)
        print(edate)

    else:
        print('time mode inapplicable')

    base_url = 'https://sinca.mma.gob.cl/cgi-bin/APUB-MMA/apub.tsindico2.cgi?outtype=xcl&macro=./{}/{}/Cal/{}//{}.{}.ic&from={}&to={}&path=/usr/airviro/data/CONAMA/&lang=esp&rsrc=&macropath='

    components = {'PM10':       'PM10', 
                'PM2.5':        'PM25', 
                'PM10disc':     'PM1D', 
                'SO2':          '0001',
                'NO2':          '0003',
                'NOX':          '0NOX',
                'NO':           '0002',
                'CO':           '0004',
                'O3':           '0008',
                'Pb':           '00Pb', 
                'As':           'ARSE', 
                'Cu':           '00Cu',
                'CH4':          '0CH4', 
                'NMHC':         'NMHC',
                'HC':           'THCM'}
    
    time_resolutions = {'diario.diario': 'daily', 'horario.horario': 'hourly'}

    with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root, version), 'r') as file:
        meta_data = json.load(file)

    if mode == 'nrt':
        version = mode #change this in order to create meaningful directories, needs to be after import of metadata

    for station_ID in meta_data:
        region = station_ID.split('_')[1]
        station = station_ID.split('_')[2]
        print(region)

        # create directory
        os.makedirs('{}/CHILE_SINCA/original_files/{}/{}/'.format(dl_root,version, station_ID), exist_ok=True)
        download_location = '{}/CHILE_SINCA/original_files/{}/{}/'.format(dl_root,version, station_ID)

        for component in components:
            if meta_data[station_ID][component+'_instrument']['values'][-1] != '': # component is measured at this station
                variable = components[component]

                for time_resolution in time_resolutions:

                    n_tries = 0
                    errcode = 999
                    url = base_url.format(region, station, variable, variable, time_resolution, bdate, edate)
                    while (n_tries < n_max_tries) and (errcode != 200):
                        r = requests.get(url, timeout=max_time_per_dl)
                        if r.status_code == 200:

                            if 'No such file or directory' in str(r.content):
                                errcode = 200
                                continue
                            else:
                                with open(download_location+component+'_'+time_resolutions[time_resolution]+'.csv', 'wb') as outfile:
                                    outfile.write(r.content)
                                print('Downloaded {} {} {} {}'.format(region, station, variable, time_resolution))
                                errcode = r.status_code

                        elif r.status_code == 404:
                            print('No {} {} {} {}'.format(region, station, variable, time_resolution))
                            errcode = 200 #skip
                            continue
                        else:
                            # try again
                            print('Response error {}, attempt {}'.format(r.status_code, n_tries))
                            errcode = r.status_code
                            n_tries += 1
                            time.sleep(n_tries * 2) # wait a lil more every time 

                    if n_tries == n_max_tries:
                        print('Failed downloading {} {} times in {} seconds, error code {}'.format(url, n_tries, max_time_per_dl, errcode))
            else:
                continue

   
    """ DOWNLOAD VIA WEBSCRAPING +++++++++++++++++++++++++++++++++++++++++

    # paths and variables
    variables_website = ['MP 10','MP 2,5', 'MP 10discreto','MP 2,5discreto', 'SO2', 'NO2', 'NOX', 'NO', 'CO', 'O3', 'Pb', 'As', 'Cu', 'MPS', 'CH4', 'HCNM', 'HC', 'V', 'Ni', 'Se', 'TRS'] 
    variables_text = ["Material particulado MP 10", "Material particulado MP 2,5", "Material particulado 10 micrometros discreto", "Material particulado 2.5 micrometros discreto", 'Dióxido de azufre', 'Dióxido de nitrógeno', 'Óxidos de nitrógeno', 'Monóxido de nitrógeno', 'Monóxido de carbono', 'Ozono', 'Plomo', 'Arsénico', 'Cobre', 'MPS', 'Metano', 'Hidrocarburos no metánicos', 'Hidrocarburos totales', 'V', 'Ni', 'Se', 'TRS']
    time_resolutions_website = ["registro diario", "registro horario"] 

    variables_ghost = ['PM10', 'PM2.5', 'PM10disc', 'PM2.5disc', 'SO2', 'NO2', 'NOX', 'NO','CO', 'O3','Pb', 'As', 'Cu', 'MPS', 'CH4', 'NMHC', 'HC', 'V', 'Ni', 'Se', 'TRS'] # needs to be same order as variables_website!
    time_resolution_ghost = ['daily', 'hourly']

    with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'r', encoding='utf-8') as f:
        metadata = json.loads(f.read())

    baseurl = 'https://sinca.mma.gob.cl/'

    if mode == 'all':
        # create download directory
        os.makedirs('{}/CHILE_SINCA/original_files/{}/temp/'.format(dl_root,version), exist_ok=True)
        download_location = '{}/CHILE_SINCA/original_files/{}/temp/'.format(dl_root,version)

    elif mode == 'nrt':
        bdate = date(date.today().year, 1, 1).strftime('%Y%m%d')[2:] #"240101"
        edate = date((date.today() + timedelta(days=31)).year, (date.today() + timedelta(days=31)).month, 1).strftime('%Y%m%d')[2:] # end date is the first of the upcoming month, for downloading reasons
        edate = date.today().strftime('%Y%m%d')[2:]
        print(bdate)
        print(edate)

        # create download directory
        version = mode
        os.makedirs('{}/CHILE_SINCA/original_files/{}/temp/'.format(dl_root,version), exist_ok=True)
        download_location = '{}/CHILE_SINCA/original_files/{}/temp/'.format(dl_root,version)

    else:
        print('time mode inapplicable')


    # let's go ------------------------------------------------------------------------------
    n_tries = 0
    errcode = 999
    while (n_tries < n_max_tries) and (errcode != 200):
        try:
            # set up driver
            options = Options()
            prefs = {'download.default_directory' : download_location}
            options.add_experimental_option('prefs', prefs)
            options.add_argument("--no-sandbox")
            #options.add_argument("disable-infobars")
            #options.add_argument("--disable-extensions")
            #options.add_argument("--disable-gpu")
            #options.add_argument("--disable-dev-shm-usage")
            # if n_tries > 0:
            options.add_argument("--headless=chrome")

            svc = webdriver.ChromeService(executable_path=binary_path)
            driver = webdriver.Chrome(service=svc, options=options)

            driver.get(baseurl)
            WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.XPATH, '//a[contains(@href, "/index.php/region/index/id/")]'))) # wait till loaded

            # navigate to regions
            html = driver.page_source
            soup = BeautifulSoup(html, features="html.parser")
            regions = soup.find_all("a", href=re.compile(r"^/index.php/region/index/id/"))
            #regions = soup.find_all("a", {"href": "/index.php/region/index/id/V"}) # Valparaiso
            
            # loop over regions
            for region in regions:

                print("Region is "+region.getText())
                #dl_logfile.write('**************************************************************************************************************\n')
                #dl_logfile.write("Region is "+region.getText()+'\n')

                driver.get("https://sinca.mma.gob.cl/"+region.get("href"))
                time.sleep(2)

                # navigate to station and component
                html_region = driver.page_source
                soup_region = BeautifulSoup(html_region, features="html.parser")
                a_titles = [a.get("title") for a in soup_region.find_all("a", title=True)] # all links
                # from all the links, choose only the components
                stations_components = []
                for a_title in a_titles:
                    #for variable_text in variables_text:
                    if '|' in a_title: # choose iframe with data
                        stations_components.append(soup_region.find("a", {"title": a_title}))

                # loop through all stations and components of the region
                for station_component in stations_components:

                    #print(station_component.get("title"))
                    station = station_component.get("title").split("| ", 1)[1]
                    component = [x for x in variables_text if x in station_component.get("title")][0] # get component name on website
                    component_choose_time_res = variables_website[variables_text.index(component)] # get component name for choosing time resolution
                    component_ghost = variables_ghost[variables_text.index(component)] # get component name accordingly in ghost

                    comune = station_component.find_parent().find_parent().find("div", {'class': 'comuna'}).text
                    station = station_component.find_parent().find_parent().find("a", href=True).text
                    # print('Comune is {}'.format(comune))
                    # print('Station is {}'.format(station))

                    # create storage directory
                    try:
                        for station_reference in metadata: # loop through metadata to find station ID of current station
                            if (metadata[station_reference]['station_name']['values'][-1] == station) and (metadata[station_reference]['commune']['values'][-1] == comune): # match station with previously referenced station reference from old file; some stations have same names, so also filter for communes
                                station_id = station_reference
                            else:
                                pass
                            
                        os.makedirs('{}/CHILE_SINCA/original_files/{}/{}/'.format(dl_root,version, station_id), exist_ok=True)
                        storage_location = '{}/CHILE_SINCA/original_files/{}/{}/'.format(dl_root,version, station_id)

                    except:
                        print("{} no station_id found in old metadata, did not download data from new station".format(station))
                        #dl_logfile.write("{} no station_id found in old metadata, did not download data from new station\n".format(station))
                        continue # ============================cover this case ==============================================

                    # go to data on website
                    driver.get('https:'+station_component.get("href"))
                    time.sleep(5)
                    
                    driver.switch_to.frame("left")

                    # select time resolution
                    dropdown_element = driver.find_element(By.ID, 'ic')
                    select = Select(dropdown_element)
                    options = [opt.get_attribute("text") for opt in select.options]
                    
                    i=0 # iterate time resolutions
                    for time_resolution in time_resolutions_website:
                        #select time resolution if existent!
                        if (component_choose_time_res+' - '+time_resolution) in options:
                            select.select_by_visible_text(component_choose_time_res+' - '+time_resolution)
                            
                            #print("Time resolution is: {}".format(time_resolution))
                            time.sleep(5)

                            if mode == "all":
                                start_date = driver.find_element(By.ID, "from").get_attribute("value")
                                end_date = driver.find_element(By.ID, "to").get_attribute("value")
                            if mode == "nrt": # update dates

                                start = driver.find_element(By.ID, "from")
                                driver.execute_script("arguments[0].removeAttribute('value')", start)
                                driver.execute_script("arguments[0].value = {};".format(bdate), start)
                                start.click()
                                time.sleep(3) # important!
                                driver.find_element(By.LINK_TEXT, '1').click() # click on calendar: 1st of January

                                end = driver.find_element(By.ID, "to")
                                driver.execute_script("arguments[0].removeAttribute('value')", end)
                                driver.execute_script("arguments[0].value = {};".format(edate), end)
                                end.click()
                                time.sleep(2) # important!
                                driver.find_element(By.LINK_TEXT, edate[-2:]).click() # click on calendar: today

                                start_date = start.get_attribute("value")
                                end_date = end.get_attribute("value")

                                time.sleep(2)

                            driver.switch_to.default_content()
                            driver.switch_to.frame("right")

                            time.sleep(5)
                            WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.LINK_TEXT, "Excel CSV"))) # wait till loaded

                            driver.find_element(By.LINK_TEXT, "Excel CSV").click()

                            # wait until download finished
                            while not os.path.exists("{}/datos_{}_{}.csv".format(download_location, start_date, end_date)):
                                time.sleep(1)

                                if os.path.isfile("{}/datos_{}_{}.csv".format(download_location, start_date, end_date)):
                                    print('{} {} download successful'.format(station_component.get("title"), time_resolution_ghost[i]))
                                    #dl_logfile.write('{} {} download successful\n'.format(station_component.get("title"), time_resolution_ghost[i]))
                                    shutil.copyfile("{}/datos_{}_{}.csv".format(download_location, start_date, end_date), storage_location+component_ghost+'_'+time_resolution_ghost[i]+'.csv')

                            os.remove("{}/datos_{}_{}.csv".format(download_location, start_date, end_date))    

                            driver.switch_to.default_content()
                            driver.switch_to.frame("left")

                        i=i+1
                    
            driver.close()
            errcode = 200

        except WebDriverException as e:
            print(e) 
            #dl_logfile.write(e)
            n_tries = n_tries+1
            print("Number of tries: {} ".format(n_tries))
            #dl_logfile.write("Number of tries: {}\n".format(n_tries))
            continue

        except ConnectionRefusedError as e:
            print(e)
            #dl_logfile.write(e)
            n_tries = n_tries+1
            print("Number of tries: {} ".format(n_tries))
            #dl_logfile.write("Number of tries: {}\n".format(n_tries))
            continue

    if n_tries == n_max_tries:
        print('Failed downloading CHILE_SINCA data {} times in {} seconds'.format(n_tries, max_time_per_dl))"""



"""
def download_metadata_webscraper(n_max_tries, max_time_per_dl, version, dl_root):


    variables_ghost = {'PM10':      ['PM10', 'MP 10', "Material particulado MP 10"], 
                       'PM2.5':     ['PM2.5', 'MP 2,5', "Material particulado MP 2,5"], 
                       'PM10disc':  ['PM10disc', 'MP 10discreto', "Material particulado 10 micrometros discreto"], 
                       'SO2':       ['SO2', 'Dióxido de azufre'],
                       'NO2':       ['NO2', 'Dióxido de nitrógeno'],
                       'NOX':       ['NOX', 'Óxidos de nitrógeno'],
                       'NO':        ['NO', 'Monóxido de nitrógeno'],
                       'CO':        ['CO', 'Monóxido de carbono'],
                        'O3':       ['O3', 'Ozono', 'Ozono.-'],
                        'Pb':       ['Pb', 'Plomo'], 
                        'As':       ['As', 'Arsénico'], 
                        'Cu':       ['Cu', 'Cobre'],
                        'MPS':      ['MPS'],
                        'CH4':      ['CH4', 'Metano'], 
                        'NMHC':     ['NMHC', 'HCNM', 'Hidrocarburos no metánicos'],
                        'HC':       ['THCM', 'HCT', 'HC', 'Hidrocarburos totales'], # ???????
                        'V':        ['V'],
                        'Ni':       ['Ni'],
                        'Se':       ['Se'],
                        'TRS':      ['TRS']}
    

    baseurl = 'https://sinca.mma.gob.cl/index.php/'
    today = date.today()
    
    # create json metadata base file! =============================================================================================
    # parameters = ['station_reference','station_name','region','province','commune','UTM_lon','UTM_lat','timezone','O3_instrument','NO_instrument','NO2_instrument','CO_instrument','CH4_instrument','SO2_instrument','NMHC_instrument','HC_instrument','PM10_instrument','PM2.5_instrument','As_instrument','Cu_instrument','Pb_instrument','comments']
    # n_tries = 0
    # errcode = 999
    # metadata_new = {}
    # while (n_tries < n_max_tries) and (errcode != 200):
    #     try:
    #         # set up driver
    #         options = Options()
    #         #prefs = {'download.default_directory' : download_location}
    #         #options.add_experimental_option('prefs', prefs)
    #         options.add_argument("--no-sandbox")
    #         options.add_argument("--headless")
    #         svc = webdriver.ChromeService(executable_path=binary_path)
    #         driver = webdriver.Chrome(service=svc, options=options)

    #         # open url
    #         driver.get(baseurl)
    #         WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.XPATH, '//a[contains(@href, "/index.php/region/index/id/")]'))) # wait till loaded

    #         # navigate to regions
    #         html = driver.page_source
    #         soup = BeautifulSoup(html, features="html.parser")
    #         regions = soup.find_all("a", href=re.compile(r"^/index.php/region/index/id/"))
    #         #regions = soup.find_all("a", {"href": "/index.php/region/index/id/VIII"}) # Biobio

    #         j = 0 
    #         for region in regions:
    #             j = j+1
    #             region_ind = str(j).zfill(2)

    #             print("Region is "+region.getText())

    #             driver.get("https://sinca.mma.gob.cl/"+region.get("href"))
    #             time.sleep(1)

    #             html = driver.page_source
    #             soup = BeautifulSoup(html, features="html.parser")
    #             stations = soup.find_all("a", href=re.compile(r"^/index.php/estacion/index/id/"), title=None)

    #             k = 0
    #             for station in stations: # loop over new stations
    #                 k = k +1
    #                 station_ind = str(k).zfill(3)

    #                 station_name_new = station.getText()
    #                 print(station_name_new)

    #                 driver.get("https://sinca.mma.gob.cl/"+station.get("href"))
    #                 time.sleep(1)
                    
    #                 # get meta info
    #                 html = driver.page_source
    #                 soup = BeautifulSoup(html, features="html.parser")
                    
    #                 region = soup.find("th", text="Región").find_next_sibling().getText().replace(',', '')
    #                 province = soup.find("th", text="Provincia").find_next_sibling().getText().replace(',', '')
    #                 commune = soup.find("th", text="Comuna").find_next_sibling().getText().replace(',', '')
    #                 UTM_coordinates = soup.find("th", text="Coordenadas UTM").find_next_sibling().getText().replace(' ', '')
    #                 lon = UTM_coordinates.split('E')[0]+'E'
    #                 lat = UTM_coordinates.split('E')[1].split("\n")[0]
    #                 timezone = soup.find("th", text="Huso horario").find_next_sibling().getText().replace(' ', '')

    #                 ins_table = soup.find('table', id="medicion")
    #                 if ins_table is not None: # check if there are instruments for air pollution at this station
    #                     instruments = ins_table.find_all("td", {"class": "helpTecnica center"})
    #                     instruments_per_component = {}
    #                 else:
    #                     continue

    #                 for instrument in instruments:
    #                     component_sinca = instrument.find_parent().find('a').getText()

    #                     for variable_ghost in variables_ghost: # map components to ghost standards
    #                         if component_sinca in variables_ghost[variable_ghost]:
    #                             component = variable_ghost

    #                     #print(component_sinca, 'mapped to -->', component)

    #                     if "No informado" in instrument.getText():
    #                         instruments_per_component[component] = ''
    #                     else:
    #                         instrument_name = instrument.getText().strip()
    #                         instrument_name = instrument_name.split("\n")[-1]
    #                         instruments_per_component[component] = instrument_name
                
    #                 station_reference = 'CL'+str(region_ind)+'_'+str(station_ind)
    #                 print(station_reference)

    #                 metadata_new[station_reference] = {} # create inner dictionary
    #                 scraped_metadata = [station_reference, station_name_new, region, province, commune, lon, lat, timezone]
    #                 i=0
    #                 for parameter in parameters: # loop through the meta parameters
    #                     if ("instrument" not in parameter) and ("comments" not in parameter): # go through all that are not instruments
    #                         metadata_new[station_reference][parameter] = {"values": [scraped_metadata[i]], "update_time": [today.strftime('%Y-%m-%d')]}
    #                         i=i+1
    #                     elif "comments" == parameter:
    #                         metadata_new[station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]}
    #                     else: # go through the instruments
    #                         metadata_new[station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]} # write in all instruments empty list
    #                         for component in instruments_per_component:
    #                             if component in parameter: # if instrument matches component, overwrite empty instrument list
    #                                 metadata_new[station_reference][parameter] = {"values": [instruments_per_component[component]], "update_time": [today.strftime('%Y-%m-%d')]}

    #         # safe
    #         with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'w', encoding='utf-8') as f:
    #             f.write(json.dumps(metadata_new, indent=4, ensure_ascii=False))

    #         driver.close()
    #         errcode = 200

    #     except WebDriverException as e:
    #         print(e) 
    #         n_tries = n_tries+1
    #         print("Number of tries: {}".format(n_tries))
    #         continue

    # if n_tries == n_max_tries:
    #     print('Failed downloading CHILE_SINCA metadata {} times in {} seconds'.format(n_tries, max_time_per_dl))

    
    with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'r', encoding='utf-8') as f:
        metadata_old = json.loads(f.read())

    station_commune_old = [str(metadata_old[station_reference]['station_name']['values'][-1]).replace(' ', '')+'_'+str(metadata_old[station_reference]['commune']['values'][-1]).replace(' ', '') for station_reference in metadata_old]
    print(station_commune_old) # needed to determine if new station appeared
    
    n_tries = 0
    errcode = 999
    metadata_new = {}
    while (n_tries < n_max_tries) and (errcode != 200):
        try:
            # set up driver
            options = Options()
            #prefs = {'download.default_directory' : download_location}
            #options.add_experimental_option('prefs', prefs)
            options.add_argument("--no-sandbox")
            options.add_argument("--headless")
            svc = webdriver.ChromeService(executable_path=binary_path)
            driver = webdriver.Chrome(service=svc, options=options)

            # open url
            driver.get(baseurl)
            WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.XPATH, '//a[contains(@href, "/index.php/region/index/id/")]'))) # wait till loaded

            # navigate to regions
            html = driver.page_source
            soup = BeautifulSoup(html, features="html.parser")
            regions = soup.find_all("a", href=re.compile(r"^/index.php/region/index/id/"))
            #regions = soup.find_all("a", {"href": "/index.php/region/index/id/VIII"}) # Biobio

            j = 0 
            for region in regions:
                j = j+1
                region_ind = str(j).zfill(2)

                print("Region is "+region.getText())

                driver.get("https://sinca.mma.gob.cl/"+region.get("href"))
                time.sleep(1)

                html = driver.page_source
                soup = BeautifulSoup(html, features="html.parser")
                stations = soup.find_all("a", href=re.compile(r"^/index.php/estacion/index/id/"), title=None)

                k = 0
                for station in stations: # loop over new stations
                    k = k +1
                    station_ind = str(k).zfill(3)

                    station_name_new = station.getText()
                    print(station_name_new)

                    driver.get("https://sinca.mma.gob.cl/"+station.get("href"))
                    time.sleep(1)
                    
                    # get meta info
                    html = driver.page_source
                    soup = BeautifulSoup(html, features="html.parser")
                    
                    region = soup.find("th", text="Región").find_next_sibling().getText().replace(',', '')
                    province = soup.find("th", text="Provincia").find_next_sibling().getText().replace(',', '')
                    commune = soup.find("th", text="Comuna").find_next_sibling().getText().replace(',', '')
                    UTM_coordinates = soup.find("th", text="Coordenadas UTM").find_next_sibling().getText().replace(' ', '')
                    lon = UTM_coordinates.split('E')[0]+'E'
                    lat = UTM_coordinates.split('E')[1].split("\n")[0]
                    timezone = soup.find("th", text="Huso horario").find_next_sibling().getText().replace(' ', '')

                    ins_table = soup.find('table', id="medicion")
                    if ins_table is not None: # check if there are instruments for air pollution at this station
                        instruments = ins_table.find_all("td", {"class": "helpTecnica center"})
                        instruments_per_component = {}
                    else:
                        continue

                    for instrument in instruments:
                        component_sinca = instrument.find_parent().find('a').getText()

                        for variable_ghost in variables_ghost: # map components to ghost standards
                            if component_sinca in variables_ghost[variable_ghost]:
                                component = variable_ghost

                        #print(component_sinca, 'mapped to -->', component)

                        if "No informado" in instrument.getText():
                            instruments_per_component[component] = ''
                        else:
                            instrument_name = instrument.getText().strip()
                            instrument_name = instrument_name.split("\n")[-1]
                            instruments_per_component[component] = instrument_name


                    for station_reference in metadata_old:

                        if (metadata_old[station_reference]['station_name']['values'][-1] == station_name_new) and (metadata_old[station_reference]['commune']['values'][-1] == commune): # match station with previously referenced station reference from old file; some stations have same names, so also filter for communes
                            i=0
                            metadata_new[station_reference] = {} # create inner dictionary
                            scraped_metadata = [station_reference, station_name_new, region, province, commune, lon, lat, timezone]
                            for parameter in metadata_old[station_reference]: # loop through the meta parameters
                                if ("instrument" not in parameter) and ("comments" not in parameter): # go through all that are not instruments
                                    metadata_new[station_reference][parameter] = {"values": [scraped_metadata[i]], "update_time": [today.strftime('%Y-%m-%d')]}
                                    i=i+1
                                elif "comments" == parameter:
                                    metadata_new[station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]}
                                else: # go through the instruments
                                    metadata_new[station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]} # write in all instruments empty list
                                    for component in instruments_per_component:
                                        if component in parameter: # if instrument matches component, overwrite empty instrument list
                                            metadata_new[station_reference][parameter] = {"values": [instruments_per_component[component]], "update_time": [today.strftime('%Y-%m-%d')]}

                        else: 
                            pass # no match
                            
                    # take care of cases of new station!
                    if str(station_name_new).replace(' ', '')+'_'+str(commune).replace(' ', '') not in station_commune_old: # new station!
                        new_station_reference = 'CL'+str(region_ind)+'_'+str(station_ind)
                        print('New station {} {} with new station reference {}'.format(station_name_new, commune, new_station_reference))

                        i=0
                        metadata_new[new_station_reference] = {} # create inner dictionary
                        scraped_metadata = [new_station_reference, station_name_new, region, province, commune, lon, lat, timezone]
                        for parameter in metadata_old[station_reference]: # loop through the meta parameters
                            if ("instrument" not in parameter) and ("comments" not in parameter): # go through all that are not instruments
                                metadata_new[new_station_reference][parameter] = {"values": [scraped_metadata[i]], "update_time": [today.strftime('%Y-%m-%d')]}
                                i=i+1
                            elif "comments" == parameter:
                                metadata_new[new_station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]}
                            else: # go through the instruments
                                metadata_new[new_station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]} # write in all instruments empty list
                                for component in instruments_per_component:
                                    if component in parameter: # if instrument matches component, overwrite empty instrument list
                                        metadata_new[new_station_reference][parameter] = {"values": [instruments_per_component[component]], "update_time": [today.strftime('%Y-%m-%d')]}

                        print("New station written to json")

                    else:
                        pass # match! already taken care of above          

            # safe
            with open('{}/CHILE_SINCA/metadata/{}/network_provided/CHILE_SINCA_META_{}.json'.format(dl_root,version,today.strftime('%Y%m%d')), 'w', encoding='utf-8') as f:
                f.write(json.dumps(metadata_new, indent=4, ensure_ascii=False))

            driver.close()
            errcode = 200

        except WebDriverException as e:
            print(e) 
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue

    if n_tries == n_max_tries:
        print('Failed downloading CHILE_SINCA metadata {} times in {} seconds'.format(n_tries, max_time_per_dl))
    
    
    # # create json from original metadata file: don't do this anymore, start metadata tracking with new version of GHOST {} =====================================================================================
    # json_metadata = {}
    # with open('{}/CHILE_SINCA/metadata/{}/network_provided/CHILE_SINCA_META.txt'.format(dl_root,version), 'r') as file:
    #     csv_filedata = csv.DictReader(file)

    #     for row in csv_filedata:  
    #         key = row['station_reference']
    #         update_date = today.strftime('%Y-%m-%d')
    #         for parameter in row:
    #             row[parameter] = {'values': [row[parameter]], 'update_time': [update_date]} # create inner dictionary for every parameter
    #         json_metadata[key] = row

    # with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'w', encoding='utf-8') as f:
    #     f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))
    
    
    # read newly scraped file
    with open('{}/CHILE_SINCA/metadata/{}/network_provided/CHILE_SINCA_META_{}.json'.format(dl_root,version,today.strftime('%Y%m%d')), 'r', encoding='utf-8') as f:
        json_metadata_now = json.loads(f.read())

    # read standardised file to compare!
    k=0
    m=0
    station_abolished=[]
    station_new = []
    with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'r', encoding='utf-8') as f:
        json_metadata = json.loads(f.read())

        for station in json_metadata: # loop through all the old stations
            if station in json_metadata_now.keys(): # if station is in current meta data, go on
                for parameter in json_metadata[station]:
                    if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
                        # if different value, append the standardised metadeta file
                        print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
                        json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
                        json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
                    else:
                        pass
            else:
                k=k+1
                station_abolished.append(json_metadata[station]['station_name']['values'])
        
        for station in json_metadata_now: # loop through all the new stations
            if station in json_metadata.keys(): # if station is in old meta data
                pass # comparison was done before
            else: # new station appeared!
                m=m+1
                station_new.append(json_metadata_now[station]['station_name']['values'])
                json_metadata.update({station: json_metadata_now[station]})

    print('{} stations were abolished'.format(k))
    print('Abolished stations: {}'.format(station_abolished))
    print('{} new stations '.format(m))
    print('New stations: {}'.format(station_new))

    # safe
    with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'w', encoding='utf-8') as f:
        f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))"""


def download_metadata(n_max_tries, max_time_per_dl, version, dl_root, meta_basefile, meta_history, meta_params):

    variables_ghost = {'PM10':      ['PM10', 'MP 10', "Material particulado MP 10"], 
                       'PM2.5':     ['PM2.5', 'MP 2,5', "Material particulado MP 2,5"], 
                       'PM10disc':  ['PM10disc', 'MP 10discreto', "Material particulado 10 micrometros discreto"], 
                       'SO2':       ['SO2', 'Dióxido de azufre'],
                       'NO2':       ['NO2', 'Dióxido de nitrógeno'],
                       'NOX':       ['NOX', 'Óxidos de nitrógeno'],
                       'NO':        ['NO', 'Monóxido de nitrógeno'],
                       'CO':        ['CO', 'Monóxido de carbono'],
                        'O3':       ['O3', 'Ozono', 'Ozono.-'],
                        'Pb':       ['Pb', 'Plomo'], 
                        'As':       ['As', 'Arsénico'], 
                        'Cu':       ['Cu', 'Cobre'],
                        'MPS':      ['MPS'],
                        'CH4':      ['CH4', 'Metano'], 
                        'NMHC':     ['NMHC', 'HCNM', 'Hidrocarburos no metánicos'],
                        'HC':       ['THCM', 'HCT', 'HC', 'Hidrocarburos totales']}
                        # 'V':        ['V'],
                        # 'Ni':       ['Ni'],
                        # 'Se':       ['Se'],
                        # 'TRS':      ['TRS']}
    

    baseurl = 'https://sinca.mma.gob.cl/index.php/'
    today = date.today()
    tot_number_components = 16
    
    #create json metadata base file! =============================================================================================
    """parameters = ['station_reference','station_name','region','province','commune','UTM_lon','UTM_lat','timezone','PM10', 'PM2.5', 
                       'PM10disc', 
                       'SO2',
                       'NO2',
                       'NOX',
                       'NO',
                       'CO',
                        'O3',
                        'Pb', 
                        'As', 
                        'Cu',
                        'MPS',
                        'CH4', 
                        'NMHC',
                        'HC','O3_instrument','NO_instrument','NO2_instrument','CO_instrument','CH4_instrument','SO2_instrument','NMHC_instrument','HC_instrument','PM10_instrument','PM2.5_instrument','As_instrument','Cu_instrument','Pb_instrument','comments']

    
    n_tries = 0
    errcode = 999
    
    metadata_new = {}
    while (n_tries < n_max_tries) and (errcode != 200):
        try:
            # set up driver
            options = Options()
            #prefs = {'download.default_directory' : download_location}
            #options.add_experimental_option('prefs', prefs)
            options.add_argument("--no-sandbox")
            options.add_argument("--headless")
            svc = webdriver.ChromeService(executable_path=binary_path)
            driver = webdriver.Chrome(service=svc, options=options)

            # open url
            driver.get(baseurl)
            WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.XPATH, '//a[contains(@href, "/index.php/region/index/id/")]'))) # wait till loaded

            # navigate to regions
            html = driver.page_source
            soup = BeautifulSoup(html, features="html.parser")
            regions = soup.find_all("a", href=re.compile(r"^/index.php/region/index/id/"))
            #regions = soup.find_all("a", {"href": "/index.php/region/index/id/VIII"}) # Biobio

            for region in regions:
                print("Region is "+region.getText())

                driver.get("https://sinca.mma.gob.cl/"+region.get("href"))
                time.sleep(1)

                # navigate to station and component
                html_region = driver.page_source
                soup_region = BeautifulSoup(html_region, features="html.parser")
                stations = soup_region.find_all("a", href=re.compile(r"^/index.php/estacion/index/id/"), title=None)

                for station in stations: # loop over new stations

                    var_list = [False] * tot_number_components # total number of components measured by SINCA
                    components_stations = [a.get("title") for a in station.find_parent().find_parent().find_all('a', title=True) if '|' in a.get("title")] # find components per station

                    for component_station in components_stations:
                        station_name_new = component_station.split("| ", 1)[1]
                        component = component_station.split(" |")[0]

                        for variable_ghost in variables_ghost:
                            if component in variables_ghost[variable_ghost]:
                                ind = list(variables_ghost.keys()).index(variable_ghost)
                                var_list[ind] = True # component is measured at given station

                    print(station_name_new)

                    driver.get("https://sinca.mma.gob.cl/"+station.get("href"))
                    time.sleep(1)
                    
                    # get meta info
                    html = driver.page_source
                    soup = BeautifulSoup(html, features="html.parser")
                    
                    region = soup.find("th", text="Región").find_next_sibling().getText().replace(',', '')
                    province = soup.find("th", text="Provincia").find_next_sibling().getText().replace(',', '')
                    commune = soup.find("th", text="Comuna").find_next_sibling().getText().replace(',', '')
                    UTM_coordinates = soup.find("th", text="Coordenadas UTM").find_next_sibling().getText().replace(' ', '')
                    lon = UTM_coordinates.split('E')[0]+'E'
                    lat = UTM_coordinates.split('E')[1].split("\n")[0]
                    timezone = soup.find("th", text="Huso horario").find_next_sibling().getText().replace(' ', '')

                    ins_table = soup.find('table', id="medicion")
                    if ins_table is not None: # check if there are instruments for air pollution at this station
                        instruments = ins_table.find_all("td", {"class": "helpTecnica center"})
                        instruments_per_component = {}
                    else:
                        continue

                    for instrument in instruments:
                        component_sinca = instrument.find_parent().find('a').getText()

                        for variable_ghost in variables_ghost: # map components to ghost standards
                            if component_sinca in variables_ghost[variable_ghost]:
                                component = variable_ghost

                        #print(component_sinca, 'mapped to -->', component)

                        if "No informado" in instrument.getText():
                            instruments_per_component[component] = ''
                        else:
                            instrument_name = instrument.getText().strip()
                            instrument_name = instrument_name.split("\n")[-1]
                            instruments_per_component[component] = instrument_name

                    # find station reference
                    link_vis_data = ins_table.find('a', {"alt": "visualizar serie de tiempo"})
                    if link_vis_data is None:
                        continue
                    else:
                        link_vis_data = link_vis_data.get('href')

                    station_ID = link_vis_data.split('/Cal/')[0].split('/')[-1]
                    region_ID = link_vis_data.split('/Cal/')[0].split('/')[-2]
                    station_reference = 'CL_'+region_ID+'_'+station_ID

                    print(station_reference)

                    metadata_new[station_reference] = {} # create inner dictionary
                    scraped_metadata = [station_reference, station_name_new, region, province, commune, lon, lat, timezone] + var_list
                    i=0
                    for parameter in parameters: # loop through the meta parameters
                        if ("instrument" not in parameter) and ("comments" not in parameter): # go through all that are not instruments
                            metadata_new[station_reference][parameter] = {"values": [scraped_metadata[i]], "update_time": [today.strftime('%Y-%m-%d')]}
                            i=i+1
                        elif "comments" == parameter:
                            metadata_new[station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]}
                        else: # go through the instruments
                            metadata_new[station_reference][parameter] = {"values": [''], "update_time": [today.strftime('%Y-%m-%d')]} # write in all instruments empty list
                            for component in instruments_per_component:
                                if component in parameter: # if instrument matches component, overwrite empty instrument list
                                    metadata_new[station_reference][parameter] = {"values": [instruments_per_component[component]], "update_time": [today.strftime('%Y-%m-%d')]}

            # safe
            with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'w', encoding='utf-8') as f:
                f.write(json.dumps(metadata_new, indent=4, ensure_ascii=False))

            driver.close()
            errcode = 200

        except WebDriverException as e:
            print(e) 
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue

    if n_tries == n_max_tries:
        print('Failed downloading CHILE_SINCA metadata {} times in {} seconds'.format(n_tries, max_time_per_dl))"""

    
    with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'r', encoding='utf-8') as f:
        metadata_old = json.loads(f.read())

    #station_commune_old = [str(metadata_old[station_reference]['station_name']['values'][-1]).replace(' ', '')+'_'+str(metadata_old[station_reference]['commune']['values'][-1]).replace(' ', '') for station_reference in metadata_old]
    # needed to determine if new station appeared
    
    n_tries = 0
    errcode = 999
    metadata_new = {}
    # create pandas
    metadata_new = pd.DataFrame(columns=['station_reference','station_name','region','province','commune','UTM_lon','UTM_lat','timezone','O3_instrument','NO_instrument','NO2_instrument','CO_instrument','CH4_instrument','SO2_instrument','NMHC_instrument','HC_instrument','PM10_instrument','PM2.5_instrument','As_instrument','Cu_instrument','Pb_instrument'])

    while (n_tries < n_max_tries) and (errcode != 200):
        try:
            # set up driver
            options = Options()
            #prefs = {'download.default_directory' : download_location}
            #options.add_experimental_option('prefs', prefs)
            options.add_argument("--no-sandbox")
            options.add_argument("--headless")
            svc = webdriver.ChromeService(executable_path=binary_path)
            driver = webdriver.Chrome(service=svc, options=options)

            # open url
            driver.get(baseurl)
            WebDriverWait(driver, max_time_per_dl).until(EC.presence_of_all_elements_located((By.XPATH, '//a[contains(@href, "/index.php/region/index/id/")]'))) # wait till loaded

            # navigate to regions
            html = driver.page_source
            soup = BeautifulSoup(html, features="html.parser")
            regions = soup.find_all("a", href=re.compile(r"^/index.php/region/index/id/"))
            #regions = soup.find_all("a", {"href": "/index.php/region/index/id/II"})

            for region in regions:

                print("Region is "+region.getText())

                driver.get("https://sinca.mma.gob.cl/"+region.get("href"))
                time.sleep(1)

                # navigate to station and component
                html_region = driver.page_source
                soup_region = BeautifulSoup(html_region, features="html.parser")
                stations = soup_region.find_all("a", href=re.compile(r"^/index.php/estacion/index/id/"), title=None)

                for station in stations: # loop over new stations

                    #var_list = [False] * tot_number_components # total number of components measured by SINCA
                    components_stations = [a.get("title") for a in station.find_parent().find_parent().find_all('a', title=True) if '|' in a.get("title")] # find components per station

                    for component_station in components_stations:
                        station_name_new = component_station.split("| ", 1)[1].replace(',', '').replace('"', '')
                        component = component_station.split(" |")[0]

                        # for variable_ghost in variables_ghost:
                        #     if component in variables_ghost[variable_ghost]:
                        #         ind = list(variables_ghost.keys()).index(variable_ghost)
                        #         var_list[ind] = True # component is measured at given station

                    print(station_name_new)

                    driver.get("https://sinca.mma.gob.cl/"+station.get("href"))
                    time.sleep(1)
                    
                    # get meta info
                    html = driver.page_source
                    soup = BeautifulSoup(html, features="html.parser")
                    
                    region = soup.find("th", text="Región").find_next_sibling().getText().replace(',', '').replace('"', '').replace('\n', '')
                    province = soup.find("th", text="Provincia").find_next_sibling().getText().replace(',', '').replace('"', '').replace('\n', '')
                    commune = soup.find("th", text="Comuna").find_next_sibling().getText().replace(',', '').replace('"', '').replace('\n', '')
                    UTM_coordinates = soup.find("th", text="Coordenadas UTM").find_next_sibling().getText().replace(' ', '')
                    lon = UTM_coordinates.split('E')[0]
                    lat = UTM_coordinates.split('E')[1].split("\n")[0].replace('N', '')
                    timezone = soup.find("th", text="Huso horario").find_next_sibling().getText().replace(' ', '')

                    ins_table = soup.find('table', id="medicion")
                    if ins_table is not None: # check if there are instruments for air pollution at this station
                        instruments = ins_table.find_all("td", {"class": "helpTecnica center"})
                        instruments_per_component = {}
                    else:
                        continue

                    for instrument in instruments:
                        component_sinca = instrument.find_parent().find('a').getText()

                        for variable_ghost in variables_ghost: # map components to ghost standards
                            if component_sinca in variables_ghost[variable_ghost]:
                                component = variable_ghost

                        #print(component_sinca, 'mapped to -->', component)

                        if "No informado" in instrument.getText():
                            instruments_per_component[component+'_instrument'] = ''
                        else:
                            instrument_name = instrument.getText().strip()
                            instrument_name = instrument_name.split("\n")[-1]
                            instruments_per_component[component+'_instrument'] = instrument_name

                    # find station reference
                    link_vis_data = ins_table.find('a', {"alt": "visualizar serie de tiempo"})
                    if link_vis_data is None:
                        continue
                    else:
                        link_vis_data = link_vis_data.get('href')

                    station_ID = link_vis_data.split('/Cal/')[0].split('/')[-1]
                    region_ID = link_vis_data.split('/Cal/')[0].split('/')[-2]
                    station_reference = 'CL_'+region_ID+'_'+station_ID

                    # collect all data in dict
                    scraped_metadata_non_ins = { 'station_reference': station_reference,
                                'station_name': station_name_new,
                                'region': region,
                                'province': province,
                                'commune': commune,
                                'UTM_lon': lon,
                                'UTM_lat': lat,
                                'timezone': timezone}

                    # append pandas frame
                    scraped_metadata = {**scraped_metadata_non_ins, **instruments_per_component}
                    metadata_new = metadata_new.append(scraped_metadata, ignore_index=True)
                    print(metadata_new)

            driver.close()
            errcode = 200

        except WebDriverException as e:
            print(e) 
            n_tries = n_tries+1
            print("Number of tries: {}".format(n_tries))
            continue

    if n_tries == n_max_tries:
        print('Failed downloading CHILE_SINCA metadata {} times in {} seconds'.format(n_tries, max_time_per_dl))

    # save metadata pandas yto csv
    metadata_new.to_csv('{}/CHILE_SINCA/metadata/{}/network_provided/CHILE_SINCA_META_{}.csv'.format(dl_root,version, today.strftime('%Y%m%d')),index=False)
    
    # JSON =====================================================================
    network = os.path.basename(__file__).split('_downloa')[0]
    present_meta_file_id = today.strftime('%Y%m%d')

    metadata_parsing_functions.json_parser(dl_root, network, version, meta_basefile, meta_history, meta_params, present_meta_file_id)


    # # read standardised file to compare!
    # k=0
    # m=0
    # station_abolished=[]
    # station_new = []
    # with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'r', encoding='utf-8') as f:
    #     json_metadata = json.loads(f.read())

    #     for station in json_metadata: # loop through all the old stations
    #         if station in json_metadata_now.keys(): # if station is in current meta data, go on
    #             for parameter in json_metadata[station]:
    #                 if json_metadata[station][parameter]['values'][-1] != json_metadata_now[station][parameter]['values'][0]: # compare last entry in standardised file to value in new file
    #                     # if different value, append the standardised metadeta file
    #                     print("old {} --- new {}".format(json_metadata[station][parameter]['values'][-1], json_metadata_now[station][parameter]['values'][0]))
    #                     json_metadata[station][parameter]['values'].append(json_metadata_now[station][parameter]['values'][0])
    #                     json_metadata[station][parameter]['update_time'].append(json_metadata_now[station][parameter]['update_time'][0])
    #                 else:
    #                     pass
    #         else:
    #             k=k+1
    #             station_abolished.append(json_metadata[station]['station_name']['values'])
        
    #     for station in json_metadata_now: # loop through all the new stations
    #         if station in json_metadata.keys(): # if station is in old meta data
    #             pass # comparison was done before
    #         else: # new station appeared!
    #             m=m+1
    #             station_new.append(json_metadata_now[station]['station_name']['values'])
    #             json_metadata.update({station: json_metadata_now[station]})

    # print('{} stations were abolished'.format(k))
    # print('Abolished stations: {}'.format(station_abolished))
    # print('{} new stations '.format(m))
    # print('New stations: {}'.format(station_new))

    # # safe
    # with open('{}/CHILE_SINCA/metadata/{}/processed/CHILE_SINCA_META.json'.format(dl_root,version), 'w', encoding='utf-8') as f:
    #     f.write(json.dumps(json_metadata, indent=4, ensure_ascii=False))