#!/usr/bin/env python
#
# This script checks every single option available in the CDS download form
# for all existing datasets within, except for those that does not have any
# kind of download form but are available through the CDS (i.e. era5-complete)
#
# AUTHOR: Barcelona Supercomputing Center-Centro Nacional de Supercomputacion
#         Earth Sciences Department
#         I. CERNICHARO: ivan.cernicharo@bsc.es
#

import requests
import json
import datetime

def main(response1):
    """This function returns a dictionary of the different datasets stored in the CDS
    ordered by CDS product Type available
            
            Climate Projection
            Reanalysis
            Satellite Observations
            Seasonal Forecasts
            Sectoral Climate Indices

    It is defined by default just for CDS configuration files
    """
    dict_datasets = {}

    for i,element in enumerate(response1):
        final_url = cds_url + "/" + element
        print("\n---------------------------------------------------------------------\n")
        print("Requesting:\n\t>Dataset no.: %s \n\t>URL: %s \n" %(i,final_url))
        ap = requests.get(final_url).json()

        if "form" in ap and ap["form"] != None:
            dict_var = {}
            p_it = [item.split(":") for item in ap["structured_data"]["keywords"] if "Product type" in item]
            p_t = p_it[0][1].strip()
            print("\t\t"+">Dataset Name: "+ap["name"]+"\n\t\t"+">CDS product type: "+p_t)
            # Updating dictionary with toolbox complaince field
            dict_var.update({"Toolbox compatibility":ap["toolbox_compat"]})

            for item in ap["form"]:

                if "dict" in str(type(item)):

                    for e in item.keys():

                        if e == "details":

                            for op in item[e].keys():

                                if "list" in str(type(item[e][op])):

                                    if op == "values": 
                                        dict_var.update({item["label"]:item[e][op]})

                                    if op == "groups" and "groups" in item[e].keys():
                                        more_vars = {}

                                        for aj in item[e][op]:
                                            more_vars.update({aj["label"]:aj["values"]})

                                        dict_var.update({"group":more_vars})

            d = {ap["name"]:dict_var}

            if p_t not in dict_datasets.keys():
                dict_datasets[p_t] = d

            else:
                dict_datasets[p_t].update(d)

        # Next two lines corresponds for those datasets that are not available directly 
        # through the CDS website but can be downloaded with the cdsapi 
        # i.e.: reanalysis-era5-complete dataset
        #else:
        #    dict_datasets.update({ap["name"]:ap})
    
    return dict_datasets
        
def save_to_json(dictionary_name):
    date_var = datetime.datetime.now().date().isoformat().replace("-","")
    json_name = "cds_metadata_" + date_var + ".json"
    print("\n\n\t **** Saving as %s **** " %json_name)

    with open(json_name,"w") as fp:
        json.dump(dictionary_name,fp,indent=4,sort_keys=True)
        
    fp.close()
    print("\n\t**** FINISHED ****\n")

if __name__ == "__main__":
    cds_url = 'https://cds.climate.copernicus.eu/api/v2.ui/resources'
    response1 = requests.get(cds_url).json()
    print("\nRunning CDS API METADATA script\n")
    save_to_json(main(response1))
