#!/usr/bin/env python
#
# Data Quality Checker also known as DQC
#
# Read the README.md for more detailed information
#
#       Barcelona Supercomputing Center - Centro Nacional de Supercomputacion
#       Earth Sciences Department - Computational Earth Sciences
#     
#     Authors: 
#       Joan Sala Calero (joan.sala@bsc.es)
#       Daniel Trujillo Viedma (daniel.trujillo@bsc.es)
#

import json
import sys
import dateutil.parser
from _functools import *
from textwrap import wrap
import _functools

if len(sys.argv) != 4:
    print('usage: python json2txt_cdm.py <input JSON> <section1.txt> <section2.txt>')
    sys.exit()

with open(sys.argv[1], 'r') as f:
    input_read = f.read()

json_parsed = json.loads(input_read)

##########
## Extract preliminary data
##########

# Any
tests_data = json_parsed["results"]
# Any -> [String]
retrieve_cdm_data = lambda result: result["output"]["cdm_compliance"][result["output"]["cdm_compliance"].find("\n")+1:].strip().split("\n") if("cdm_compliance" in result["output"]) else ["<The integration test failed to execute. No output produced>"]
# List[List[String]]
cdm_data = list(map(retrieve_cdm_data,tests_data))
# String
tool_name = json_parsed["info"]["tool_name"]
# String
test_timestamp = dateutil.parser.isoparse(json_parsed["info"]["check_run_time"]).strftime('%Y-%m-%d')

##########
## First, build the summary information
##########

# List[String] -> Bool
is_test_failed = lambda result: reduce(lambda x,y: x|y, map(lambda x: x.startswith("ERROR") | (x == "<The integration test failed to execute. No output produced>"), result))
# Int
number_of_tests_failed = len(list(filter(is_test_failed, cdm_data)))
# Int
percentage_failed = round(100 * number_of_tests_failed / len(cdm_data), 1)

# List[String] -> Bool
is_test_warning = lambda result: reduce(lambda x,y: x&y, map(lambda x: x.startswith("WARN"), result))
# Int
number_of_tests_warning = len(list(filter(is_test_warning, cdm_data)))
# Int
percentage_warning = round(100 * number_of_tests_warning / len(cdm_data), 1)

# Int
number_of_test_passed = len(cdm_data) - number_of_tests_warning - number_of_tests_failed
# Int
percentage_passed = round(100 - percentage_warning - percentage_failed, 2)

# (All of the following are) String
results = "res = {}% passed, {}% passed with warnings, {}% not passed".format(percentage_passed, percentage_warning, percentage_failed)
message = "msg = The tool " + tool_name + (" has " if (percentage_failed == 0) else " hasn't ") + "passed all the CDM compliance checks"
number_of_tests = "number of tests = {}".format(len(tests_data))
generation_time = "time = generated on " + test_timestamp
introduction_paragraph = "This check verifies whether the output of the tool complies with the metadata standard defined for the Toolbox Common Data Model (CDM). Only a representative selection of different CDS datasets and variables are checked. When compliant, it implies that the output of this tool can be further processed by other tools. The test checks the definitions of the metadata keys stored in the tool's output and gives the result in binary form.\n \nThe checks were performed using a Python-based software package, which runs on the ECMWF virtual machines dedicated to the evaluation and quality control. The code is available upon request at copernicus-support@ecmwf.int.\n \nThe summary of the result is given below, the log details are available in the following pages.\n________________________________________________________________________________"

# Any -> String
get_dataset_name = lambda result: result["input"][0]
# List[String]
dataset_names = map(get_dataset_name, tests_data)
# String -> String
get_dataset_type = lambda dataset_name: dataset_name[0:dataset_name.find("-")]
# List[String]
dataset_types = list(set(map(get_dataset_type, dataset_names)))

extended_dataset_type_names = {"projections": "climate projections", "seasonal": "seasonal forecasts", "satellite": "satellite observations"}
def extend_dataset_type_name(type_name: str) -> str:
    return extended_dataset_type_names[type_name] if (type_name in extended_dataset_type_names) else type_name
# String
dataset_types_message = "dataset types considered = " + reduce(lambda msg,new_type: "{}, {}".format(msg, extend_dataset_type_name(new_type)), dataset_types[1:], extend_dataset_type_name(dataset_types[0]))


##########
## Now extract relevant data for the report  
##########

# Any -> String
def dictToString(obj):
    return reduce(lambda a,b : "{}\n  {}: {}".format(a, b[0],b[1]), obj.items(), "")

# List[String] -> String
def test_outcome(cdm_check_output:[str]):
    return "With warnings" if(is_test_warning(cdm_check_output)) else ("False" if(is_test_failed(cdm_check_output)) else "True")

# List[String] -> String
def format_CDM_check_results(cdm_check_output:[str]):
    return reduce(lambda a,b : "{}\n  {}".format(a, b), cdm_check_output, "")

# Any -> String
extract_data = lambda result: "**  CDM check number {}  **\nPassed: {}\nDataset: {}\nParameters applied: {}\nResults:{}\n \n \n \n".format(result[1], test_outcome(result[0][1]), result[0][0]["input"][0], dictToString(result[0][0]["input"][1]), format_CDM_check_results(result[0][1]))

# List[T] -> List[(T, Int)]
def zipWithIndex(array):
    return list(zip(array, range(0, len(array))))

# List[(Any, 
combined_data = list(zip(tests_data, cdm_data))
combined_indexed_data = zipWithIndex(combined_data) 

relevant_data = list(map(extract_data, combined_indexed_data))

#print(tests_data)

#sys.exit()

# The final step is to format the relevant data
format_log  = lambda data: "{}\n\n".format(data)
     
relevant_data_formatted = "{}\n \n[compliance]\n \n[compliance_result]\n{}\n{}\n \n[metadata]\n{}\n{}\ntest = cdm_compliance\n{}".format(introduction_paragraph, results, message, number_of_tests, dataset_types_message, generation_time)
relevant_log_formatted  = relevant_data

# And write the files
with open(sys.argv[2], 'w') as f:
    list(map(lambda s: f.write(s), relevant_data_formatted))

with open(sys.argv[3], 'w') as f:
    list(map(lambda s: f.write(s), relevant_log_formatted))




