Commit ef91fddc authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Improved days over logging

parent 868683df
......@@ -16,8 +16,7 @@ CON_FILES = /esnas/autosubmit/con_files/
# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or
# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it
# empty
DIAGS = discretize,atmos,sfcWind,2000,0,40
# climpercent,atmos,sfcWind,1991,1992,11
DIAGS = climpercent,atmos,sfcWind,1981,2012,11
# DIAGS = OHC
# Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores
# its results at monthly frequency (obvious) and has a parameter to specify input's frequency.
......
......@@ -2,15 +2,19 @@
import os
import six
from bscearth.utils.log import Log
from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str
from bscearth.utils.config_parser import ConfigParser
from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str
from bscearth.utils.log import Log
from earthdiagnostics.frequency import Frequency, Frequencies
from earthdiagnostics.variable import VariableManager
from modelingrealm import ModelingRealm
class ConfigException(Exception):
pass
class Config(object):
"""
Class to read and manage the configuration
......@@ -103,7 +107,7 @@ class Config(object):
self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid)
self.cmor = CMORConfig(parser)
self.cmor = CMORConfig(parser, self.var_manager)
self.thredds = THREDDSConfig(parser)
self.report = ReportConfig(parser)
......@@ -149,12 +153,15 @@ class CMORConfig(object):
splitted = domain_var.split(':')
cmor_var = self.var_manager.get_variable(splitted[1], silent=True)
if not cmor_var:
Log.warning('Variable {0} not recognized. It will not be cmorized', domain_var)
continue
if ModelingRealm(splitted[0]) != cmor_var.domain:
Log.warning('Domain {0} for variable {1} is not correct: is {2}', splitted[0], cmor_var.short_name,
cmor_var.domain)
return
continue
self._variable_list.append('{0.domain}:{0.short_name}'.format(cmor_var))
if len(self._variable_list) == 0:
raise ConfigException('Variable list value is specified, but no variables were found')
else:
self._variable_list = None
......@@ -223,7 +230,7 @@ class CMORConfig(object):
return self._var_daily
elif frequency == Frequencies.monthly:
return self._var_monthly
raise Exception('Frequency not recognized: {0}'.format(frequency))
raise ValueError('Frequency not recognized: {0}'.format(frequency))
def get_levels(self, frequency, variable):
return self.get_variables(frequency)[variable]
......
# coding=utf-8
from bscearth.utils.date import parse_date, add_months
from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile
from earthdiagnostics.diagnostic import *
from earthdiagnostics.frequency import Frequencies
import iris
import iris.exceptions
import iris.coord_categorisation
from iris.time import PartialDateTime
import iris.analysis
import iris.coord_categorisation
import iris.coords
import iris.exceptions
import numpy as np
from bscearth.utils.date import parse_date, add_months
from bscearth.utils.log import Log
from iris.time import PartialDateTime
from earthdiagnostics.diagnostic import *
from earthdiagnostics.frequency import Frequencies
from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile
from earthdiagnostics.utils import Utils, TempFile
......@@ -28,15 +28,14 @@ class DaysOverPercentile(Diagnostic):
alias = 'daysover'
"Diagnostic alias for the configuration file"
def __init__(self, data_manager, domain, variable, start_year, end_year, year_to_compute, forecast_month):
def __init__(self, data_manager, domain, variable, start_year, end_year, startdate, forecast_month):
Diagnostic.__init__(self, data_manager)
self.variable = variable
self.domain = domain
self.start_year = start_year
self.end_year = end_year
self.year_to_compute = year_to_compute
self.forecast_month = forecast_month
self.startdate = '{0}{1:02}01'.format(self.year_to_compute, self.forecast_month)
self.startdate = startdate
def __eq__(self, other):
return self.startdate == other.startdate and self.domain == other.domain and \
......@@ -66,13 +65,11 @@ class DaysOverPercentile(Diagnostic):
options = cls.process_options(options, options_available)
job_list = list()
year = options['start_year']
while year <= options['end_year']:
for startdate in diags.config.experiment.startdates:
for forecast_month in options['forecast_month']:
job_list.append(DaysOverPercentile(diags.data_manager, options['domain'], options['variable'],
options['start_year'], options['end_year'],
year, forecast_month))
year += 1
startdate, forecast_month))
return job_list
def request_data(self):
......@@ -150,6 +147,7 @@ class DaysOverPercentile(Diagnostic):
'climatology'.format(self)
for leadtime in leadtimes.keys():
Log.debug('Computing startdate {0} leadtime {1}', self.startdate, leadtime)
leadtime_slice = var.extract(iris.Constraint(leadtime=leadtime))
if len(percentiles.coords('leadtime')) > 0:
percentiles_leadtime = percentiles.extract(iris.Constraint(leadtime=leadtime))
......@@ -175,6 +173,7 @@ class DaysOverPercentile(Diagnostic):
time_coord, var_days_below, long_name_days_below)
results_below[percentile].append(result)
Log.debug('Saving percentiles startdate {0}', self.startdate)
for perc in ClimatologicalPercentile.Percentiles:
iris.FUTURE.netcdf_no_unlimited = True
temp = TempFile.get()
......@@ -189,6 +188,11 @@ class DaysOverPercentile(Diagnostic):
must_exist=False, rename_dimension=True)
self.days_below_file[perc].set_local_file(temp, rename_var='daysbelow')
del self.days_over_file
del self.days_below_file
del self.lat_coord
del self.lon_coord
def create_results_cube(self, days_over, percentile, realization_coord, time_coord,
var_name, long_name):
result = iris.cube.Cube(days_over.astype(np.float32), var_name=var_name, long_name=long_name, units=1.0)
......
# coding=utf-8
from unittest import TestCase
from earthdiagnostics.config import CMORConfig
from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig
from earthdiagnostics.frequency import Frequencies
from earthdiagnostics.modelingrealm import ModelingRealms
......@@ -43,6 +44,9 @@ class ParserMock(object):
def get_bool_option(self, section, var, default):
return self.get_value(section, var, default)
def get_path_option(self, section, var, default):
return self.get_value(section, var, default)
def get_int_option(self, section, var, default):
return self.get_value(section, var, default)
......@@ -85,7 +89,32 @@ class TestCMORConfig(TestCase):
def test_cmorize(self):
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertTrue(config.cmorize(VariableMock))
self.assertTrue(config.cmorize(VariableMock()))
self.assertTrue(config.cmorize(None))
def test_cmorize_list(self):
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:thetao ocean:tos')
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertTrue(config.cmorize(VariableMock()))
thetao_mock = VariableMock()
thetao_mock.domain = ModelingRealms.ocean
thetao_mock.short_name = 'thetao'
self.assertTrue(config.cmorize(thetao_mock))
def test_bad_list(self):
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', '#ocean:tos')
with self.assertRaises(ConfigException):
CMORConfig(self.mock_parser, self.var_manager)
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'atmos:tos')
with self.assertRaises(ConfigException):
CMORConfig(self.mock_parser, self.var_manager)
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:bad')
with self.assertRaises(ConfigException):
CMORConfig(self.mock_parser, self.var_manager)
def test_not_cmorize(self):
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos')
......@@ -93,6 +122,8 @@ class TestCMORConfig(TestCase):
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertTrue(config.cmorize(VariableMock()))
self.assertFalse(config.cmorize(None))
tas_mock = VariableMock()
tas_mock.domain = ModelingRealms.atmos
tas_mock.short_name = 'tas'
......@@ -103,6 +134,21 @@ class TestCMORConfig(TestCase):
thetao_mock.short_name = 'thetao'
self.assertFalse(config.cmorize(thetao_mock))
def test_comment(self):
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos #ocean:thetao ')
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertTrue(config.cmorize(VariableMock()))
thetao_mock = VariableMock()
thetao_mock.domain = ModelingRealms.ocean
thetao_mock.short_name = 'thetao'
self.assertFalse(config.cmorize(thetao_mock))
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', '#ocean:tos ocean:thetao ')
with self.assertRaises(ConfigException):
CMORConfig(self.mock_parser, self.var_manager)
def test_cmorization_chunk(self):
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertTrue(config.chunk_cmorization_requested(1))
......@@ -115,3 +161,110 @@ class TestCMORConfig(TestCase):
self.assertFalse(config.chunk_cmorization_requested(1))
self.assertFalse(config.chunk_cmorization_requested(4))
self.assertFalse(config.chunk_cmorization_requested(6))
def test_any_required(self):
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertTrue(config.any_required(['tos']))
self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos ocean:thetao')
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertTrue(config.any_required(['tos', 'thetao', 'tas']))
self.assertTrue(config.any_required(['tos', 'tas']))
self.assertTrue(config.any_required(['thetao']))
self.assertFalse(config.any_required(['tas']))
def test_hourly_vars(self):
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertEquals(config.get_variables(Frequencies.six_hourly), {})
self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5')
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertEquals(config.get_variables(Frequencies.six_hourly), {128: None,
129: '1',
130: '1,2',
131: '1,2,3,4,5,6,7,8,9',
132: '0,5'})
self.assertEquals(config.get_levels(Frequencies.six_hourly, 128), None)
self.assertEquals(config.get_levels(Frequencies.six_hourly, 129), '1')
self.assertEquals(config.get_levels(Frequencies.six_hourly, 130), '1,2')
self.assertEquals(config.get_levels(Frequencies.six_hourly, 131), '1,2,3,4,5,6,7,8,9',)
self.assertEquals(config.get_levels(Frequencies.six_hourly, 132), '0,5')
def test_daily_vars(self):
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertEquals(config.get_variables(Frequencies.daily), {})
self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5')
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertEquals(config.get_variables(Frequencies.daily), {128: None,
129: '1',
130: '1,2',
131: '1,2,3,4,5,6,7,8,9',
132: '0,5'})
self.assertEquals(config.get_levels(Frequencies.daily, 128), None)
self.assertEquals(config.get_levels(Frequencies.daily, 129), '1')
self.assertEquals(config.get_levels(Frequencies.daily, 130), '1,2')
self.assertEquals(config.get_levels(Frequencies.daily, 131), '1,2,3,4,5,6,7,8,9',)
self.assertEquals(config.get_levels(Frequencies.daily, 132), '0,5')
def test_monthly_vars(self):
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertEquals(config.get_variables(Frequencies.monthly), {})
self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5')
config = CMORConfig(self.mock_parser, self.var_manager)
self.assertEquals(config.get_variables(Frequencies.monthly), {128: None,
129: '1',
130: '1,2',
131: '1,2,3,4,5,6,7,8,9',
132: '0,5'})
self.assertEquals(config.get_levels(Frequencies.monthly, 128), None)
self.assertEquals(config.get_levels(Frequencies.monthly, 129), '1')
self.assertEquals(config.get_levels(Frequencies.monthly, 130), '1,2')
self.assertEquals(config.get_levels(Frequencies.monthly, 131), '1,2,3,4,5,6,7,8,9',)
self.assertEquals(config.get_levels(Frequencies.monthly, 132), '0,5')
def test_bad_frequency_vars(self):
config = CMORConfig(self.mock_parser, self.var_manager)
with self.assertRaises(ValueError):
self.assertEquals(config.get_variables(Frequencies.climatology), {})
class TestTHREDDSConfig(TestCase):
def setUp(self):
self.mock_parser = ParserMock()
def test_basic_config(self):
config = THREDDSConfig(self.mock_parser)
self.assertEquals(config.server_url, '')
def test_url(self):
self.mock_parser.add_value('THREDDS', 'SERVER_URL', 'test_url')
config = THREDDSConfig(self.mock_parser)
self.assertEquals(config.server_url, 'test_url')
class TestReportConfig(TestCase):
def setUp(self):
self.mock_parser = ParserMock()
def test_basic_config(self):
config = ReportConfig(self.mock_parser)
self.assertEquals(config.path, '')
self.assertEquals(config.maximum_priority, 10)
def test_path(self):
self.mock_parser.add_value('REPORT', 'PATH', 'new_path')
config = ReportConfig(self.mock_parser)
self.assertEquals(config.path, 'new_path')
def test_priority(self):
self.mock_parser.add_value('REPORT', 'MAXIMUM_PRIORITY', 3)
config = ReportConfig(self.mock_parser)
self.assertEquals(config.maximum_priority, 3)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment