From c12b7278a0dd348c4fb4376534792d0eab9335b4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 2 Dec 2016 14:36:56 +0100 Subject: [PATCH 1/5] Created option classes for better diagnostic generation --- .gitignore | 1 + earthdiagnostics/cdftools.py | 7 +- earthdiagnostics/diagnostic.py | 55 ++++++++++ earthdiagnostics/general/monthlymean.py | 2 +- test/unit/__init__.py | 2 +- test/unit/test_diagnostic.py | 137 +++++++++++++++++++++++- 6 files changed, 197 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index c583596b..cafa800b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ doc/build/* *.err *.out +*.nc .coverage htmlcov \ No newline at end of file diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index 06331b1d..845aae65 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -71,18 +71,17 @@ class CDFTools(object): if not os.path.exists(element): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) - @staticmethod - def is_exe(fpath): + def is_exe(self, fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) def _check_command_existence(self, command): if self.path: - if CDFTools.is_exe(os.path.join(self.path, command)): + if self.is_exe(os.path.join(self.path, command)): return else: for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, command) - if CDFTools.is_exe(exe_file): + if self.is_exe(exe_file): return raise ValueError('Error executing {0}\n Command does not exist in {1}', command, self.path) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 5eac3424..c0ddea5f 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -103,9 +103,64 @@ class Diagnostic(object): """ raise NotImplementedError("Class must override generate_jobs class method") + @staticmethod + def process_options(options, options_available): + processed = dict() + options = options[1:] + for x in range(len(options_available)): + option_definition = options_available[x] + if len(options) <= x: + option_value = '' + else: + option_value = options[x] + processed[option_definition.name] = option_definition.parse(option_value) + return processed + def __str__(self): """ Must be implemented by derived classes :return: """ return 'Developer must override base class __str__ method' + + +class DiagnosticOption(object): + + def __init__(self, name, default_value=None): + self.name = name + self.default_value = default_value + + def parse(self, option_value): + option_value = self.check_default(option_value) + return option_value + + def check_default(self, option_value): + if option_value == '': + if self.default_value is None: + raise DiagnosticOptionError('Option {0} is not optional') + else: + return self.default_value + return option_value + + +class DiagnosticFloatOption(DiagnosticOption): + def parse(self, option_value): + return float(self.check_default(option_value)) + + +class DiagnosticIntOption(DiagnosticOption): + def parse(self, option_value): + return int(self.check_default(option_value)) + + +class DiagnosticBoolOption(DiagnosticOption): + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, bool): + return option_value + else: + return option_value.lower() in ('true', 't', 'yes') + + +class DiagnosticOptionError(Exception): + pass diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 69983f37..38224e85 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -79,7 +79,7 @@ class MonthlyMean(Diagnostic): if num_options >= 4: grid = options[4] else: - grid = None + grid = '' job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 7a15d906..c611a13f 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -3,7 +3,7 @@ from test_data_manager import TestConversion from test.unit.test_variable import TestVariable from test_constants import TestBasin, TestBasins from test_box import TestBox -from test_diagnostic import TestDiagnostic +from test_diagnostic import * from test_cdftools import TestCDFTools from test_utils import TestTempFile, TestUtils from test_psi import TestPsi diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index 117e8e22..42bf9f1f 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -1,5 +1,6 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticOptionError, DiagnosticFloatOption, \ + DiagnosticIntOption, DiagnosticBoolOption from unittest import TestCase @@ -46,3 +47,137 @@ class TestDiagnostic(TestCase): def test_repr(self): self.assertEquals(self.diagnostic.__repr__(), str(self.diagnostic)) + def test_empty_process_options(self): + self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) + + # def test_empty_process_options(self): + # self.assertEqual(len(Diagnostic.process_options(('diag_name', ), tuple())), 0) + + +class TestDiagnosticOption(TestCase): + + def test_good_default_value(self): + diag = DiagnosticOption('option', 'default') + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticOption('option') + self.assertEqual('value', diag.parse('value')) + + +class TestDiagnosticFloatOption(TestCase): + def test_float_default_value(self): + diag = DiagnosticFloatOption('option', 3.0) + self.assertEqual(3.0, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticFloatOption('option', '3') + self.assertEqual(3.0, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticFloatOption('option', 'default') + with self.assertRaises(ValueError): + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticFloatOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticFloatOption('option') + self.assertEqual(3.25, diag.parse('3.25')) + + +class TestDiagnosticFloatOption(TestCase): + def test_float_default_value(self): + diag = DiagnosticFloatOption('option', 3.0) + self.assertEqual(3.0, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticFloatOption('option', '3') + self.assertEqual(3.0, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticFloatOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticFloatOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticFloatOption('option') + self.assertEqual(3.25, diag.parse('3.25')) + + +class TestDiagnosticIntOption(TestCase): + def test_float_default_value(self): + diag = DiagnosticIntOption('option', 3) + self.assertEqual(3, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticIntOption('option', '3') + self.assertEqual(3, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticIntOption('option') + self.assertEqual(3, diag.parse('3')) + + def test_parse_bad_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + + +class TestDiagnosticBoolOption(TestCase): + def test_bool_default_value(self): + diag = DiagnosticBoolOption('option', True) + self.assertEqual(True, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticBoolOption('option', 'False') + self.assertEqual(False, diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticBoolOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_True(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_true(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_t(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('t')) + + def test_parse_yes(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('YES')) + + def test_parse_bad_value(self): + diag = DiagnosticBoolOption('option') + self.assertFalse(diag.parse('3.5')) + -- GitLab From dabdc078808d4d165b0847d4791e3112134f7ab0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 2 Dec 2016 15:15:41 +0100 Subject: [PATCH 2/5] Changed option parsing for all general diagnostics --- earthdiagnostics/diagnostic.py | 14 ++++++-- earthdiagnostics/general/attribute.py | 27 +++++++-------- earthdiagnostics/general/monthlymean.py | 26 +++++---------- earthdiagnostics/general/relink.py | 20 ++++------- earthdiagnostics/general/rewrite.py | 20 ++++------- earthdiagnostics/general/scale.py | 42 +++++++++-------------- earthdiagnostics/variable.py | 6 ++++ test/unit/test_diagnostic.py | 44 ++++++++++++++++++------- 8 files changed, 98 insertions(+), 101 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index c0ddea5f..7d946a67 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.variable import VarType +from earthdiagnostics.variable import VarType, Domain class Diagnostic(object): @@ -137,7 +137,7 @@ class DiagnosticOption(object): def check_default(self, option_value): if option_value == '': if self.default_value is None: - raise DiagnosticOptionError('Option {0} is not optional') + raise DiagnosticOptionError('Option {0} is not optional'.format(self.name)) else: return self.default_value return option_value @@ -153,6 +153,16 @@ class DiagnosticIntOption(DiagnosticOption): return int(self.check_default(option_value)) +class DiagnosticDomainOption(DiagnosticOption): + def parse(self, option_value): + return Domain.parse(self.check_default(option_value)) + + +class DiagnosticComplexStrOption(DiagnosticOption): + def parse(self, option_value): + return self.check_default(option_value).replace('&;', ',').replace('&.', ' ') + + class DiagnosticBoolOption(DiagnosticOption): def parse(self, option_value): option_value = self.check_default(option_value) diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 59edf3c6..97756149 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils from earthdiagnostics.variable import Domain @@ -63,23 +63,18 @@ class Attribute(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the variable, domain, attributte name and value to write') - if num_options > 5: - raise Exception('You must between 4 and 5 parameters for the rewrite diagnostic') - variable = options[1] - domain = Domain(options[2]) - name = options[3] - value = options[4] - value = value.replace('&;', ',').replace('&.', ' ') - if num_options >= 5: - grid = options[5] - else: - grid = None + + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('name'), + DiagnosticComplexStrOption('value'), + DiagnosticOption('grid')) + options = Diagnostic.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Attribute(diags.data_manager, startdate, member, chunk, domain, variable, grid, name, value)) + job_list.append(Attribute(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'], options['grid'], + options['value'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 38224e85..286b2546 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -1,7 +1,7 @@ # coding=utf-8 import os -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domain @@ -65,26 +65,16 @@ class MonthlyMean(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to average monthly') - if num_options > 4: - raise Exception('You must specify between 2 and 4 parameters for the monthly mean diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - frequency = options[3] - else: - frequency = 'day' - if num_options >= 4: - grid = options[4] - else: - grid = '' + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('frequency', 'day'), + DiagnosticOption('grid', '')) + options = Diagnostic.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, domain, variable, - frequency, grid)) + job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 3009f20e..2c2cdabd 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption from earthdiagnostics.variable import Domain @@ -59,20 +59,14 @@ class Relink(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to link') - if num_options > 3: - raise Exception('You must between 2 and 3 parameters for the relink diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - move_old = bool(options[3].lower()) - else: - move_old = True + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticBoolOption('move_old', True)) + options = Diagnostic.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Relink(diags.data_manager, startdate, member, chunk, domain, variable, move_old)) + job_list.append(Relink(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['move_old'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index c7acd744..8751e42c 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.variable import Domain @@ -58,20 +58,14 @@ class Rewrite(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to rewrite') - if num_options > 3: - raise Exception('You must between 2 and 3 parameters for the rewrite diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - grid = options[3] - else: - grid = None + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('grid', '')) + options = Diagnostic.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, domain, variable, grid)) + job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index d41797b5..c4c8769c 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,8 +1,9 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticFloatOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils from earthdiagnostics.variable import Domain import numpy as np +import math class Scale(Diagnostic): @@ -74,32 +75,19 @@ class Scale(Diagnostic): raise Exception('You must specify the acale and offset values and the variable and domain to scale') if num_options > 5: raise Exception('You must between 4 and 5 parameters for the rewrite diagnostic') - value = float(options[1]) - offset = float(options[2]) - variable = options[3] - domain = Domain(options[4]) - if num_options >= 5: - grid = options[5] - else: - grid = None - if num_options >= 6: - if options[6].lower() == 'none': - min_limit = None - else: - min_limit = float(options[6]) - else: - min_limit = None - if num_options >= 7: - if options[7].lower() == 'none': - max_limit = None - else: - max_limit = float(options[7]) - else: - max_limit = None + options_available = (DiagnosticFloatOption('value'), + DiagnosticFloatOption('offset'), + DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('grid'), + DiagnosticFloatOption('min_limit', float('nan')), + DiagnosticFloatOption('max_limit', float('nan'))) + options = Diagnostic.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Scale(diags.data_manager, startdate, member, chunk, value, offset, domain, variable, grid, - min_limit, max_limit)) + job_list.append(Scale(diags.data_manager, startdate, member, chunk, + options['value'], options['offset'], options['domain'], options['variable'], + options['grid'], options['min_limit'], options['max_limit'])) return job_list def compute(self): @@ -119,9 +107,9 @@ class Scale(Diagnostic): grid=self.grid) def _check_limits(self): - if self.min_limit is not None and np.amin(self.original_values) < self.min_limit: + if not math.isnan(self.min_limit) and np.amin(self.original_values) < self.min_limit: return False - if self.max_limit is not None and np.amax(self.original_values) > self.max_limit: + if not math.isnan(self.max_limit) is not None and np.amax(self.original_values) > self.max_limit: return False return True diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index faf7c74a..603087f2 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -66,6 +66,12 @@ class Variable(object): class Domain(object): + @staticmethod + def parse(domain_name): + if isinstance(domain_name, Domain): + return domain_name + return Domain(domain_name) + def __init__(self, domain_name): domain_name = domain_name.lower() if domain_name == 'seaice': diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index 42bf9f1f..b064fbca 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -1,8 +1,9 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticOptionError, DiagnosticFloatOption, \ - DiagnosticIntOption, DiagnosticBoolOption +from earthdiagnostics.diagnostic import * from unittest import TestCase +from earthdiagnostics.variable import Domains + class TestDiagnostic(TestCase): @@ -94,28 +95,28 @@ class TestDiagnosticFloatOption(TestCase): self.assertEqual(3.25, diag.parse('3.25')) -class TestDiagnosticFloatOption(TestCase): - def test_float_default_value(self): - diag = DiagnosticFloatOption('option', 3.0) - self.assertEqual(3.0, diag.parse('')) +class TestDiagnosticDomainOption(TestCase): + def test_domain_default_value(self): + diag = DiagnosticDomainOption('option', Domains.ocean) + self.assertEqual(Domains.ocean, diag.parse('')) def test_str_default_value(self): - diag = DiagnosticFloatOption('option', '3') - self.assertEqual(3.0, diag.parse('')) + diag = DiagnosticDomainOption('option', 'atmos') + self.assertEqual(Domains.atmos, diag.parse('')) def test_bad_default_value(self): - diag = DiagnosticFloatOption('option', 'default') + diag = DiagnosticDomainOption('option', 'default') with self.assertRaises(ValueError): diag.parse('') def test_no_default_value(self): - diag = DiagnosticFloatOption('option') + diag = DiagnosticDomainOption('option') with self.assertRaises(DiagnosticOptionError): diag.parse('') def test_parse_value(self): - diag = DiagnosticFloatOption('option') - self.assertEqual(3.25, diag.parse('3.25')) + diag = DiagnosticDomainOption('option') + self.assertEqual(Domains.seaIce, diag.parse('seaice')) class TestDiagnosticIntOption(TestCase): @@ -181,3 +182,22 @@ class TestDiagnosticBoolOption(TestCase): diag = DiagnosticBoolOption('option') self.assertFalse(diag.parse('3.5')) + +class TestDiagnosticComplexStrOption(TestCase): + def test_complex_default_value(self): + diag = DiagnosticComplexStrOption('option', 'default&.str&;&.working') + self.assertEqual('default str, working', diag.parse('')) + + def test_simple_default_value(self): + diag = DiagnosticComplexStrOption('default str, working', 'default str, working') + self.assertEqual('default str, working', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticComplexStrOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticComplexStrOption('option') + self.assertEqual('complex string, for testing', diag.parse('complex&.string&;&.for&.testing')) + -- GitLab From e41852d77dbfb31e4b99b8b2713a4c44938cf1c0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 2 Dec 2016 15:59:08 +0100 Subject: [PATCH 3/5] Updated statistics diagnostics --- earthdiagnostics/diagnostic.py | 21 ++++++- .../statistics/climatologicalpercentile.py | 24 +++----- .../statistics/monthlypercentile.py | 20 ++----- test/unit/test_diagnostic.py | 58 ++++++++++++++++++- 4 files changed, 92 insertions(+), 31 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 7d946a67..43b2ce1a 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -149,8 +149,27 @@ class DiagnosticFloatOption(DiagnosticOption): class DiagnosticIntOption(DiagnosticOption): + + def __init__(self, name, default_value=None, min_limit=None, max_limit=None): + super(DiagnosticIntOption, self).__init__(name, default_value) + self.min_limit = min_limit + self.max_limit = max_limit + def parse(self, option_value): - return int(self.check_default(option_value)) + value = int(self.check_default(option_value)) + if self.min_limit is not None and value < self.min_limit: + raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) + if self.max_limit is not None and value > self.max_limit: + raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) + return value + + +class DiagnosticListIntOption(DiagnosticOption): + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, tuple) or isinstance(option_value, list): + return option_value + return [int(i) for i in option_value.split('-')] class DiagnosticDomainOption(DiagnosticOption): diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 5df47419..b51eb453 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -1,7 +1,8 @@ # coding=utf-8 from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption, \ + DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domain, Variable, VarType import numpy as np @@ -63,22 +64,15 @@ class ClimatologicalPercentile(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable (and its domain) and the leadtimes you want to compute ' - 'the percentiles on') - if num_options > 4: - raise Exception('You must specify between three and 4 parameters for the climatological percentiles') - domain = Domain(options[1]) - variable = options[2] - leadtimes = [int(i) for i in options[3].split('-')] - if num_options > 3: - num_bins = int(options[4]) - else: - num_bins = 2000 + options_available = (DiagnosticOption('domain'), + DiagnosticDomainOption('variable'), + DiagnosticListIntOption('leadtimes'), + DiagnosticIntOption('bins', 2000)) + options = Diagnostic.process_options(options, options_available) job_list = list() - job_list.append(ClimatologicalPercentile(diags.data_manager, domain, variable, leadtimes, num_bins, + job_list.append(ClimatologicalPercentile(diags.data_manager, options['domain'], options['variable'], + options['leadtimes'], options['bins'], diags.config.experiment)) return job_list diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index d746911e..a5fbe167 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -3,7 +3,7 @@ import shutil from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domain, VarType from calendar import monthrange @@ -58,23 +58,15 @@ class MonthlyPercentile(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable (and its domain) to average vertically and ' - 'the percentil you want') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - - domain = Domain(options[1]) - variable = options[2] - percentile = int(options[3]) - if percentile < 0 or percentile > 100: - raise Exception('Percentile value must be in the interval [0,100]') + options_available = (DiagnosticOption('domain'), + DiagnosticDomainOption('variable'), + DiagnosticIntOption('percentile', None, 0, 100)) + options = Diagnostic.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(MonthlyPercentile(diags.data_manager, startdate, member, chunk, - variable, domain, percentile)) + options['variable'], options['domain'], options['percentile'])) return job_list def compute(self): diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index b064fbca..f02c13e2 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -120,7 +120,7 @@ class TestDiagnosticDomainOption(TestCase): class TestDiagnosticIntOption(TestCase): - def test_float_default_value(self): + def test_int_default_value(self): diag = DiagnosticIntOption('option', 3) self.assertEqual(3, diag.parse('')) @@ -147,6 +147,24 @@ class TestDiagnosticIntOption(TestCase): with self.assertRaises(ValueError): diag.parse('3.5') + def test_good_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + self.assertEqual(1, diag.parse('1')) + + def test_bad_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('-1') + + def test_good_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + self.assertEqual(-1, diag.parse('-1')) + + def test_bad_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('1') + class TestDiagnosticBoolOption(TestCase): def test_bool_default_value(self): @@ -201,3 +219,41 @@ class TestDiagnosticComplexStrOption(TestCase): diag = DiagnosticComplexStrOption('option') self.assertEqual('complex string, for testing', diag.parse('complex&.string&;&.for&.testing')) + +class TestDiagnosticListIntOption(TestCase): + def test_tuple_default_value(self): + diag = DiagnosticListIntOption('option', (3,)) + self.assertEqual((3,), diag.parse('')) + + def test_list_default_value(self): + diag = DiagnosticListIntOption('option', [3]) + self.assertEqual([3], diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticListIntOption('option', '3-4') + self.assertEqual([3, 4], diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticListIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3, 2], diag.parse('3-2')) + + def test_parse_single_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3], diag.parse('3')) + + def test_parse_bad_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + + -- GitLab From e1fda179505798ab22df7b81dc9e3352ba2b0191 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 5 Dec 2016 12:01:16 +0100 Subject: [PATCH 4/5] Updated some ocean diagnostics. Changed process_options from static to class method --- earthdiagnostics/diagnostic.py | 13 ++++++-- earthdiagnostics/general/attribute.py | 2 +- earthdiagnostics/general/monthlymean.py | 2 +- earthdiagnostics/general/relink.py | 2 +- earthdiagnostics/general/rewrite.py | 2 +- earthdiagnostics/general/scale.py | 2 +- earthdiagnostics/ocean/areamoc.py | 29 ++++++++--------- earthdiagnostics/ocean/averagesection.py | 31 +++++++++---------- earthdiagnostics/ocean/cutsection.py | 23 ++++++-------- earthdiagnostics/ocean/heatcontent.py | 21 ++++++------- earthdiagnostics/ocean/heatcontentlayer.py | 22 ++++++------- .../statistics/climatologicalpercentile.py | 2 +- .../statistics/monthlypercentile.py | 2 +- test/unit/test_diagnostic.py | 2 +- 14 files changed, 74 insertions(+), 81 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 43b2ce1a..54dd6140 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,4 +1,5 @@ # coding=utf-8 +from earthdiagnostics.constants import Basins from earthdiagnostics.variable import VarType, Domain @@ -103,10 +104,13 @@ class Diagnostic(object): """ raise NotImplementedError("Class must override generate_jobs class method") - @staticmethod - def process_options(options, options_available): + @classmethod + def process_options(cls, options, options_available): processed = dict() options = options[1:] + if len(options) > len(options_available): + raise DiagnosticOptionError('You have specified more options than available for diagnostic ' + '{0}'.format(cls.alias)) for x in range(len(options_available)): option_definition = options_available[x] if len(options) <= x: @@ -177,6 +181,11 @@ class DiagnosticDomainOption(DiagnosticOption): return Domain.parse(self.check_default(option_value)) +class DiagnosticBasinOption(DiagnosticOption): + def parse(self, option_value): + return Basins.parse(self.check_default(option_value)) + + class DiagnosticComplexStrOption(DiagnosticOption): def parse(self, option_value): return self.check_default(option_value).replace('&;', ',').replace('&.', ' ') diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 97756149..c503424c 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -69,7 +69,7 @@ class Attribute(Diagnostic): DiagnosticOption('name'), DiagnosticComplexStrOption('value'), DiagnosticOption('grid')) - options = Diagnostic.process_options(options, options_available) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Attribute(diags.data_manager, startdate, member, chunk, diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 286b2546..cf382d29 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -70,7 +70,7 @@ class MonthlyMean(Diagnostic): DiagnosticDomainOption('domain'), DiagnosticOption('frequency', 'day'), DiagnosticOption('grid', '')) - options = Diagnostic.process_options(options, options_available) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 2c2cdabd..a5956406 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -62,7 +62,7 @@ class Relink(Diagnostic): options_available = (DiagnosticOption('variable'), DiagnosticDomainOption('domain'), DiagnosticBoolOption('move_old', True)) - options = Diagnostic.process_options(options, options_available) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Relink(diags.data_manager, startdate, member, chunk, diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 8751e42c..dcb7b8aa 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -61,7 +61,7 @@ class Rewrite(Diagnostic): options_available = (DiagnosticOption('variable'), DiagnosticDomainOption('domain'), DiagnosticOption('grid', '')) - options = Diagnostic.process_options(options, options_available) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index c4c8769c..06178e41 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -82,7 +82,7 @@ class Scale(Diagnostic): DiagnosticOption('grid'), DiagnosticFloatOption('min_limit', float('nan')), DiagnosticFloatOption('max_limit', float('nan'))) - options = Diagnostic.process_options(options, options_available) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Scale(diags.data_manager, startdate, member, chunk, diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 3942b7f3..361ab8c7 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -1,7 +1,7 @@ # coding=utf-8 import numpy as np from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.box import Box from earthdiagnostics.utils import Utils, TempFile import os @@ -68,24 +68,21 @@ class AreaMoc(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the box to use') - if num_options > 5: - raise Exception('You must specify between 4 and 5 parameters for area moc diagnostic') - box = Box() - box.min_lat = int(options[1]) - box.max_lat = int(options[2]) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) - if num_options > 4: - basin = Basins.parse(options[5]) - else: - basin = Basins.Global + options_available = (DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) + box = Box() + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, basin, box)) + job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, options['basin'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 50961df6..dadc4ceb 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -1,7 +1,7 @@ # coding=utf-8 import os from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domain from earthdiagnostics.variable import Domains @@ -65,25 +65,22 @@ class AverageSection(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 5: - raise Exception('You must specify the variable and the box to average') - if num_options > 6: - raise Exception('You must specify between 5 and 6 parameters for the section average diagnostic') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticIntOption('min_lon'), + DiagnosticIntOption('max_lon'), + DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticDomainOption('domain', Domains.ocean)) + options = cls.process_options(options, options_available) box = Box() - box.min_lon = int(options[2]) - box.max_lon = int(options[3]) - box.min_lat = int(options[4]) - box.max_lat = int(options[5]) - if num_options >= 6: - domain = Domain(options[6]) - else: - domain = Domains.ocean - + box.min_lon = options['min_lon'] + box.max_lon = options['max_lon'] + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, domain, variable, box)) + job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index 95701ab8..fd02854e 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -2,7 +2,8 @@ import numpy as np from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticBoolOption, DiagnosticIntOption, \ + DiagnosticDomainOption from earthdiagnostics.box import Box from earthdiagnostics.utils import Utils from earthdiagnostics.variable import Domain @@ -71,22 +72,16 @@ class CutSection(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable, coordinate and coordinate value') - if num_options > 4: - raise Exception('You must specify between 3 and 4 parameters for the interpolation diagnostic') - variable = options[1] - zonal = options[2].lower() == 'true' - value = int(options[3]) - if num_options >= 4: - domain = Domain(options[4]) - else: - domain = Domains.ocean + options_available = (DiagnosticOption('variable'), + DiagnosticBoolOption('zonal'), + DiagnosticIntOption('value'), + DiagnosticDomainOption('domain', Domains.ocean)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(CutSection(diags.data_manager, startdate, member, chunk, domain, variable, zonal, value)) + job_list.append(CutSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['zonal'], options['value'])) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 5d639e3b..a8eb631f 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -6,7 +6,7 @@ from autosubmit.config.log import Log from earthdiagnostics import cdftools from earthdiagnostics.constants import Basins from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticIntOption from earthdiagnostics.box import Box from earthdiagnostics.variable import Domains @@ -71,19 +71,18 @@ class HeatContent(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the basin, mixed layer option and minimum and maximum depth to use') - if num_options > 4: - raise Exception('You must specify 4 parameters for the heat content diagnostic') - basin = Basins.parse(options[1]) - mixed_layer = int(options[2]) + options_available = (DiagnosticBasinOption('basin'), + DiagnosticIntOption('mixed_layer', None, -1, 1), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth')) + options = cls.process_options(options, options_available) box = Box(True) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(HeatContent(diags.data_manager, startdate, member, chunk, basin, mixed_layer, box)) + job_list.append(HeatContent(diags.data_manager, startdate, member, chunk, + options['basin'], options['mixed_layer'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 65f86cb1..a22e5bc3 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -3,7 +3,7 @@ import numpy as np from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domains @@ -61,22 +61,18 @@ class HeatContentLayer(Diagnostic): :param options: minimum depth, maximum depth, basin=Global :type options: list[str] """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the minimum and maximum depth to use') - if num_options > 3: - raise Exception('You must specify between 2 and 3 parameters for the heat content layer diagnostic') + options_available = (DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) + box = Box(True) - box.min_depth = int(options[1]) - box.max_depth = int(options[2]) - if len(options) > 3: - basin = Basins.parse(options[3]) - else: - basin = Basins.Global + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() handler = Utils.openCdf('mesh_zgr.nc') - mask = Utils.get_mask(basin) + mask = Utils.get_mask(options['basin']) if 'e3t' in handler.variables: mask = handler.variables['e3t'][:] * mask diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index b51eb453..c7c22067 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -68,7 +68,7 @@ class ClimatologicalPercentile(Diagnostic): DiagnosticDomainOption('variable'), DiagnosticListIntOption('leadtimes'), DiagnosticIntOption('bins', 2000)) - options = Diagnostic.process_options(options, options_available) + options = cls.process_options(options, options_available) job_list = list() job_list.append(ClimatologicalPercentile(diags.data_manager, options['domain'], options['variable'], diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index a5fbe167..9f4d1f7c 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -61,7 +61,7 @@ class MonthlyPercentile(Diagnostic): options_available = (DiagnosticOption('domain'), DiagnosticDomainOption('variable'), DiagnosticIntOption('percentile', None, 0, 100)) - options = Diagnostic.process_options(options, options_available) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index f02c13e2..f26a8cf8 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -52,7 +52,7 @@ class TestDiagnostic(TestCase): self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) # def test_empty_process_options(self): - # self.assertEqual(len(Diagnostic.process_options(('diag_name', ), tuple())), 0) + # self.assertEqual(len(cls.process_options(('diag_name', ), tuple())), 0) class TestDiagnosticOption(TestCase): -- GitLab From b17206d7d987db24ab1d3c7864af1b310630963a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 5 Dec 2016 13:27:22 +0100 Subject: [PATCH 5/5] Finished conversion and code cleaned --- earthdiagnostics/cdftools.py | 2 ++ earthdiagnostics/cmorizer.py | 4 ++- earthdiagnostics/cmormanager.py | 14 ++++++--- earthdiagnostics/datamanager.py | 9 ++++-- earthdiagnostics/diagnostic.py | 2 ++ earthdiagnostics/earthdiags.py | 1 - earthdiagnostics/ocean/convectionsites.py | 3 +- earthdiagnostics/ocean/cutsection.py | 3 +- earthdiagnostics/ocean/gyres.py | 3 +- earthdiagnostics/ocean/interpolate.py | 31 +++++++------------ earthdiagnostics/ocean/interpolatecdo.py | 31 ++++++------------- earthdiagnostics/ocean/maxmoc.py | 27 +++++++--------- earthdiagnostics/ocean/siasiesiv.py | 14 ++++----- earthdiagnostics/ocean/verticalmean.py | 22 ++++++------- earthdiagnostics/ocean/verticalmeanmeters.py | 23 +++++++------- .../statistics/climatologicalpercentile.py | 5 +-- .../statistics/monthlypercentile.py | 7 ++--- earthdiagnostics/threddsmanager.py | 24 +++++++++----- earthdiagnostics/utils.py | 1 + test/unit/test_averagesection.py | 2 +- test/unit/test_cdftools.py | 8 +---- 21 files changed, 114 insertions(+), 122 deletions(-) diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index 845aae65..96368d7a 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -58,6 +58,7 @@ class CDFTools(object): if not os.path.exists(output): raise Exception('Error executing {0}\n Output file not created', ' '.join(line)) + # noinspection PyShadowingBuiltins @staticmethod def _check_input(command, input, line): if input: @@ -71,6 +72,7 @@ class CDFTools(object): if not os.path.exists(element): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) + # noinspection PyMethodMayBeStatic def is_exe(self, fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index fa90b12d..39d0b820 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -262,6 +262,7 @@ class Cmorizer(object): handler.close() os.remove(filename) + # noinspection PyMethodMayBeStatic def _remove_valid_limits(self, filename): handler = Utils.openCdf(filename) for variable in handler.variables.keys(): @@ -500,7 +501,8 @@ class Cmorizer(object): var.standard_name = "forecast_period" leadtime = Utils.get_datetime_from_netcdf(handler) startdate = parse_date(self.startdate) - leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate for time in leadtime] + leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate + for time in leadtime] for lt in range(0, len(leadtime)): var[lt] = leadtime[lt].days handler.close() diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 785b515e..96b056ec 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -49,7 +49,9 @@ class CMORManager(DataManager): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: str|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -80,11 +82,11 @@ class CMORManager(DataManager): :param grid: file's grid :type grid: str|NoneType :param year: file's year - :type year: int|str + :type year: int|str|NoneType :param date_str: date string to add directly. Overrides year or chunk configurations - :type date_str: str + :type date_str: str|NoneType :return: path to the file - :rtype: str + :rtype: str|NoneType """ if not frequency: frequency = self.config.frequency @@ -155,6 +157,8 @@ class CMORManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -206,6 +210,8 @@ class CMORManager(DataManager): :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ original_var = var cmor_var = Variable.get_variable(var) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 312300d2..6c6670fc 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -28,7 +28,6 @@ class DataManager(object): UnitConversion.load_conversions() self.lock = threading.Lock() - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VarType.MEAN): """ @@ -50,6 +49,8 @@ class DataManager(object): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -95,6 +96,8 @@ class DataManager(object): :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ raise NotImplementedError() @@ -221,6 +224,8 @@ class DataManager(object): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -427,8 +432,6 @@ class NetCDFFile(object): handler.close() - - class UnitConversion(object): """ Class to manage unit conversions diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 54dd6140..42bae82b 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -75,6 +75,8 @@ class Diagnostic(object): :param year: :param date_str: :param move_old: + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: """ self.data_manager.send_file(filetosend, domain, var, startdate, member, chunk, grid, region, diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 74f718a2..aedb0286 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -237,7 +237,6 @@ class EarthDiags(object): Diagnostic.register(HeatContentLayer) Diagnostic.register(HeatContent) - def clean(self): Log.info('Removing scratch folder...') if os.path.exists(self.config.scratch_dir): diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 3ff6a65b..936f5c9d 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -4,6 +4,7 @@ from autosubmit.config.log import Log from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.constants import Models +from earthdiagnostics.variable import Domains class ConvectionSites(Diagnostic): @@ -113,7 +114,7 @@ class ConvectionSites(Diagnostic): self.mlotst_handler.close() handler.close() - self.send_file(output, 'ocean', 'site', self.startdate, self.member, self.chunk) + self.send_file(output, Domains.ocean, 'site', self.startdate, self.member, self.chunk) Log.info('Finished convection sites for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index fd02854e..786d4d20 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -6,7 +6,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticDomainOption from earthdiagnostics.box import Box from earthdiagnostics.utils import Utils -from earthdiagnostics.variable import Domain from earthdiagnostics.variable import Domains @@ -31,7 +30,7 @@ class CutSection(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: str + :type domain: Domain :param zonal: specifies if section is zonal or meridional :type zonal: bool :param value: value of the section's coordinate diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index 93bdb607..48d612d1 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -5,6 +5,7 @@ from autosubmit.config.log import Log from earthdiagnostics.constants import Models from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.variable import Domains class Gyres(Diagnostic): @@ -143,7 +144,7 @@ class Gyres(Diagnostic): handler.close() handler_original.close() - self.send_file(output, 'ocean', 'gyre', self.startdate, self.member, self.chunk) + self.send_file(output, Domains.ocean, 'gyre', self.startdate, self.member, self.chunk) Log.info('Finished gyres for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) def _gyre(self, site, invert=False): diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 5401dffa..1410be56 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -4,9 +4,9 @@ import threading import os from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, Domains +from earthdiagnostics.variable import Domains class Interpolate(Diagnostic): @@ -31,7 +31,7 @@ class Interpolate(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: str + :type domain: Domain :param model_version: model version :type model_version: str """ @@ -79,27 +79,18 @@ class Interpolate(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the grid and variable to interpolate') - if num_options > 4: - raise Exception('You must specify between 2 and 4 parameters for the interpolation diagnostic') - target_grid = options[1] - variable = options[2] - if num_options >= 3: - domain = Domain(options[3]) - else: - domain = Domains.ocean - if num_options >= 4: - invert_lat = bool(options[4].lower()) - else: - invert_lat = False + options_available = (DiagnosticOption('target_grid'), + DiagnosticOption('variable'), + DiagnosticDomainOption('domain', Domains.ocean), + DiagnosticBoolOption('invert_lat', False)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append( - Interpolate(diags.data_manager, startdate, member, chunk, domain, variable, target_grid, - diags.config.experiment.model_version, invert_lat)) + Interpolate(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['target_grid'], + diags.config.experiment.model_version, options['invert_lat'])) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index a80e8da8..73d9ff07 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -1,6 +1,6 @@ # coding=utf-8 from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile import numpy as np @@ -71,29 +71,16 @@ class InterpolateCDO(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to interpolate') - if num_options > 3: - raise Exception('You must specify between 1 and 3 parameters for the interpolation with CDO diagnostic') - variable = options[1] - - if num_options >= 3: - target_grid = options[2] - else: - target_grid = diags.config.experiment.atmos_grid.lower() - - target_grid = cls._translate_ifs_grids_to_cdo_names(target_grid) - - if num_options >= 3: - domain = Domain(options[3]) - else: - domain = Domains.ocean + options_available = (DiagnosticOption('variable'), + DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), + DiagnosticDomainOption('domain', Domains.ocean)) + options = cls.process_options(options, options_available) + target_grid = cls._translate_ifs_grids_to_cdo_names(options['target_grid']) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append( - InterpolateCDO(diags.data_manager, startdate, member, chunk, domain, variable, target_grid, - diags.config.experiment.model_version)) + job_list.append(InterpolateCDO(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], target_grid, + diags.config.experiment.model_version)) return job_list @classmethod diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index fe212894..847ff84a 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -5,7 +5,7 @@ import os from autosubmit.config.log import Log from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.utils import Utils from earthdiagnostics.variable import Domains @@ -66,20 +66,17 @@ class MaxMoc(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the box to use') - if num_options > 5: - raise Exception('You must specify between 4 and 5 parameters for area moc diagnostic') + options_available = (DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) box = Box() - box.min_lat = int(options[1]) - box.max_lat = int(options[2]) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) - if num_options > 4: - basin = Basins.parse(options[5]) - else: - basin = Basins.Global + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate in diags.startdates: @@ -89,7 +86,7 @@ class MaxMoc(Diagnostic): Log.user_warning('No complete years are available with the given configuration. ' 'MaxMoc can not be computed') for year in years: - job_list.append(MaxMoc(diags.data_manager, startdate, member, year, basin, box)) + job_list.append(MaxMoc(diags.data_manager, startdate, member, year, options['basin'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 6e46264e..1df3841b 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,9 +1,10 @@ # coding=utf-8 import netCDF4 import os -from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption from earthdiagnostics.utils import Utils, TempFile +# noinspection PyUnresolvedReferences import earthdiagnostics.cdftoolspython as cdftoolspython import numpy as np @@ -68,15 +69,14 @@ class Siasiesiv(Diagnostic): :type options: list[str] :return: """ - if len(options) != 2: - raise Exception('You must specify the basin for the siasiesiv diagnostic (and nothing else)') - basin = Basins.parse(options[1]) + options_available = (DiagnosticBasinOption('basin')) + options = cls.process_options(options, options_available) - mask = Utils.get_mask(basin) + mask = Utils.get_mask(options['basin']) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, basin, mask)) + job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask)) mesh_handler = Utils.openCdf('mesh_hgr.nc') Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 7783ab24..41b6e24c 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -1,7 +1,7 @@ # coding=utf-8 from earthdiagnostics import cdftools from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domains @@ -64,23 +64,21 @@ class VerticalMean(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to average vertically') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticIntOption('min_depth', -1), + DiagnosticIntOption('max_depth', -1)) + options = cls.process_options(options, options_available) box = Box() - if num_options >= 2: - box.min_depth = float(options[2]) - if num_options >= 3: - box.max_depth = float(options[3]) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(VerticalMean(diags.data_manager, startdate, member, chunk, - variable, box)) + options['variable'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 5d43196b..9eed80fc 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -1,7 +1,7 @@ # coding=utf-8 from earthdiagnostics import cdftools from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticFloatOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domains @@ -63,21 +63,20 @@ class VerticalMeanMeters(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to average vertically') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticFloatOption('min_depth', -1), + DiagnosticFloatOption('max_depth', -1)) + options = cls.process_options(options, options_available) + box = Box(True) - if num_options >= 2: - box.min_depth = float(options[2]) - if num_options >= 3: - box.max_depth = float(options[3]) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, variable, box)) + job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, options['variable'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index c7c22067..99cd56f0 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -4,7 +4,7 @@ from autosubmit.config.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption, \ DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, Variable, VarType +from earthdiagnostics.variable import Variable, VarType import numpy as np @@ -183,7 +183,8 @@ class ClimatologicalPercentile(Diagnostic): Log.debug('Discretizing realization {0}', realization) def calculate_histogram(time_series): - histogram, self._bins = np.histogram(time_series, bins=self.num_bins, range=(self.min_value, self.max_value)) + histogram, self._bins = np.histogram(time_series, bins=self.num_bins, + range=(self.min_value, self.max_value)) return histogram var = handler.variables[self.variable] diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 9f4d1f7c..15c886ad 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -5,9 +5,8 @@ from autosubmit.config.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, VarType +from earthdiagnostics.variable import VarType from calendar import monthrange -import numpy as np class MonthlyPercentile(Diagnostic): @@ -111,8 +110,8 @@ class MonthlyPercentile(Diagnostic): Log.debug('Computing percentile') Utils.cdo.monpctl(str(self.percentile), input=[variable_file, monmin_file, monmax_file], output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, self.member, - self.chunk, frequency='mon', rename_var=self.variable, vartype=VarType.STATISTIC) + self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, + self.member, self.chunk, frequency='mon', rename_var=self.variable, vartype=VarType.STATISTIC) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index ae8acb00..1f2d28cc 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,6 +1,6 @@ # coding=utf-8 import os -from autosubmit.date.chunk_date_lib import parse_date, add_months, chunk_start_date, chunk_end_date, date2str +from autosubmit.date.chunk_date_lib import parse_date, add_months, chunk_start_date, chunk_end_date from earthdiagnostics.datamanager import DataManager, NetCDFFile from earthdiagnostics.utils import TempFile, Utils @@ -27,9 +27,10 @@ class THREDDSManager(DataManager): if not self.config.data_dir: raise Exception('Can not find model data') - if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size !=1 : + if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: raise Exception('For obs and recon data chunk_size must be always 1') + # noinspection PyUnusedLocal def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VarType.MEAN): aggregation_path = self.get_var_url(variable, startdate, frequency, None, vartype) @@ -70,6 +71,8 @@ class THREDDSManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -81,7 +84,6 @@ class THREDDSManager(DataManager): thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) return thredds_subset.download() - def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, diagnostic=None, cmorized=False, vartype=VarType.MEAN): @@ -122,7 +124,8 @@ class THREDDSManager(DataManager): :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool - + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ if cmorized: raise ValueError('cmorized is not supported in THREDDS manager') @@ -164,6 +167,8 @@ class THREDDSManager(DataManager): :type grid: str :return: path to the file :rtype: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ if not frequency: frequency = self.config.frequency @@ -206,13 +211,14 @@ class THREDDSManager(DataManager): :type grid: str :param box: variable's box :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: """ aggregation_path = self.get_var_url(var, startdate, None, box, vartype) thredds_subset = THREDDSSubset(aggregation_path, var, datetime(year, 1, 1), datetime(year+1, 1, 1)) return thredds_subset.download() - def get_var_url(self, var, startdate, frequency, box, vartype): if not frequency: frequency = self.config.frequency @@ -258,6 +264,8 @@ class THREDDSManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -317,7 +325,8 @@ class THREDDSSubset: time_end += 1 self.dimension_indexes['time'] = (time_start, time_end) - def _download_url(self, url): + @staticmethod + def _download_url(url): temp = TempFile.get() Utils.execute_shell_command(['nccopy', url, temp]) if not Utils.check_netcdf_file(temp): @@ -337,7 +346,8 @@ class THREDDSSubset: return '{0}?{1}{2}'.format(self.thredds_path, dimensions_slice, var_slice) - def _get_slice_index(self, index_tuple): + @staticmethod + def _get_slice_index(index_tuple): return '[{0[0]}:1:{0[1]}]'.format(index_tuple) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7e545c6d..81818c48 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -165,6 +165,7 @@ class Utils(object): original_handler.close() new_handler.close() + # noinspection PyPep8Naming @staticmethod def convert_to_ASCII_if_possible(string, encoding='ascii'): if isinstance(string, basestring): diff --git a/test/unit/test_averagesection.py b/test/unit/test_averagesection.py index 01d32184..8df41be7 100644 --- a/test/unit/test_averagesection.py +++ b/test/unit/test_averagesection.py @@ -5,7 +5,7 @@ from earthdiagnostics.box import Box from earthdiagnostics.ocean.averagesection import AverageSection from mock import Mock -from earthdiagnostics.variable import Domains, Domain +from earthdiagnostics.variable import Domains class TestAverageSection(TestCase): diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py index 0d0fadb8..9ebdc65f 100644 --- a/test/unit/test_cdftools.py +++ b/test/unit/test_cdftools.py @@ -14,14 +14,8 @@ class TestCDFTools(TestCase): # noinspection PyTypeChecker def test_run(self): + # noinspection PyUnusedLocal def mock_exists(path, access=None): - """ - Function for os.path.exists mock - :param path: path to check - :type path: str - :return: true if path does not start with 'bad' - :rtype: bool - """ return not os.path.basename(path.startswith('bad')) with mock.patch('os.path.exists') as exists_mock: -- GitLab