diff --git a/.gitignore b/.gitignore index c583596b75d541f47c68b9ca0c3513c10efc4e10..cafa800b40cfb7157c94ee50969e873bfa644429 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ doc/build/* *.err *.out +*.nc .coverage htmlcov \ No newline at end of file diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index 06331b1d4d9e390f6cf7143fd5c356fb506ebb37..96368d7a88fdc17628f948d3ce2a878c1a5f53c5 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -58,6 +58,7 @@ class CDFTools(object): if not os.path.exists(output): raise Exception('Error executing {0}\n Output file not created', ' '.join(line)) + # noinspection PyShadowingBuiltins @staticmethod def _check_input(command, input, line): if input: @@ -71,18 +72,18 @@ class CDFTools(object): if not os.path.exists(element): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) - @staticmethod - def is_exe(fpath): + # noinspection PyMethodMayBeStatic + def is_exe(self, fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) def _check_command_existence(self, command): if self.path: - if CDFTools.is_exe(os.path.join(self.path, command)): + if self.is_exe(os.path.join(self.path, command)): return else: for path in os.environ["PATH"].split(os.pathsep): path = path.strip('"') exe_file = os.path.join(path, command) - if CDFTools.is_exe(exe_file): + if self.is_exe(exe_file): return raise ValueError('Error executing {0}\n Command does not exist in {1}', command, self.path) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index fa90b12dd3a2b548e24ba3497e248e5a8b056fa4..39d0b820653e45b2b288e425d2ae58011317b103 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -262,6 +262,7 @@ class Cmorizer(object): handler.close() os.remove(filename) + # noinspection PyMethodMayBeStatic def _remove_valid_limits(self, filename): handler = Utils.openCdf(filename) for variable in handler.variables.keys(): @@ -500,7 +501,8 @@ class Cmorizer(object): var.standard_name = "forecast_period" leadtime = Utils.get_datetime_from_netcdf(handler) startdate = parse_date(self.startdate) - leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate for time in leadtime] + leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate + for time in leadtime] for lt in range(0, len(leadtime)): var[lt] = leadtime[lt].days handler.close() diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 785b515ecc886bb226a07b5e4247674b6102ae0e..96b056ecabbe78db3b91a863bb36050ea76f289f 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -49,7 +49,9 @@ class CMORManager(DataManager): :param box: file's box (only needed to retrieve sections or averages) :type box: Box :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str + :type frequency: str|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -80,11 +82,11 @@ class CMORManager(DataManager): :param grid: file's grid :type grid: str|NoneType :param year: file's year - :type year: int|str + :type year: int|str|NoneType :param date_str: date string to add directly. Overrides year or chunk configurations - :type date_str: str + :type date_str: str|NoneType :return: path to the file - :rtype: str + :rtype: str|NoneType """ if not frequency: frequency = self.config.frequency @@ -155,6 +157,8 @@ class CMORManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -206,6 +210,8 @@ class CMORManager(DataManager): :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ original_var = var cmor_var = Variable.get_variable(var) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 312300d2ecf48429981f8d1a1940d99910e0a713..6c6670fca60cbdac533c8e6f820d698b903be62d 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -28,7 +28,6 @@ class DataManager(object): UnitConversion.load_conversions() self.lock = threading.Lock() - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VarType.MEAN): """ @@ -50,6 +49,8 @@ class DataManager(object): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -95,6 +96,8 @@ class DataManager(object): :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ raise NotImplementedError() @@ -221,6 +224,8 @@ class DataManager(object): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -427,8 +432,6 @@ class NetCDFFile(object): handler.close() - - class UnitConversion(object): """ Class to manage unit conversions diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 5eac342437954275e5147ff98d09f7a253432aa2..42bae82b7027344eef4690d20022cc4fb349bcb3 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,5 +1,6 @@ # coding=utf-8 -from earthdiagnostics.variable import VarType +from earthdiagnostics.constants import Basins +from earthdiagnostics.variable import VarType, Domain class Diagnostic(object): @@ -74,6 +75,8 @@ class Diagnostic(object): :param year: :param date_str: :param move_old: + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: """ self.data_manager.send_file(filetosend, domain, var, startdate, member, chunk, grid, region, @@ -103,9 +106,101 @@ class Diagnostic(object): """ raise NotImplementedError("Class must override generate_jobs class method") + @classmethod + def process_options(cls, options, options_available): + processed = dict() + options = options[1:] + if len(options) > len(options_available): + raise DiagnosticOptionError('You have specified more options than available for diagnostic ' + '{0}'.format(cls.alias)) + for x in range(len(options_available)): + option_definition = options_available[x] + if len(options) <= x: + option_value = '' + else: + option_value = options[x] + processed[option_definition.name] = option_definition.parse(option_value) + return processed + def __str__(self): """ Must be implemented by derived classes :return: """ return 'Developer must override base class __str__ method' + + +class DiagnosticOption(object): + + def __init__(self, name, default_value=None): + self.name = name + self.default_value = default_value + + def parse(self, option_value): + option_value = self.check_default(option_value) + return option_value + + def check_default(self, option_value): + if option_value == '': + if self.default_value is None: + raise DiagnosticOptionError('Option {0} is not optional'.format(self.name)) + else: + return self.default_value + return option_value + + +class DiagnosticFloatOption(DiagnosticOption): + def parse(self, option_value): + return float(self.check_default(option_value)) + + +class DiagnosticIntOption(DiagnosticOption): + + def __init__(self, name, default_value=None, min_limit=None, max_limit=None): + super(DiagnosticIntOption, self).__init__(name, default_value) + self.min_limit = min_limit + self.max_limit = max_limit + + def parse(self, option_value): + value = int(self.check_default(option_value)) + if self.min_limit is not None and value < self.min_limit: + raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) + if self.max_limit is not None and value > self.max_limit: + raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) + return value + + +class DiagnosticListIntOption(DiagnosticOption): + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, tuple) or isinstance(option_value, list): + return option_value + return [int(i) for i in option_value.split('-')] + + +class DiagnosticDomainOption(DiagnosticOption): + def parse(self, option_value): + return Domain.parse(self.check_default(option_value)) + + +class DiagnosticBasinOption(DiagnosticOption): + def parse(self, option_value): + return Basins.parse(self.check_default(option_value)) + + +class DiagnosticComplexStrOption(DiagnosticOption): + def parse(self, option_value): + return self.check_default(option_value).replace('&;', ',').replace('&.', ' ') + + +class DiagnosticBoolOption(DiagnosticOption): + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, bool): + return option_value + else: + return option_value.lower() in ('true', 't', 'yes') + + +class DiagnosticOptionError(Exception): + pass diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 74f718a266632a6328156d7c298a07c13b36a6d3..aedb0286043be683a955e9d64008463c06825b96 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -237,7 +237,6 @@ class EarthDiags(object): Diagnostic.register(HeatContentLayer) Diagnostic.register(HeatContent) - def clean(self): Log.info('Removing scratch folder...') if os.path.exists(self.config.scratch_dir): diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 59edf3c65059657bb560882fc2cdd5f063edd2bd..c503424cff688cf097eb6256f943380375e1c5d4 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils from earthdiagnostics.variable import Domain @@ -63,23 +63,18 @@ class Attribute(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the variable, domain, attributte name and value to write') - if num_options > 5: - raise Exception('You must between 4 and 5 parameters for the rewrite diagnostic') - variable = options[1] - domain = Domain(options[2]) - name = options[3] - value = options[4] - value = value.replace('&;', ',').replace('&.', ' ') - if num_options >= 5: - grid = options[5] - else: - grid = None + + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('name'), + DiagnosticComplexStrOption('value'), + DiagnosticOption('grid')) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Attribute(diags.data_manager, startdate, member, chunk, domain, variable, grid, name, value)) + job_list.append(Attribute(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'], options['grid'], + options['value'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 69983f37176c2b6774e302ee2681bc69131c8a08..cf382d2985428bab89a1be11f04465d40e1dceaa 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -1,7 +1,7 @@ # coding=utf-8 import os -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domain @@ -65,26 +65,16 @@ class MonthlyMean(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to average monthly') - if num_options > 4: - raise Exception('You must specify between 2 and 4 parameters for the monthly mean diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - frequency = options[3] - else: - frequency = 'day' - if num_options >= 4: - grid = options[4] - else: - grid = None + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('frequency', 'day'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, domain, variable, - frequency, grid)) + job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 3009f20e1c9cfd8116668617406537de9e3354ee..a5956406e89591b8b6481963470001dcdce1a62b 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption from earthdiagnostics.variable import Domain @@ -59,20 +59,14 @@ class Relink(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to link') - if num_options > 3: - raise Exception('You must between 2 and 3 parameters for the relink diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - move_old = bool(options[3].lower()) - else: - move_old = True + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticBoolOption('move_old', True)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Relink(diags.data_manager, startdate, member, chunk, domain, variable, move_old)) + job_list.append(Relink(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['move_old'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index c7acd74401dd0fd5f9ce625d306849b7f62fbaae..dcb7b8aaa4db061ddb808467e0b443f88420d8ed 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.variable import Domain @@ -58,20 +58,14 @@ class Rewrite(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the variable and domain to rewrite') - if num_options > 3: - raise Exception('You must between 2 and 3 parameters for the rewrite diagnostic') - variable = options[1] - domain = Domain(options[2]) - if num_options >= 3: - grid = options[3] - else: - grid = None + options_available = (DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, domain, variable, grid)) + job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'])) return job_list def compute(self): diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index d41797b5ffb629f93af3af691cdf8756904c1f5d..06178e41365c722b07f9818c4c795b7dde6733e2 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,8 +1,9 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticFloatOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils from earthdiagnostics.variable import Domain import numpy as np +import math class Scale(Diagnostic): @@ -74,32 +75,19 @@ class Scale(Diagnostic): raise Exception('You must specify the acale and offset values and the variable and domain to scale') if num_options > 5: raise Exception('You must between 4 and 5 parameters for the rewrite diagnostic') - value = float(options[1]) - offset = float(options[2]) - variable = options[3] - domain = Domain(options[4]) - if num_options >= 5: - grid = options[5] - else: - grid = None - if num_options >= 6: - if options[6].lower() == 'none': - min_limit = None - else: - min_limit = float(options[6]) - else: - min_limit = None - if num_options >= 7: - if options[7].lower() == 'none': - max_limit = None - else: - max_limit = float(options[7]) - else: - max_limit = None + options_available = (DiagnosticFloatOption('value'), + DiagnosticFloatOption('offset'), + DiagnosticOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticOption('grid'), + DiagnosticFloatOption('min_limit', float('nan')), + DiagnosticFloatOption('max_limit', float('nan'))) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Scale(diags.data_manager, startdate, member, chunk, value, offset, domain, variable, grid, - min_limit, max_limit)) + job_list.append(Scale(diags.data_manager, startdate, member, chunk, + options['value'], options['offset'], options['domain'], options['variable'], + options['grid'], options['min_limit'], options['max_limit'])) return job_list def compute(self): @@ -119,9 +107,9 @@ class Scale(Diagnostic): grid=self.grid) def _check_limits(self): - if self.min_limit is not None and np.amin(self.original_values) < self.min_limit: + if not math.isnan(self.min_limit) and np.amin(self.original_values) < self.min_limit: return False - if self.max_limit is not None and np.amax(self.original_values) > self.max_limit: + if not math.isnan(self.max_limit) is not None and np.amax(self.original_values) > self.max_limit: return False return True diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 3942b7f3b3b37a27372b3143d466fb9f7f2d74ab..361ab8c72131964d6e90e2fd36603d157c08b128 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -1,7 +1,7 @@ # coding=utf-8 import numpy as np from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.box import Box from earthdiagnostics.utils import Utils, TempFile import os @@ -68,24 +68,21 @@ class AreaMoc(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the box to use') - if num_options > 5: - raise Exception('You must specify between 4 and 5 parameters for area moc diagnostic') - box = Box() - box.min_lat = int(options[1]) - box.max_lat = int(options[2]) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) - if num_options > 4: - basin = Basins.parse(options[5]) - else: - basin = Basins.Global + options_available = (DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) + box = Box() + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, basin, box)) + job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, options['basin'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 50961df6910496634cb2c8bdb602d532db7385e3..dadc4cebd3587b73243bc43155b145fac7f39847 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -1,7 +1,7 @@ # coding=utf-8 import os from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domain from earthdiagnostics.variable import Domains @@ -65,25 +65,22 @@ class AverageSection(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 5: - raise Exception('You must specify the variable and the box to average') - if num_options > 6: - raise Exception('You must specify between 5 and 6 parameters for the section average diagnostic') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticIntOption('min_lon'), + DiagnosticIntOption('max_lon'), + DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticDomainOption('domain', Domains.ocean)) + options = cls.process_options(options, options_available) box = Box() - box.min_lon = int(options[2]) - box.max_lon = int(options[3]) - box.min_lat = int(options[4]) - box.max_lat = int(options[5]) - if num_options >= 6: - domain = Domain(options[6]) - else: - domain = Domains.ocean - + box.min_lon = options['min_lon'] + box.max_lon = options['max_lon'] + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, domain, variable, box)) + job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 3ff6a65b1837b3a13f068ea36637ff6efa1abb0c..936f5c9da8e2cd972790d2e87202f37f3591a9c6 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -4,6 +4,7 @@ from autosubmit.config.log import Log from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.constants import Models +from earthdiagnostics.variable import Domains class ConvectionSites(Diagnostic): @@ -113,7 +114,7 @@ class ConvectionSites(Diagnostic): self.mlotst_handler.close() handler.close() - self.send_file(output, 'ocean', 'site', self.startdate, self.member, self.chunk) + self.send_file(output, Domains.ocean, 'site', self.startdate, self.member, self.chunk) Log.info('Finished convection sites for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index 95701ab806f7106acbc81932ccf1d44d3a287986..786d4d20958edf98185b06fe2d7280798a8826b8 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -2,10 +2,10 @@ import numpy as np from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticBoolOption, DiagnosticIntOption, \ + DiagnosticDomainOption from earthdiagnostics.box import Box from earthdiagnostics.utils import Utils -from earthdiagnostics.variable import Domain from earthdiagnostics.variable import Domains @@ -30,7 +30,7 @@ class CutSection(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: str + :type domain: Domain :param zonal: specifies if section is zonal or meridional :type zonal: bool :param value: value of the section's coordinate @@ -71,22 +71,16 @@ class CutSection(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable, coordinate and coordinate value') - if num_options > 4: - raise Exception('You must specify between 3 and 4 parameters for the interpolation diagnostic') - variable = options[1] - zonal = options[2].lower() == 'true' - value = int(options[3]) - if num_options >= 4: - domain = Domain(options[4]) - else: - domain = Domains.ocean + options_available = (DiagnosticOption('variable'), + DiagnosticBoolOption('zonal'), + DiagnosticIntOption('value'), + DiagnosticDomainOption('domain', Domains.ocean)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(CutSection(diags.data_manager, startdate, member, chunk, domain, variable, zonal, value)) + job_list.append(CutSection(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['zonal'], options['value'])) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index 93bdb6076718f9ecd5077bff3d6ac7bdc586e831..48d612d187e0425685beb8304463bacbe482003d 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -5,6 +5,7 @@ from autosubmit.config.log import Log from earthdiagnostics.constants import Models from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.variable import Domains class Gyres(Diagnostic): @@ -143,7 +144,7 @@ class Gyres(Diagnostic): handler.close() handler_original.close() - self.send_file(output, 'ocean', 'gyre', self.startdate, self.member, self.chunk) + self.send_file(output, Domains.ocean, 'gyre', self.startdate, self.member, self.chunk) Log.info('Finished gyres for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) def _gyre(self, site, invert=False): diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 5d639e3b9cc2b90daee4a385e27f5fa0d2a6bf26..a8eb631f9605db3c07a094954be0d5f0ba780c9a 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -6,7 +6,7 @@ from autosubmit.config.log import Log from earthdiagnostics import cdftools from earthdiagnostics.constants import Basins from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticIntOption from earthdiagnostics.box import Box from earthdiagnostics.variable import Domains @@ -71,19 +71,18 @@ class HeatContent(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the basin, mixed layer option and minimum and maximum depth to use') - if num_options > 4: - raise Exception('You must specify 4 parameters for the heat content diagnostic') - basin = Basins.parse(options[1]) - mixed_layer = int(options[2]) + options_available = (DiagnosticBasinOption('basin'), + DiagnosticIntOption('mixed_layer', None, -1, 1), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth')) + options = cls.process_options(options, options_available) box = Box(True) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(HeatContent(diags.data_manager, startdate, member, chunk, basin, mixed_layer, box)) + job_list.append(HeatContent(diags.data_manager, startdate, member, chunk, + options['basin'], options['mixed_layer'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 65f86cb16e64bbc3cc77bb460d64bb688fb39c3f..a22e5bc3ae84c4ee67e5569d5285f7be9c009480 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -3,7 +3,7 @@ import numpy as np from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domains @@ -61,22 +61,18 @@ class HeatContentLayer(Diagnostic): :param options: minimum depth, maximum depth, basin=Global :type options: list[str] """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the minimum and maximum depth to use') - if num_options > 3: - raise Exception('You must specify between 2 and 3 parameters for the heat content layer diagnostic') + options_available = (DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) + box = Box(True) - box.min_depth = int(options[1]) - box.max_depth = int(options[2]) - if len(options) > 3: - basin = Basins.parse(options[3]) - else: - basin = Basins.Global + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() handler = Utils.openCdf('mesh_zgr.nc') - mask = Utils.get_mask(basin) + mask = Utils.get_mask(options['basin']) if 'e3t' in handler.variables: mask = handler.variables['e3t'][:] * mask diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 5401dffa4aeb18afe59869eefc1466a1a6895fe9..1410be5662eea2a3171c538297f34b2448adf902 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -4,9 +4,9 @@ import threading import os from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, Domains +from earthdiagnostics.variable import Domains class Interpolate(Diagnostic): @@ -31,7 +31,7 @@ class Interpolate(Diagnostic): :param variable: variable's name :type variable: str :param domain: variable's domain - :type domain: str + :type domain: Domain :param model_version: model version :type model_version: str """ @@ -79,27 +79,18 @@ class Interpolate(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 2: - raise Exception('You must specify the grid and variable to interpolate') - if num_options > 4: - raise Exception('You must specify between 2 and 4 parameters for the interpolation diagnostic') - target_grid = options[1] - variable = options[2] - if num_options >= 3: - domain = Domain(options[3]) - else: - domain = Domains.ocean - if num_options >= 4: - invert_lat = bool(options[4].lower()) - else: - invert_lat = False + options_available = (DiagnosticOption('target_grid'), + DiagnosticOption('variable'), + DiagnosticDomainOption('domain', Domains.ocean), + DiagnosticBoolOption('invert_lat', False)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append( - Interpolate(diags.data_manager, startdate, member, chunk, domain, variable, target_grid, - diags.config.experiment.model_version, invert_lat)) + Interpolate(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['target_grid'], + diags.config.experiment.model_version, options['invert_lat'])) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index a80e8da834d2800dc8af9cf29601aded87b8fa95..73d9ff079d2a87009598750d8dbaf0309a56b182 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -1,6 +1,6 @@ # coding=utf-8 from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption from earthdiagnostics.utils import Utils, TempFile import numpy as np @@ -71,29 +71,16 @@ class InterpolateCDO(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to interpolate') - if num_options > 3: - raise Exception('You must specify between 1 and 3 parameters for the interpolation with CDO diagnostic') - variable = options[1] - - if num_options >= 3: - target_grid = options[2] - else: - target_grid = diags.config.experiment.atmos_grid.lower() - - target_grid = cls._translate_ifs_grids_to_cdo_names(target_grid) - - if num_options >= 3: - domain = Domain(options[3]) - else: - domain = Domains.ocean + options_available = (DiagnosticOption('variable'), + DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), + DiagnosticDomainOption('domain', Domains.ocean)) + options = cls.process_options(options, options_available) + target_grid = cls._translate_ifs_grids_to_cdo_names(options['target_grid']) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append( - InterpolateCDO(diags.data_manager, startdate, member, chunk, domain, variable, target_grid, - diags.config.experiment.model_version)) + job_list.append(InterpolateCDO(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], target_grid, + diags.config.experiment.model_version)) return job_list @classmethod diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index fe212894032b88ac6f379b51d114e2f04311ae52..847ff84a502d77d14929df87805f56a4418ce01c 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -5,7 +5,7 @@ import os from autosubmit.config.log import Log from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticBasinOption from earthdiagnostics.utils import Utils from earthdiagnostics.variable import Domains @@ -66,20 +66,17 @@ class MaxMoc(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 4: - raise Exception('You must specify the box to use') - if num_options > 5: - raise Exception('You must specify between 4 and 5 parameters for area moc diagnostic') + options_available = (DiagnosticIntOption('min_lat'), + DiagnosticIntOption('max_lat'), + DiagnosticIntOption('min_depth'), + DiagnosticIntOption('max_depth'), + DiagnosticBasinOption('basin', Basins.Global)) + options = cls.process_options(options, options_available) box = Box() - box.min_lat = int(options[1]) - box.max_lat = int(options[2]) - box.min_depth = int(options[3]) - box.max_depth = int(options[4]) - if num_options > 4: - basin = Basins.parse(options[5]) - else: - basin = Basins.Global + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] job_list = list() for startdate in diags.startdates: @@ -89,7 +86,7 @@ class MaxMoc(Diagnostic): Log.user_warning('No complete years are available with the given configuration. ' 'MaxMoc can not be computed') for year in years: - job_list.append(MaxMoc(diags.data_manager, startdate, member, year, basin, box)) + job_list.append(MaxMoc(diags.data_manager, startdate, member, year, options['basin'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 6e46264e3930d3f7179d9241b8e9b107e54de4d1..1df3841b4e06f2ce1f3cc8b91aac302dece1d897 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,9 +1,10 @@ # coding=utf-8 import netCDF4 import os -from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption from earthdiagnostics.utils import Utils, TempFile +# noinspection PyUnresolvedReferences import earthdiagnostics.cdftoolspython as cdftoolspython import numpy as np @@ -68,15 +69,14 @@ class Siasiesiv(Diagnostic): :type options: list[str] :return: """ - if len(options) != 2: - raise Exception('You must specify the basin for the siasiesiv diagnostic (and nothing else)') - basin = Basins.parse(options[1]) + options_available = (DiagnosticBasinOption('basin')) + options = cls.process_options(options, options_available) - mask = Utils.get_mask(basin) + mask = Utils.get_mask(options['basin']) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, basin, mask)) + job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask)) mesh_handler = Utils.openCdf('mesh_hgr.nc') Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 7783ab245f5ac14dec5a62f6fdbb8de87c5238d6..41b6e24c816e8822b22c0bdd4f85e31b1665e1b0 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -1,7 +1,7 @@ # coding=utf-8 from earthdiagnostics import cdftools from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domains @@ -64,23 +64,21 @@ class VerticalMean(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to average vertically') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticIntOption('min_depth', -1), + DiagnosticIntOption('max_depth', -1)) + options = cls.process_options(options, options_available) box = Box() - if num_options >= 2: - box.min_depth = float(options[2]) - if num_options >= 3: - box.max_depth = float(options[3]) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(VerticalMean(diags.data_manager, startdate, member, chunk, - variable, box)) + options['variable'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 5d43196b262926a32c0da3f6911e662d5cebe943..9eed80fc4d2db8355169386e3b7aa3b2aa996c72 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -1,7 +1,7 @@ # coding=utf-8 from earthdiagnostics import cdftools from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticFloatOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domains @@ -63,21 +63,20 @@ class VerticalMeanMeters(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 1: - raise Exception('You must specify the variable to average vertically') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - variable = options[1] + options_available = (DiagnosticOption('variable'), + DiagnosticFloatOption('min_depth', -1), + DiagnosticFloatOption('max_depth', -1)) + options = cls.process_options(options, options_available) + box = Box(True) - if num_options >= 2: - box.min_depth = float(options[2]) - if num_options >= 3: - box.max_depth = float(options[3]) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, variable, box)) + job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, options['variable'], box)) return job_list def compute(self): diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 5df474194dbcfc818ce629d0930a25fe6f5b5bfc..99cd56f05d98e82e5cd19133e05b91e394b1eff5 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -1,9 +1,10 @@ # coding=utf-8 from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption, \ + DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, Variable, VarType +from earthdiagnostics.variable import Variable, VarType import numpy as np @@ -63,22 +64,15 @@ class ClimatologicalPercentile(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable (and its domain) and the leadtimes you want to compute ' - 'the percentiles on') - if num_options > 4: - raise Exception('You must specify between three and 4 parameters for the climatological percentiles') - domain = Domain(options[1]) - variable = options[2] - leadtimes = [int(i) for i in options[3].split('-')] - if num_options > 3: - num_bins = int(options[4]) - else: - num_bins = 2000 + options_available = (DiagnosticOption('domain'), + DiagnosticDomainOption('variable'), + DiagnosticListIntOption('leadtimes'), + DiagnosticIntOption('bins', 2000)) + options = cls.process_options(options, options_available) job_list = list() - job_list.append(ClimatologicalPercentile(diags.data_manager, domain, variable, leadtimes, num_bins, + job_list.append(ClimatologicalPercentile(diags.data_manager, options['domain'], options['variable'], + options['leadtimes'], options['bins'], diags.config.experiment)) return job_list @@ -189,7 +183,8 @@ class ClimatologicalPercentile(Diagnostic): Log.debug('Discretizing realization {0}', realization) def calculate_histogram(time_series): - histogram, self._bins = np.histogram(time_series, bins=self.num_bins, range=(self.min_value, self.max_value)) + histogram, self._bins = np.histogram(time_series, bins=self.num_bins, + range=(self.min_value, self.max_value)) return histogram var = handler.variables[self.variable] diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index d746911e9879978f3471b3b1751e6f3b493741ed..15c886adf94c17fdf80f034c12e5c5dd5274d999 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -3,11 +3,10 @@ import shutil from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticIntOption from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, VarType +from earthdiagnostics.variable import VarType from calendar import monthrange -import numpy as np class MonthlyPercentile(Diagnostic): @@ -58,23 +57,15 @@ class MonthlyPercentile(Diagnostic): :type options: list[str] :return: """ - num_options = len(options) - 1 - if num_options < 3: - raise Exception('You must specify the variable (and its domain) to average vertically and ' - 'the percentil you want') - if num_options > 3: - raise Exception('You must specify between one and three parameters for the vertical mean') - - domain = Domain(options[1]) - variable = options[2] - percentile = int(options[3]) - if percentile < 0 or percentile > 100: - raise Exception('Percentile value must be in the interval [0,100]') + options_available = (DiagnosticOption('domain'), + DiagnosticDomainOption('variable'), + DiagnosticIntOption('percentile', None, 0, 100)) + options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(MonthlyPercentile(diags.data_manager, startdate, member, chunk, - variable, domain, percentile)) + options['variable'], options['domain'], options['percentile'])) return job_list def compute(self): @@ -119,8 +110,8 @@ class MonthlyPercentile(Diagnostic): Log.debug('Computing percentile') Utils.cdo.monpctl(str(self.percentile), input=[variable_file, monmin_file, monmax_file], output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, self.member, - self.chunk, frequency='mon', rename_var=self.variable, vartype=VarType.STATISTIC) + self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, + self.member, self.chunk, frequency='mon', rename_var=self.variable, vartype=VarType.STATISTIC) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index ae8acb004b6ecbad6c41e67e0b00d3e3a4629d59..1f2d28ccec66e7291d11a229bfbbc5f6194976ab 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,6 +1,6 @@ # coding=utf-8 import os -from autosubmit.date.chunk_date_lib import parse_date, add_months, chunk_start_date, chunk_end_date, date2str +from autosubmit.date.chunk_date_lib import parse_date, add_months, chunk_start_date, chunk_end_date from earthdiagnostics.datamanager import DataManager, NetCDFFile from earthdiagnostics.utils import TempFile, Utils @@ -27,9 +27,10 @@ class THREDDSManager(DataManager): if not self.config.data_dir: raise Exception('Can not find model data') - if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size !=1 : + if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: raise Exception('For obs and recon data chunk_size must be always 1') + # noinspection PyUnusedLocal def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VarType.MEAN): aggregation_path = self.get_var_url(variable, startdate, frequency, None, vartype) @@ -70,6 +71,8 @@ class THREDDSManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -81,7 +84,6 @@ class THREDDSManager(DataManager): thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) return thredds_subset.download() - def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, diagnostic=None, cmorized=False, vartype=VarType.MEAN): @@ -122,7 +124,8 @@ class THREDDSManager(DataManager): :type diagnostic: Diagnostic :param cmorized: flag to indicate if file was generated in cmorization process :type cmorized: bool - + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ if cmorized: raise ValueError('cmorized is not supported in THREDDS manager') @@ -164,6 +167,8 @@ class THREDDSManager(DataManager): :type grid: str :return: path to the file :rtype: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType """ if not frequency: frequency = self.config.frequency @@ -206,13 +211,14 @@ class THREDDSManager(DataManager): :type grid: str :param box: variable's box :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: """ aggregation_path = self.get_var_url(var, startdate, None, box, vartype) thredds_subset = THREDDSSubset(aggregation_path, var, datetime(year, 1, 1), datetime(year+1, 1, 1)) return thredds_subset.download() - def get_var_url(self, var, startdate, frequency, box, vartype): if not frequency: frequency = self.config.frequency @@ -258,6 +264,8 @@ class THREDDSManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VarType :return: path to the copy created on the scratch folder :rtype: str """ @@ -317,7 +325,8 @@ class THREDDSSubset: time_end += 1 self.dimension_indexes['time'] = (time_start, time_end) - def _download_url(self, url): + @staticmethod + def _download_url(url): temp = TempFile.get() Utils.execute_shell_command(['nccopy', url, temp]) if not Utils.check_netcdf_file(temp): @@ -337,7 +346,8 @@ class THREDDSSubset: return '{0}?{1}{2}'.format(self.thredds_path, dimensions_slice, var_slice) - def _get_slice_index(self, index_tuple): + @staticmethod + def _get_slice_index(index_tuple): return '[{0[0]}:1:{0[1]}]'.format(index_tuple) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7e545c6d507cffdcdac456b70709e8ecbd26749f..81818c483db9cb5f801e114104891713d374ec71 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -165,6 +165,7 @@ class Utils(object): original_handler.close() new_handler.close() + # noinspection PyPep8Naming @staticmethod def convert_to_ASCII_if_possible(string, encoding='ascii'): if isinstance(string, basestring): diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index faf7c74a601680593b1ea80c47952248249b1365..603087f221cdc2b40897474bbf4097f4398e6e82 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -66,6 +66,12 @@ class Variable(object): class Domain(object): + @staticmethod + def parse(domain_name): + if isinstance(domain_name, Domain): + return domain_name + return Domain(domain_name) + def __init__(self, domain_name): domain_name = domain_name.lower() if domain_name == 'seaice': diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 7a15d906c9002de2c6185d56ed22c90d2d2af217..c611a13fce561d2dad3affb9a882cd98f030f050 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -3,7 +3,7 @@ from test_data_manager import TestConversion from test.unit.test_variable import TestVariable from test_constants import TestBasin, TestBasins from test_box import TestBox -from test_diagnostic import TestDiagnostic +from test_diagnostic import * from test_cdftools import TestCDFTools from test_utils import TestTempFile, TestUtils from test_psi import TestPsi diff --git a/test/unit/test_averagesection.py b/test/unit/test_averagesection.py index 01d32184a19c4686015f9aeb24e95347c860ea41..8df41be7c5a8c6a02b29cc14acf9af755cf1c3b3 100644 --- a/test/unit/test_averagesection.py +++ b/test/unit/test_averagesection.py @@ -5,7 +5,7 @@ from earthdiagnostics.box import Box from earthdiagnostics.ocean.averagesection import AverageSection from mock import Mock -from earthdiagnostics.variable import Domains, Domain +from earthdiagnostics.variable import Domains class TestAverageSection(TestCase): diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py index 0d0fadb85cec7192220fde39d2d72856b94976c4..9ebdc65f9289ea0a217f046800e258cde95904e6 100644 --- a/test/unit/test_cdftools.py +++ b/test/unit/test_cdftools.py @@ -14,14 +14,8 @@ class TestCDFTools(TestCase): # noinspection PyTypeChecker def test_run(self): + # noinspection PyUnusedLocal def mock_exists(path, access=None): - """ - Function for os.path.exists mock - :param path: path to check - :type path: str - :return: true if path does not start with 'bad' - :rtype: bool - """ return not os.path.basename(path.startswith('bad')) with mock.patch('os.path.exists') as exists_mock: diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index 117e8e22d77e32fb7ffd2bd37321db2b538362ba..f26a8cf843ff3d376af72d49e0a6bf2eec19cac6 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -1,7 +1,9 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.diagnostic import * from unittest import TestCase +from earthdiagnostics.variable import Domains + class TestDiagnostic(TestCase): @@ -46,3 +48,212 @@ class TestDiagnostic(TestCase): def test_repr(self): self.assertEquals(self.diagnostic.__repr__(), str(self.diagnostic)) + def test_empty_process_options(self): + self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) + + # def test_empty_process_options(self): + # self.assertEqual(len(cls.process_options(('diag_name', ), tuple())), 0) + + +class TestDiagnosticOption(TestCase): + + def test_good_default_value(self): + diag = DiagnosticOption('option', 'default') + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticOption('option') + self.assertEqual('value', diag.parse('value')) + + +class TestDiagnosticFloatOption(TestCase): + def test_float_default_value(self): + diag = DiagnosticFloatOption('option', 3.0) + self.assertEqual(3.0, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticFloatOption('option', '3') + self.assertEqual(3.0, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticFloatOption('option', 'default') + with self.assertRaises(ValueError): + self.assertEqual('default', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticFloatOption('option') + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('default', diag.parse('')) + + def test_parse_value(self): + diag = DiagnosticFloatOption('option') + self.assertEqual(3.25, diag.parse('3.25')) + + +class TestDiagnosticDomainOption(TestCase): + def test_domain_default_value(self): + diag = DiagnosticDomainOption('option', Domains.ocean) + self.assertEqual(Domains.ocean, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticDomainOption('option', 'atmos') + self.assertEqual(Domains.atmos, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticDomainOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticDomainOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticDomainOption('option') + self.assertEqual(Domains.seaIce, diag.parse('seaice')) + + +class TestDiagnosticIntOption(TestCase): + def test_int_default_value(self): + diag = DiagnosticIntOption('option', 3) + self.assertEqual(3, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticIntOption('option', '3') + self.assertEqual(3, diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticIntOption('option') + self.assertEqual(3, diag.parse('3')) + + def test_parse_bad_value(self): + diag = DiagnosticIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + + def test_good_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + self.assertEqual(1, diag.parse('1')) + + def test_bad_low_limit(self): + diag = DiagnosticIntOption('option', None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('-1') + + def test_good_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + self.assertEqual(-1, diag.parse('-1')) + + def test_bad_high_limit(self): + diag = DiagnosticIntOption('option', None, None, 0) + with self.assertRaises(DiagnosticOptionError): + diag.parse('1') + + +class TestDiagnosticBoolOption(TestCase): + def test_bool_default_value(self): + diag = DiagnosticBoolOption('option', True) + self.assertEqual(True, diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticBoolOption('option', 'False') + self.assertEqual(False, diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticBoolOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_True(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_true(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('true')) + + def test_parse_t(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('t')) + + def test_parse_yes(self): + diag = DiagnosticBoolOption('option') + self.assertTrue(diag.parse('YES')) + + def test_parse_bad_value(self): + diag = DiagnosticBoolOption('option') + self.assertFalse(diag.parse('3.5')) + + +class TestDiagnosticComplexStrOption(TestCase): + def test_complex_default_value(self): + diag = DiagnosticComplexStrOption('option', 'default&.str&;&.working') + self.assertEqual('default str, working', diag.parse('')) + + def test_simple_default_value(self): + diag = DiagnosticComplexStrOption('default str, working', 'default str, working') + self.assertEqual('default str, working', diag.parse('')) + + def test_no_default_value(self): + diag = DiagnosticComplexStrOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticComplexStrOption('option') + self.assertEqual('complex string, for testing', diag.parse('complex&.string&;&.for&.testing')) + + +class TestDiagnosticListIntOption(TestCase): + def test_tuple_default_value(self): + diag = DiagnosticListIntOption('option', (3,)) + self.assertEqual((3,), diag.parse('')) + + def test_list_default_value(self): + diag = DiagnosticListIntOption('option', [3]) + self.assertEqual([3], diag.parse('')) + + def test_str_default_value(self): + diag = DiagnosticListIntOption('option', '3-4') + self.assertEqual([3, 4], diag.parse('')) + + def test_bad_default_value(self): + diag = DiagnosticListIntOption('option', 'default') + with self.assertRaises(ValueError): + diag.parse('') + + def test_no_default_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(DiagnosticOptionError): + diag.parse('') + + def test_parse_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3, 2], diag.parse('3-2')) + + def test_parse_single_value(self): + diag = DiagnosticListIntOption('option') + self.assertEqual([3], diag.parse('3')) + + def test_parse_bad_value(self): + diag = DiagnosticListIntOption('option') + with self.assertRaises(ValueError): + diag.parse('3.5') + +