diff --git a/VERSION b/VERSION index fa1beb5eb5beaa35834de59e47d6ac0d9e2388e0..e2859adcb157b1787daaca9a0489d008bbfbab4b 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc8 +3.0.0rc9 diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index efdadcfec05dea121913251ed6c6cbd99502c28d..87c2169ac64df3590c5fec0adf402829c4679e2b 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -447,37 +447,8 @@ class Cmorizer(object): return temp = TempFile.get() - handler = Utils.open_cdf(file_path) - coords = [self.lon_name, self.lat_name, 'time'] - if 'leadtime' in handler.variables.keys(): - coords.append('leadtime') - - if var_cmor.domain == ModelingRealms.ocean: - lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', - 'depth': 'lev'} - elif var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: - lev_dimensions = {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', - 'depth_4': 'sdepth'} - elif var_cmor.domain == ModelingRealms.atmos: - lev_dimensions = {'depth': 'plev'} - else: - lev_dimensions = {} - - for lev_dim in lev_dimensions.keys(): - if lev_dim in handler.variables[variable].dimensions: - coords.append(lev_dim) - - handler.variables[variable].coordinates = ' '.join(set(coords)) - handler.close() - - cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable)) - for lev_original, lev_target in six.iteritems(lev_dimensions): - try: - cube.coord(lev_original).var_name = lev_target - except iris.exceptions.CoordinateNotFoundError: - pass - - iris.save(cube, temp, zlib=True) + lev_dimensions = self._set_coordinates_attribute(file_path, var_cmor, variable) + self._rename_level_coords(file_path, lev_dimensions, temp, variable) if alias.basin is None: region = None @@ -516,6 +487,41 @@ class Cmorizer(object): region_str = '' Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str) + def _rename_level_coords(self, file_path, lev_dimensions, temp, variable): + cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable)) + for lev_original, lev_target in six.iteritems(lev_dimensions): + try: + cube.coord(lev_original).var_name = lev_target + except iris.exceptions.CoordinateNotFoundError: + pass + iris.save(cube, temp, zlib=True) + + def _set_coordinates_attribute(self, file_path, var_cmor, variable): + handler = Utils.open_cdf(file_path) + coords = [self.lon_name, self.lat_name, 'time'] + if 'leadtime' in handler.variables.keys(): + coords.append('leadtime') + lev_dimensions = self._get_lev_dimensions(var_cmor) + for lev_dim in lev_dimensions.keys(): + if lev_dim in handler.variables[variable].dimensions: + coords.append(lev_dim) + handler.variables[variable].coordinates = ' '.join(set(coords)) + handler.close() + return lev_dimensions + + def _get_lev_dimensions(self, var_cmor): + if var_cmor.domain == ModelingRealms.ocean: + lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', + 'depth': 'lev'} + elif var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: + lev_dimensions = {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', + 'depth_4': 'sdepth'} + elif var_cmor.domain == ModelingRealms.atmos: + lev_dimensions = {'depth': 'plev'} + else: + lev_dimensions = {} + return lev_dimensions + def _get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') valid_starts = (self.experiment.expid, 'MMA', 'MMASH', 'MMAGG', 'MMO') diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index ecd0003186f900e8a5d8a9b0a9ebf8a606abbd2c..781a199a7d071b03dcf169a2cc4215b39f1b8616 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -138,32 +138,10 @@ class Config(object): self.mask_regions = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS', '') self.mask_regions_3d = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS_3D', '') - data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', - ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), - 'specs', - ignore_case=True) - - if data_convention == 'specs': - self.data_convention = SPECSConvention(data_convention, self) - elif data_convention == 'primavera': - self.data_convention = PrimaveraConvention(data_convention, self) - elif data_convention == 'cmip6': - self.data_convention = CMIP6Convention(data_convention, self) - elif data_convention == 'preface': - self.data_convention = PrefaceConvention(data_convention, self) - elif data_convention == 'meteofrance': - self.data_convention = MeteoFranceConvention(data_convention, self) - - self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) - namelist_file = os.path.join(os.path.dirname(__file__), - 'CDFTOOLS_{0}.namlist'.format(self.data_convention.name)) - Log.debug(namelist_file) - if os.path.isfile(namelist_file): - Log.debug('Setting namelist {0}', namelist_file) - os.environ['NAM_CDF_NAMES'] = namelist_file + self._parse_dataconvention(parser) self.var_manager = VariableManager() - self.var_manager.load_variables(data_convention) + self.var_manager.load_variables(self.data_convention.name) self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') self.skip_diags_done = parser.get_bool_option('DIAGNOSTICS', 'SKIP_DIAGS_DONE', True) self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) @@ -185,6 +163,16 @@ class Config(object): Log.debug('Preparing command list') commands = self._diags.split() self._real_commands = list() + self._apply_aliases(commands) + Log.debug('Command list ready ') + + self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid) + + self.cmor = CMORConfig(parser, self.var_manager) + self.thredds = THREDDSConfig(parser) + self.report = ReportConfig(parser) + + def _apply_aliases(self, commands): for command in commands: command = command.strip() if command.startswith('#'): @@ -196,13 +184,28 @@ class Config(object): self._real_commands.append(add_command) else: self._real_commands.append(command) - Log.debug('Command list ready ') - self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid) - - self.cmor = CMORConfig(parser, self.var_manager) - self.thredds = THREDDSConfig(parser) - self.report = ReportConfig(parser) + def _parse_dataconvention(self, parser): + data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', + ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), + 'specs', + ignore_case=True) + if data_convention == 'specs': + self.data_convention = SPECSConvention(data_convention, self) + elif data_convention == 'primavera': + self.data_convention = PrimaveraConvention(data_convention, self) + elif data_convention == 'cmip6': + self.data_convention = CMIP6Convention(data_convention, self) + elif data_convention == 'preface': + self.data_convention = PrefaceConvention(data_convention, self) + elif data_convention == 'meteofrance': + self.data_convention = MeteoFranceConvention(data_convention, self) + self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) + namelist_file = os.path.join(os.path.dirname(__file__), + 'CDFTOOLS_{0}.namlist'.format(self.data_convention.name)) + Log.debug(namelist_file) + Log.debug('Setting namelist {0}', namelist_file) + os.environ['NAM_CDF_NAMES'] = namelist_file def get_commands(self): """ diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 4207f844da4875e0f7205dfc3f4e07f44ccb5a41..43122007251749b3e181b6091f3051130dbf5f7f 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -1,3 +1,4 @@ +"""Module to manage the different data conventions supported by EarthDiagnostics""" import os import shutil import re @@ -24,6 +25,21 @@ class DataConvention(object): self.lock = threading.Lock() self._checked_vars = list() + def get_scratch_masks(self, scratch_masks): + """ + Get the final scratch_masks path + + Parameters + ---------- + scratch_masks: str + + Returns + ------- + str + + """ + return scratch_masks + def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, grid=None, year=None, date_str=None): """ @@ -262,6 +278,8 @@ class DataConvention(object): self.lock.release() def _get_time_component(self, chunk, date_str, frequency, startdate, year): + if len([x for x in (chunk, date_str, year) if x is not None]) > 1: + raise ValueError('Only one of the parameters chunk, year or date_str may be provided') if chunk is not None: time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency) elif year: @@ -296,16 +314,54 @@ class DataConvention(object): return current_count def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + Raises + ------ + NotImplementedError: + If not implemented by the derived classes + + """ raise NotImplementedError class Cmor2Convention(DataConvention): """Base class for CMOR2-based conventions""" - def get_scratch_masks(self, scratch_masks): - return scratch_masks - def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + """ if cmor_var is None: cmor_table = domain.get_table(frequency, self.config.data_convention) else: @@ -329,6 +385,23 @@ class Cmor2Convention(DataConvention): return folder_path def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ template = 'r{0}i{1}p1' return template.format(member + 1 - self.config.experiment.member_count_start, self.config.cmor.initialization_number) @@ -346,14 +419,29 @@ class Cmor2Convention(DataConvention): for name in os.listdir(os.path.join(path, freq, domain, var, member)): filepath = os.path.join(path, freq, domain, var, member, name) if os.path.isfile(filepath): - self.create_link(domain, filepath, frequency, var, "", False, + self.create_link(ModelingRealms.parse(domain), filepath, frequency, var, "", False, vartype=VariableType.MEAN) else: for filename in os.listdir(filepath): - self.create_link(domain, os.path.join(filepath, filename), frequency, var, - "", False, vartype=VariableType.MEAN) + self.create_link(ModelingRealms.parse(domain), os.path.join(filepath, filename), + frequency, var, "", False, vartype=VariableType.MEAN) def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + """ startdate_path = self.get_startdate_path(startdate) if not os.path.isdir(startdate_path): return False @@ -371,6 +459,18 @@ class SPECSConvention(Cmor2Convention): """Base class for CMOR2-based conventions""" def get_startdate_path(self, startdate): + """ + Return the path to the startdate's CMOR folder + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.experiment.institute, self.config.experiment.model, self.experiment_name(startdate), 'S' + startdate) @@ -386,11 +486,24 @@ class PrefaceConvention(Cmor2Convention): config: Config """ + def __init__(self, name, config): super(PrefaceConvention, self).__init__(name, config) self.time_separator = '_' def get_startdate_path(self, startdate): + """ + Return the path to the startdate's CMOR folder + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.experiment.institute, self.experiment_name(startdate), 'S' + startdate) @@ -406,15 +519,51 @@ class Cmor3Convention(DataConvention): config: Config """ + def __init__(self, name, config): super(Cmor3Convention, self).__init__(name, config) self.lat_name = 'latitude' self.lon_name = 'longitude' def get_scratch_masks(self, scratch_masks): + """ + Get the final scratch_masks path + + Adds a folder matching the convention name to the configured path + + Parameters + ---------- + scratch_masks: str + + Returns + ------- + str + + """ return os.path.join(scratch_masks, self.name) def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + """ if cmor_var is None: cmor_table = domain.get_table(frequency, self.config.data_convention) else: @@ -497,11 +646,38 @@ class Cmor3Convention(DataConvention): False, vartype=VariableType.MEAN) def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + """ template = 'r{0}i{1}p1f1' return template.format(member + 1 - self.config.experiment.member_count_start, self.config.cmor.initialization_number) def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + """ startdate_path = self.get_startdate_path(startdate) if not os.path.isdir(startdate_path): return False @@ -522,11 +698,13 @@ class Cmor3Convention(DataConvention): class CMIP6Convention(Cmor3Convention): """Class managing CMIP6 file conventions""" + pass class PrimaveraConvention(Cmor3Convention): """Class managing Primavera file conventions""" + pass @@ -534,6 +712,27 @@ class MeteoFranceConvention(DataConvention): """Class managing MeteoFrance file conventions""" def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + """ if year is not None: raise ValueError('Year not supported with MeteoFrance convention') if date_str is not None: @@ -551,6 +750,18 @@ class MeteoFranceConvention(DataConvention): return folder_path def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + """ return '{0:02d}'.format(member) def _get_chunk_time_bounds(self, startdate, chunk, frequency): @@ -561,10 +772,57 @@ class MeteoFranceConvention(DataConvention): return time_bound def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + """ + Create file link + + In this convention, it does nothing + + Parameters + ---------- + domain: ModelingRealm + filepath: str + frequency: Frequency + var: str + grid: str + move_old: bool + vartype: VariableType + + """ pass def create_links(self, startdate, member=None): + """ + Create links for a given startdate or member + + In this convention, it does nothing + + Parameters + ---------- + startdate: str + member: int or None + + """ pass def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + Raises + ------ + NotImplementedError: + If not implemented by the derived classes + + """ return True diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 852dab7a722fb66a658864eaad5d951914487cce..07981fc68cfe964b72bed1538942807b2a9d3730 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -660,16 +660,7 @@ class NetCDFFile(DataFile): if self.region: try: cubes = iris.load(self.remote_file) - for cube in cubes: - try: - if isinstance(self.region, six.string_types): - regions = {self.region.name} - else: - regions = {basin.name for basin in self.region} - if regions.issubset(set(cube.coord('region').points)): - self.storage_status = StorageStatus.READY - except iris.exceptions.CoordinateNotFoundError: - pass + self._check_regions(cubes) except iris.exceptions.TranslationError as ex: # If the check goes wrong, we must execute everything os.remove(self.remote_file) @@ -678,6 +669,18 @@ class NetCDFFile(DataFile): else: self.storage_status = StorageStatus.READY + def _check_regions(self, cubes): + for cube in cubes: + try: + if isinstance(self.region, six.string_types): + regions = {self.region.name} + else: + regions = {basin.name for basin in self.region} + if regions.issubset(set(cube.coord('region').points)): + self.storage_status = StorageStatus.READY + except iris.exceptions.CoordinateNotFoundError: + pass + def create_link(self): """Create a link from the original data in the _ folder""" try: diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index 602fc5cd72a1a33c07cca0698704670c35392d70..80f013907c7d805686f31fbeb764782deaa77690 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -105,7 +105,7 @@ class VerticalMeanMetersIris(Diagnostic): var_cube = iris.load_cube(self.variable_file.local_file) - lev_names = ('lev', 'depth') + lev_names = ('lev', 'depth', 'air_pressure') coord = None for coord_name in lev_names: try: @@ -122,7 +122,7 @@ class VerticalMeanMetersIris(Diagnostic): lev_max = coord.points[-1] else: lev_max = self.box.max_depth - lev_constraint = iris.Constraint(coord_values={coord.var_name: lambda cell: lev_min <= cell <= lev_max}) + lev_constraint = iris.Constraint(coord_values={coord.name(): lambda cell: lev_min <= cell <= lev_max}) var_cube = var_cube.extract(lev_constraint) var_cube = var_cube.collapsed(coord, iris.analysis.MEAN) temp = TempFile.get() diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 6c5d33855790389db00fc6c8fbe0c120ad16d795..c1df586be11cb9b4e14bccff64e36dce22c6d8d3 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -3,6 +3,8 @@ import os import six +import numpy as np + import iris import iris.analysis import iris.coords @@ -135,9 +137,9 @@ class Siasiesiv(Diagnostic): sic = iris.load_cube(self.sic.local_file) if sic.units.origin == '%' and sic.data.max() < 2: sic.units = '1.0' - sic.convert_units('1.0') + + extent = sic.copy((sic.data >= 0.15).astype(np.int8)) * Siasiesiv.area.data sic *= Siasiesiv.area.data - extent = sic.data > 0.15 if not self.omit_volume: handler = Utils.open_cdf(self.sit.local_file) @@ -154,9 +156,8 @@ class Siasiesiv(Diagnostic): self.results['sivoln'][basin] = self.sum(volume, mask, north=True) self.results['sivols'][basin] = self.sum(volume, mask, north=False) - extent_mask = mask * extent - self.results['siextentn'][basin] = self.sum(sic, extent_mask, north=True) - self.results['siextents'][basin] = self.sum(sic, extent_mask, north=False) + self.results['siextentn'][basin] = self.sum(extent, mask, north=True) + self.results['siextents'][basin] = self.sum(extent, mask, north=False) self.save() diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 9f343a6a65e1851774727fc77f7fb3b471d3a5f4..2e12e4662d9474f49c07f8c789fbd6d5f8ca718e 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -611,6 +611,22 @@ class Utils(object): if new_name in destiny.variables.keys(): return + translated_dimensions = Utils._copy_dimensions(add_dimensions, destiny, must_exist, new_names, rename_dimension, + source, variable) + if new_name in destiny.variables.keys(): + # Just in case the variable we are copying match a dimension name + return + original_var = source.variables[variable] + new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) + Utils.copy_attributes(new_var, original_var) + if hasattr(new_var, 'coordinates'): + coords = [new_names[coord] if coord in new_names else coord for coord in new_var.coordinates.split(' ')] + new_var.coordinates = Utils.convert_to_ascii_if_possible(' '.join(coords)) + + new_var[:] = original_var[:] + + @staticmethod + def _copy_dimensions(add_dimensions, destiny, must_exist, new_names, rename_dimension, source, variable): if rename_dimension: translated_dimensions = Utils._translate(source.variables[variable].dimensions, new_names) else: @@ -621,17 +637,7 @@ class Utils(object): '{1} {2}'.format(variable, translated_dimensions, destiny.dimensions)) for dimension in source.variables[variable].dimensions: Utils.copy_dimension(source, destiny, dimension, must_exist, new_names, rename_dimension) - if new_name in destiny.variables.keys(): - # Just in case the variable we are copying match a dimension name - return - original_var = source.variables[variable] - new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) - Utils.copy_attributes(new_var, original_var) - if hasattr(new_var, 'coordinates'): - coords = [new_names[coord] if coord in new_names else coord for coord in new_var.coordinates.split(' ')] - new_var.coordinates = Utils.convert_to_ascii_if_possible(' '.join(coords)) - - new_var[:] = original_var[:] + return translated_dimensions @staticmethod def copy_attributes(new_var, original_var, omitted_attributtes=None): diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 597c274775dfc2c32910b45da1b5c45cb144d38c..705aea3b566a7d478c3138a4cc365958f7712c1d 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -167,7 +167,9 @@ class VariableManager(object): if 'variable_entry' in data: Log.debug('Parsing file {0}'.format(json_path)) table_id = data['Header']['table_id'][6:] - table = CMORTable(table_id, Frequency(data['Header']['frequency']), data['Header']['table_date']) + table = CMORTable(table_id, + Frequency(data['variable_entry'].values()[0]['frequency']), + data['Header']['table_date']) self.tables[table_id] = table self._load_json_variables(data['variable_entry'], table) diff --git a/environment.yml b/environment.yml index 42dcdbbe63415a11d105b89a57ebd06736da32de..c64b1ce5a52dfc80936e9fb0859de1b35b9baf1f 100644 --- a/environment.yml +++ b/environment.yml @@ -31,3 +31,4 @@ dependencies: - exrex - xxhash - pytest-profiling + - dummydata diff --git a/test/unit/data_convention/test_data_convention.py b/test/unit/data_convention/test_data_convention.py index 4ce42ac0b5a802edf6b8fbf223a2c902193072d9..152198eceb3a22d2090d2c844be1c9e865b739ab 100644 --- a/test/unit/data_convention/test_data_convention.py +++ b/test/unit/data_convention/test_data_convention.py @@ -17,6 +17,7 @@ class TestDataConvention(TestCase): os.mkdir(os.path.join(self.tmp_dir, 'expid')) self.config = Mock() + self.config.frequency = 'mon' self.config.experiment.experiment_name = 'expname' self.config.data_dir = self.tmp_dir self.config.experiment.expid = 'expid' @@ -61,28 +62,35 @@ class TestDataConvention(TestCase): with self.assertRaises(NotImplementedError): self.convention.get_member_str(None) - def test_get_file_path_raise_incompatible_date_info(self): + def test_is_cmorized(self): + with self.assertRaises(NotImplementedError): + self.convention.is_cmorized(None, None, None, None) + + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_cmor_folder_path', autospec=True) + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_file_name', autospec=True) + def test_get_file_path(self, mock_filename, mock_cmor_path): + mock_cmor_path.return_value = 'path' + mock_filename.return_value = 'filename' cmor_var = Mock() omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.frequency = 'monthly' - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' - - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, None, frequency, year='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, 1, frequency, year='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, 1, frequency, date_str='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, None, frequency, year='1998', date_str='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, 1, frequency, year='1998', date_str='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, None, frequency) + + filepath = self.convention.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 1, 'mon') + self.assertEqual('path/filename', filepath) + + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_cmor_folder_path', autospec=True) + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_file_name', autospec=True) + def test_get_file_path_no_freq(self, mock_filename, mock_cmor_path): + mock_cmor_path.return_value = 'path' + mock_filename.return_value = 'filename' + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + + filepath = self.convention.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 1, None) + self.assertEqual('path/filename', filepath) def test_create_link(self): file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) diff --git a/test/unit/data_convention/test_primavera.py b/test/unit/data_convention/test_primavera.py index 271f8db2b2b98a7e2193af6f3f45928095a33a9c..2eb75af909e805fbb78ad127c20b3dcda064fd43 100644 --- a/test/unit/data_convention/test_primavera.py +++ b/test/unit/data_convention/test_primavera.py @@ -63,6 +63,13 @@ class TestPrimaveraConvention(TestCase): os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' 'r2i1p1f1/Omon/var/ocean_grid/version')) + def test_get_cmor_folder_path_no_cmor_var(self): + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' + 'r2i1p1f1/Omon/var/ocean_grid/version')) + def test_get_cmor_folder_path_atmos(self): cmor_var = Mock() omon = Mock() @@ -115,6 +122,16 @@ class TestPrimaveraConvention(TestCase): self.assertEqual(file_path, 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_no_cmor_var(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', None, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_daily(self): cmor_var = Mock() omon = Mock() diff --git a/test/unit/data_convention/test_specs.py b/test/unit/data_convention/test_specs.py index 82e7e982ed722f21cb9fca23ccdb309bb0f019ac..8e054d10779a7816662f2e4e1d4bfe88177baebb 100644 --- a/test/unit/data_convention/test_specs.py +++ b/test/unit/data_convention/test_specs.py @@ -116,6 +116,12 @@ class TestSpecsConvention(TestCase): self.assertEqual(file_path, 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + def test_get_filename_no_cmor_var(self): + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', None, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + def test_get_filename_daily(self): cmor_var = Mock() omon = Mock() @@ -177,6 +183,26 @@ class TestSpecsConvention(TestCase): self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, Frequencies.monthly, None, None, None, None) + def test_get_file_path_raise_incompatible_date_info(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = 'mon' + + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency, year='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, year='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency) + @mock.patch('os.path.isfile') def test_is_cmorized(self, mock_is_file): mock_is_file.return_value = True @@ -205,6 +231,19 @@ class TestSpecsConvention(TestCase): 'r2i1p1/version')) self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + @mock.patch('os.path.isfile') + def test_is_cmorized_no_startdate_path(self, mock_is_file): + mock_is_file.return_value = False + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + def test_is_cmorized_false_not_member_folder(self): cmor_var = Mock() omon = Mock() diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py index 426e893116d07d26deee340e3a8a3ae40dca5e0f..741779a4f1f2e689caf4ee3a147c2425839dae4d 100644 --- a/test/unit/general/test_dailymean.py +++ b/test/unit/general/test_dailymean.py @@ -1,9 +1,11 @@ # coding=utf-8 from unittest import TestCase - +from tempfile import mktemp from mock import Mock, patch +import os + +import dummydata -from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.timemean import DailyMean @@ -13,15 +15,18 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestDailyMean(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() self.diags.model_version = 'model_version' self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 + self.daymean = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -50,7 +55,15 @@ class TestDailyMean(TestCase): DailyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - mixed = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') - self.assertEqual(str(mixed), + self.assertEqual(str(self.daymean), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Original frequency: freq Grid: ') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, start_year=2000, stop_year=2000, method='constant', constant=1) + + self.daymean.variable_file = Mock() + self.daymean.variable_file.local_file = self.var_file + self.daymean.mean_file = Mock() + self.daymean.compute() + self.daymean.mean_file.set_local_file.assert_called_once() diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py index 5672c4622025ca1e28523898a7473a49b877984f..0ee4dee8481922c79cbe4e5619ceebd63a21d17b 100644 --- a/test/unit/general/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -1,9 +1,11 @@ # coding=utf-8 +import os from unittest import TestCase +from tempfile import mktemp from mock import Mock, patch +import dummydata.model3 -from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.timemean import MonthlyMean @@ -13,17 +15,18 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestMonthlyMean(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() self.diags.model_version = 'model_version' self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 + self.monmean = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'ta', 'freq', '') - self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -51,6 +54,15 @@ class TestMonthlyMean(TestCase): MonthlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), + self.assertEqual(str(self.monmean), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + 'Variable: atmos:ta Original frequency: freq Grid: ') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, start_year=2000, stop_year=2000, method='constant', constant=1) + + self.monmean.variable_file = Mock() + self.monmean.variable_file.local_file = self.var_file + self.monmean.mean_file = Mock() + self.monmean.compute() + self.monmean.mean_file.set_local_file.assert_called_once() diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py index 49f6a1b5e71c9c8075567ffa91eed4012cf2ff0d..99dd57664405cdaedfe43c6aa113cea975281729 100644 --- a/test/unit/general/test_verticalmeanmetersiris.py +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -1,6 +1,9 @@ # coding=utf-8 from unittest import TestCase +import os +from tempfile import mktemp +import dummydata from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError from earthdiagnostics.box import Box from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris @@ -13,6 +16,7 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestVerticalMeanMetersIris(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() @@ -24,6 +28,10 @@ class TestVerticalMeanMetersIris(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) + def fake_parse(self, value): if not value: raise DiagnosticOptionError @@ -74,3 +82,18 @@ class TestVerticalMeanMetersIris(TestCase): self.assertEqual(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Box: 0-100m') + + def test_compute(self): + box = Box() + box.min_depth = 1 + box.max_depth = 50000 + + dummydata.model3.Model3(oname=self.var_file, var='ta', start_year=2000, stop_year=2000, method='constant', + constant=1) + + diag = VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', box) + + diag.variable_file = Mock() + diag.variable_file.local_file = self.var_file + diag.results = Mock() + diag.compute() diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py index 5c0d7c037633e8d239d7bab2204d9151d114979b..a37b70aad1715ba1b1f567f837c9e8cc9459706f 100644 --- a/test/unit/general/test_yearlymean.py +++ b/test/unit/general/test_yearlymean.py @@ -1,9 +1,10 @@ # coding=utf-8 from unittest import TestCase - +from tempfile import mktemp from mock import Mock, patch +import os +import dummydata -from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.timemean import YearlyMean @@ -13,17 +14,18 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestYearlyMean(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.diags.config.frequency = Frequencies.monthly - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 + self.yearly_mean = YearlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') - self.mixed = YearlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -51,6 +53,15 @@ class TestYearlyMean(TestCase): YearlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), + self.assertEqual(str(self.yearly_mean), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Original frequency: freq Grid: ') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, start_year=2000, stop_year=2000, method='constant', constant=1) + + self.yearly_mean.variable_file = Mock() + self.yearly_mean.variable_file.local_file = self.var_file + self.yearly_mean.mean_file = Mock() + self.yearly_mean.compute() + self.yearly_mean.mean_file.set_local_file.assert_called_once() diff --git a/test/unit/test_config.py b/test/unit/test_config.py index b588881a1ba89e0e7f518523f157a74f2140e716..4a0e51a7948459866a98651870e0595f2f390140 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -7,7 +7,8 @@ import os from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig, Config from earthdiagnostics.frequency import Frequencies from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.data_convention import SPECSConvention, PrimaveraConvention +from earthdiagnostics.data_convention import SPECSConvention, PrimaveraConvention, PrefaceConvention, \ + MeteoFranceConvention, CMIP6Convention class VariableMock(object): @@ -507,6 +508,11 @@ class TestConfig(TestCase): self.assertEqual(config.con_files, '') self.assertEqual(config.data_adaptor, 'CMOR') self.assertEqual(config.get_commands(), (['diag1', 'diag2'])) + self.assertIsInstance(config.data_convention, SPECSConvention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_specs.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) def test_alias(self): config = Config() @@ -531,3 +537,33 @@ class TestConfig(TestCase): namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'earthdiagnostics/CDFTOOLS_primavera.namlist')) self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) + + def test_data_convention_cmip6(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'cmip6') + self._parse(config) + self.assertIsInstance(config.data_convention, CMIP6Convention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks/cmip6') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_cmip6.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) + + def test_data_convention_meteofrance(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'meteofrance') + self._parse(config) + self.assertIsInstance(config.data_convention, MeteoFranceConvention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_meteofrance.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) + + def test_data_convention_preface(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'preface') + self._parse(config) + self.assertIsInstance(config.data_convention, PrefaceConvention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_preface.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index a3d7983e5c2a950b7cac038074b7a2cc68adf5af..1a5b96fbcf8b8a1aa1135ac2f643d2c4cf372378 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -1,5 +1,6 @@ # coding=utf-8 from unittest import TestCase +from mock import Mock from earthdiagnostics.constants import Basin, Basins @@ -16,9 +17,36 @@ class TestBasin(TestCase): self.assertTrue(Basin('Basin') == self.basin) self.assertFalse(Basin('bas') == self.basin) + def test__neq__(self): + self.assertFalse(Basin('Basin') != self.basin) + self.assertTrue(Basin('bas') != self.basin) + + def test__greater(self): + self.assertFalse(Basin('Aasin') > self.basin) + self.assertFalse(Basin('Basin') > self.basin) + self.assertTrue(Basin('Casin') > self.basin) + + def test__lower(self): + self.assertTrue(Basin('Aasin') < self.basin) + self.assertFalse(Basin('Basin') < self.basin) + self.assertFalse(Basin('Casin') < self.basin) + + def test_ge(self): + self.assertFalse(Basin('Aasin') >= self.basin) + self.assertTrue(Basin('Basin') >= self.basin) + self.assertTrue(Basin('Casin') >= self.basin) + + def test__le(self): + self.assertTrue(Basin('Aasin') <= self.basin) + self.assertTrue(Basin('Basin') <= self.basin) + self.assertFalse(Basin('Casin') <= self.basin) + def test__str__(self): self.assertEqual(str(self.basin), 'Basin') + def test__repr__(self): + self.assertEqual(repr(self.basin), 'Basin') + def test_order(self): self.assertTrue(self.basin < Basin('Vasin')) self.assertTrue(self.basin > Basin('Asin')) @@ -28,3 +56,18 @@ class TestBasins(TestCase): def test_singleton(self): self.assertIs(Basins(), Basins()) + + def test_get_basins(self): + mock_handler = Mock() + mock_handler.variables.keys.return_value = ('var1', 'lat') + Basins().get_available_basins(mock_handler) + self.assertTrue(hasattr(Basins(), 'var1')) + self.assertFalse(hasattr(Basins(), 'lat')) + + def test_get_basins_with_alias(self): + mock_handler = Mock() + mock_handler.variables.keys.return_value = ('Kara_Sea', 'lat') + basins = Basins() + Basins().get_available_basins(mock_handler) + self.assertTrue(hasattr(basins, 'Kara_Sea')) + self.assertFalse(hasattr(basins, 'lat')) diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index 2ee054807a3df7094d43e33eb9e5fb3bb7a72414..f425083b93010990a3dd1a510f2acf609988ebfc 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -367,6 +367,32 @@ class TestDiagnostic(TestCase): def test_empty_process_options(self): self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) + def test_diagnostic_can_be_skipped(self): + diag = self._get_diag_for_skipping() + self.assertTrue(diag.can_skip_run()) + + def _get_diag_for_skipping(self, modified=False, status=StorageStatus.READY): + mock = Mock() + datafile_mock = Mock() + datafile_mock.storage_status = status + datafile_mock.has_modifiers.return_value = modified + mock.declare_chunk.return_value = datafile_mock + diag = Diagnostic(mock) + diag.declare_chunk(None, None, None, None, None) + return diag + + def test_diagnostic_can_not_be_skipped_do_not_generate_files(self): + diag = Diagnostic(None) + self.assertFalse(diag.can_skip_run()) + + def test_diagnostic_can_not_be_skipped_modified(self): + diag = self._get_diag_for_skipping(modified=True) + self.assertFalse(diag.can_skip_run()) + + def test_diagnostic_can_not_be_skipped_data_not_ready(self): + diag = self._get_diag_for_skipping(status=StorageStatus.PENDING) + self.assertFalse(diag.can_skip_run()) + class TestDiagnosticBasinOption(TestCase): diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 99c74fdf9f031d0eac0731dee11f7368724f7527..8340ffa7237f1c5905fab9f06db00fa19ce6432b 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -12,6 +12,7 @@ class TestModellingRealms(TestCase): self.assertEqual(ModelingRealms.parse('atmos'), ModelingRealms.atmos) self.assertEqual(ModelingRealms.parse('atmoschem'), ModelingRealms.atmosChem) self.assertEqual(ModelingRealms.parse('atmoSChem'), ModelingRealms.atmosChem) + self.assertIsNone(ModelingRealms.parse('')) with self.assertRaises(ValueError): ModelingRealms.parse('badrealm') diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index b5072f42a2fe02d5a838830f44af8a1b50358bff..f6c86e2a70fdf20a3626f9e977737b3562303b4d 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -223,15 +223,19 @@ class TestVariableManager(TestCase): def test_load_primavera(self): self.var_manager.load_variables('primavera') + self.assertTrue(self.var_manager.get_all_variables()) def test_load_cmip6(self): self.var_manager.load_variables('cmip6') + self.assertTrue(self.var_manager.get_all_variables()) def test_load_specs(self): self.var_manager.load_variables('specs') + self.assertTrue(self.var_manager.get_all_variables()) def test_load_preface(self): self.var_manager.load_variables('preface') + self.assertTrue(self.var_manager.get_all_variables()) def test_bad_load(self): with self.assertRaises(Exception):