diff --git a/VERSION b/VERSION index 6f9b87cf6358b66ba00f4c1254c2f407fc509414..78238d724c42dbffea754dea8ebb9dffe6b614fc 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc6 +3.0.0rc7 diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 9f7a9e1096417d2b932fa6bae1c482ecf947dc17..efdadcfec05dea121913251ed6c6cbd99502c28d 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -705,13 +705,13 @@ class Cmorizer(object): handler.physics_description = cmor.physics_description handler.model_id = experiment.model handler.associated_model = cmor.associated_model - handler.project_id = self.config.data_convention.upper() + handler.project_id = self.config.data_convention.name.upper() handler.realization = str(self.member + 1) handler.source = cmor.source handler.startdate = 'S{0}'.format(self.startdate) handler.tracking_id = str(uuid.uuid1()) handler.title = "{0} model output prepared for {2} {1}".format(experiment.model, experiment.experiment_name, - self.config.data_convention.upper()) + self.config.data_convention.name.upper()) def _gribfiles_available(self): grb_path = os.path.join(self.original_files_path, '*.grb') diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index e4679e7469245902b05742e9accb822d39357cec..f2df52f7272311332090df9826814880daa1c979 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -23,6 +23,18 @@ class Basin(object): def __ne__(self, other): return not self == other + def __gt__(self, other): + return self.name > other.name + + def __lt__(self, other): + return self.name < other.name + + def __ge__(self, other): + return self.name >= other.name + + def __le__(self, other): + return self.name <= other.name + def __str__(self): return self._name diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index e5fdb82a50543037fae0a979f9e1a2042342bcb6..852dab7a722fb66a658864eaad5d951914487cce 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -681,8 +681,8 @@ class NetCDFFile(DataFile): def create_link(self): """Create a link from the original data in the _ folder""" try: - self.data_manager.create_link(self.domain, self.remote_file, self.frequency, self.final_name, - self.grid, True, self.var_type) + self.data_convention.create_link(self.domain, self.remote_file, self.frequency, self.final_name, + self.grid, True, self.var_type) except (ValueError, Exception) as ex: Log.error('Can not create link to {1}: {0}'.format(ex, self.remote_file)) diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index 7072935803a83182926a5c532d089c487cdf214b..7c9b607c4dd7d579af467c627fa3f16f0d4b2e38 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -76,7 +76,7 @@ class ModelingRealm(object): """ if self.name == 'seaIce': - if data_convention in ('specs', 'preface'): + if data_convention.name in ('specs', 'preface'): prefix = 'OI' else: prefix = 'SI' @@ -88,8 +88,8 @@ class ModelingRealm(object): if frequency == Frequencies.six_hourly: table_name = '6hrPlev' else: - if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention not in ('specs', - 'preface'): + if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention.name not in ('specs', + 'preface'): table_name = prefix + str(frequency) else: table_name = frequency.frequency diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 703301ebc9aba1744bbc43b2c29381acd1bebab0..99979bb2f20c5ce9c577383eed4a639d548c321e 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -79,12 +79,14 @@ class Siasiesiv(Diagnostic): DiagnosticBoolOption('omit_volume', False)) options = cls.process_options(options, options_available) - if not options['basins']: + basins = options['basins'] + if not basins: Log.error('Basins not recognized') return () masks = {} - for basin in options['basins']: + basins.sort() + for basin in basins: masks[basin] = Utils.get_mask(basin) job_list = list() @@ -121,12 +123,12 @@ class Siasiesiv(Diagnostic): def _declare_var(self, var_name): self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, - self.startdate, self.member, self.chunk, - region=self.masks.keys()) + self.startdate, self.member, self.chunk) def compute(self): """Run the diagnostic""" - coordinates = ' '.join(('time', 'leadtime', 'time_centered', 'latitude', 'longitude')) + coordinates = ' '.join(('time', 'leadtime', 'time_centered', + self.data_convention.lon_name, self.data_convention.lat_name)) handler = Utils.open_cdf(self.sic.local_file) handler.variables[self.sic_varname].coordinates = coordinates handler.close() @@ -179,9 +181,9 @@ class Siasiesiv(Diagnostic): results.append(result) if not results: continue - self._save_file(results.merge_cube(), self.generated[var], basins) + self._save_file(results.merge_cube(), self.generated[var]) - def _save_file(self, data, generated_file, basins): + def _save_file(self, data, generated_file): temp = TempFile.get() region = data.coord('region').points data.remove_coord('region') @@ -192,5 +194,4 @@ class Siasiesiv(Diagnostic): var[...] = region handler.close() Utils.rename_variable(temp, 'region2', 'region', True) - - generated_file.set_local_file(temp, region=basins.keys()) + generated_file.set_local_file(temp) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 25e1cc4abe644b01c6e2ce6dff624b1111a4ea0b..a6aff46cf23f7398b5a7b1d9e90fd707a934878a 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -1,5 +1,6 @@ # coding=utf-8 """Earthdiagnostics workflow manager""" +import time import datetime import operator import sys @@ -144,9 +145,9 @@ class WorkManager(object): else: step = 0.5 for _ in range(int(interval / step)): - if lock.acquire(blocking=False): + if lock.acquire(False): return - sys.sleep(step) + time.sleep(step) Log.info('Current status:') Log.info('===============') diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index e7b0d1291b65a8977fd542aa2e900f4fdb7dfae7..abe27695a77bc23c5fc31fe4a8df778c9290eb42 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -1,9 +1,10 @@ from earthdiagnostics.cmorizer import Cmorizer from earthdiagnostics.utils import TempFile, Utils +from earthdiagnostics.data_convention import DataConvention from bscearth.utils import log from unittest import TestCase -from mock import Mock +from mock import Mock, create_autospec import os import tempfile import shutil @@ -49,7 +50,8 @@ class TestCmorizer(TestCase): self.data_manager.config.data_dir = os.path.join(self.tmp_dir, 'data') self.data_manager.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') TempFile.scratch_folder = self.data_manager.config.scratch_dir - self.data_manager.config.data_convention = 'data_convention' + self.data_manager.config.data_convention = create_autospec(DataConvention) + self.data_manager.config.data_convention.name = 'data_convention' self.data_manager.config.var_manager.get_variable_and_alias = self._get_variable_and_alias self.data_manager.config.var_manager.get_variable = self._get_variable diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index 943a2b3e722674948e22275ee23a11e612863545..a3d7983e5c2a950b7cac038074b7a2cc68adf5af 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -19,6 +19,10 @@ class TestBasin(TestCase): def test__str__(self): self.assertEqual(str(self.basin), 'Basin') + def test_order(self): + self.assertTrue(self.basin < Basin('Vasin')) + self.assertTrue(self.basin > Basin('Asin')) + class TestBasins(TestCase): diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 82baa52929fb213b6a0c530d9c31aa9414d30af7..99c74fdf9f031d0eac0731dee11f7368724f7527 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -20,6 +20,8 @@ class TestModellingRealm(TestCase): def setUp(self): self.realm = ModelingRealm('ocean') + self.data_convention = Mock() + self.data_convention.name = 'specs' def test__repr__(self): self.assertEqual(str(self.realm), repr(self.realm)) @@ -37,19 +39,20 @@ class TestModellingRealm(TestCase): self.assertEqual(self.realm.get_table_name(Frequencies.monthly, 'specs'), 'Omon') def test_get_oimon(self): - self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'specs'), 'OImon') + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, self.data_convention), 'OImon') def test_get_simon(self): - self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'cmip6'), 'SImon') + self.data_convention.name = 'cmip6' + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, self.data_convention), 'SImon') def test_get_limon(self): - self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, 'specs'), 'LImon') + self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, self.data_convention), 'LImon') def test_get_day(self): - self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, 'specs'), 'day') + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, self.data_convention), 'day') def test_get_6hrplev(self): - self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev') + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, self.data_convention), '6hrPlev') def test_get_varfolder(self): self.assertEqual(ModelingRealm('ocean').get_varfolder('var', 6, 3), 'var_f6h') diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 6fd11e7b85714b7b18dbe3d76663ad4efac66f3b..b5072f42a2fe02d5a838830f44af8a1b50358bff 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -184,27 +184,33 @@ class TestVariable(TestCase): self.assertEqual(table.date, 'December 2013') def test_get_table_added(self): + convention = Mock() + convention.name = 'specs' var = Variable() var.domain = ModelingRealms.atmos var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) - table = var.get_table(Frequencies.monthly, 'specs') + table = var.get_table(Frequencies.monthly, convention) self.assertEqual(table.frequency, Frequencies.monthly) self.assertEqual(table.name, 'Amon') self.assertEqual(table.date, 'December 2013') def test_get_table_not_added(self): + convention = Mock() + convention.name = 'specs' var = Variable() var.domain = ModelingRealms.atmos var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) - table = var.get_table(Frequencies.daily, 'specs') + table = var.get_table(Frequencies.daily, convention) self.assertEqual(table.frequency, Frequencies.daily) self.assertEqual(table.name, 'day') self.assertEqual(table.date, 'December 2013') def test_get_table_not_matching(self): + convention = Mock() + convention.name = 'specs' var = Variable() with self.assertRaises(ValueError): - var.get_table(Frequencies.daily, 'specs') + var.get_table(Frequencies.daily, convention) class TestVariableManager(TestCase):