From e647b9a195c635b75e651c694f076e150e0755c1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 13:58:23 +0200 Subject: [PATCH 1/6] Removed region from siasiesiv and fixed some bugs --- earthdiagnostics/constants.py | 12 ++++++++++++ earthdiagnostics/datafile.py | 4 ++-- earthdiagnostics/modelingrealm.py | 6 +++--- earthdiagnostics/ocean/siasiesiv.py | 19 ++++++++++--------- earthdiagnostics/work_manager.py | 5 +++-- test/unit/test_constants.py | 4 ++++ 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index e4679e74..f2df52f7 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -23,6 +23,18 @@ class Basin(object): def __ne__(self, other): return not self == other + def __gt__(self, other): + return self.name > other.name + + def __lt__(self, other): + return self.name < other.name + + def __ge__(self, other): + return self.name >= other.name + + def __le__(self, other): + return self.name <= other.name + def __str__(self): return self._name diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index e5fdb82a..852dab7a 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -681,8 +681,8 @@ class NetCDFFile(DataFile): def create_link(self): """Create a link from the original data in the _ folder""" try: - self.data_manager.create_link(self.domain, self.remote_file, self.frequency, self.final_name, - self.grid, True, self.var_type) + self.data_convention.create_link(self.domain, self.remote_file, self.frequency, self.final_name, + self.grid, True, self.var_type) except (ValueError, Exception) as ex: Log.error('Can not create link to {1}: {0}'.format(ex, self.remote_file)) diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index 70729358..7c9b607c 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -76,7 +76,7 @@ class ModelingRealm(object): """ if self.name == 'seaIce': - if data_convention in ('specs', 'preface'): + if data_convention.name in ('specs', 'preface'): prefix = 'OI' else: prefix = 'SI' @@ -88,8 +88,8 @@ class ModelingRealm(object): if frequency == Frequencies.six_hourly: table_name = '6hrPlev' else: - if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention not in ('specs', - 'preface'): + if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention.name not in ('specs', + 'preface'): table_name = prefix + str(frequency) else: table_name = frequency.frequency diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 703301eb..99979bb2 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -79,12 +79,14 @@ class Siasiesiv(Diagnostic): DiagnosticBoolOption('omit_volume', False)) options = cls.process_options(options, options_available) - if not options['basins']: + basins = options['basins'] + if not basins: Log.error('Basins not recognized') return () masks = {} - for basin in options['basins']: + basins.sort() + for basin in basins: masks[basin] = Utils.get_mask(basin) job_list = list() @@ -121,12 +123,12 @@ class Siasiesiv(Diagnostic): def _declare_var(self, var_name): self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, - self.startdate, self.member, self.chunk, - region=self.masks.keys()) + self.startdate, self.member, self.chunk) def compute(self): """Run the diagnostic""" - coordinates = ' '.join(('time', 'leadtime', 'time_centered', 'latitude', 'longitude')) + coordinates = ' '.join(('time', 'leadtime', 'time_centered', + self.data_convention.lon_name, self.data_convention.lat_name)) handler = Utils.open_cdf(self.sic.local_file) handler.variables[self.sic_varname].coordinates = coordinates handler.close() @@ -179,9 +181,9 @@ class Siasiesiv(Diagnostic): results.append(result) if not results: continue - self._save_file(results.merge_cube(), self.generated[var], basins) + self._save_file(results.merge_cube(), self.generated[var]) - def _save_file(self, data, generated_file, basins): + def _save_file(self, data, generated_file): temp = TempFile.get() region = data.coord('region').points data.remove_coord('region') @@ -192,5 +194,4 @@ class Siasiesiv(Diagnostic): var[...] = region handler.close() Utils.rename_variable(temp, 'region2', 'region', True) - - generated_file.set_local_file(temp, region=basins.keys()) + generated_file.set_local_file(temp) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 25e1cc4a..a6aff46c 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -1,5 +1,6 @@ # coding=utf-8 """Earthdiagnostics workflow manager""" +import time import datetime import operator import sys @@ -144,9 +145,9 @@ class WorkManager(object): else: step = 0.5 for _ in range(int(interval / step)): - if lock.acquire(blocking=False): + if lock.acquire(False): return - sys.sleep(step) + time.sleep(step) Log.info('Current status:') Log.info('===============') diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index 943a2b3e..a3d7983e 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -19,6 +19,10 @@ class TestBasin(TestCase): def test__str__(self): self.assertEqual(str(self.basin), 'Basin') + def test_order(self): + self.assertTrue(self.basin < Basin('Vasin')) + self.assertTrue(self.basin > Basin('Asin')) + class TestBasins(TestCase): -- GitLab From 90b6b05b7337644bb34846ce0788091302c5ccd4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 14:57:10 +0200 Subject: [PATCH 2/6] Fixed tests --- test/unit/test_modelling_realm.py | 13 ++++++++----- test/unit/test_variable.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 82baa529..99c74fdf 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -20,6 +20,8 @@ class TestModellingRealm(TestCase): def setUp(self): self.realm = ModelingRealm('ocean') + self.data_convention = Mock() + self.data_convention.name = 'specs' def test__repr__(self): self.assertEqual(str(self.realm), repr(self.realm)) @@ -37,19 +39,20 @@ class TestModellingRealm(TestCase): self.assertEqual(self.realm.get_table_name(Frequencies.monthly, 'specs'), 'Omon') def test_get_oimon(self): - self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'specs'), 'OImon') + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, self.data_convention), 'OImon') def test_get_simon(self): - self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'cmip6'), 'SImon') + self.data_convention.name = 'cmip6' + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, self.data_convention), 'SImon') def test_get_limon(self): - self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, 'specs'), 'LImon') + self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, self.data_convention), 'LImon') def test_get_day(self): - self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, 'specs'), 'day') + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, self.data_convention), 'day') def test_get_6hrplev(self): - self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev') + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, self.data_convention), '6hrPlev') def test_get_varfolder(self): self.assertEqual(ModelingRealm('ocean').get_varfolder('var', 6, 3), 'var_f6h') diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 6fd11e7b..7c6b9c0a 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -184,27 +184,33 @@ class TestVariable(TestCase): self.assertEqual(table.date, 'December 2013') def test_get_table_added(self): + convention= Mock() + convention.name = 'specs' var = Variable() var.domain = ModelingRealms.atmos var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) - table = var.get_table(Frequencies.monthly, 'specs') + table = var.get_table(Frequencies.monthly, convention) self.assertEqual(table.frequency, Frequencies.monthly) self.assertEqual(table.name, 'Amon') self.assertEqual(table.date, 'December 2013') def test_get_table_not_added(self): + convention = Mock() + convention.name = 'specs' var = Variable() var.domain = ModelingRealms.atmos var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) - table = var.get_table(Frequencies.daily, 'specs') + table = var.get_table(Frequencies.daily, convention) self.assertEqual(table.frequency, Frequencies.daily) self.assertEqual(table.name, 'day') self.assertEqual(table.date, 'December 2013') def test_get_table_not_matching(self): + convention = Mock() + convention.name = 'specs' var = Variable() with self.assertRaises(ValueError): - var.get_table(Frequencies.daily, 'specs') + var.get_table(Frequencies.daily, convention) class TestVariableManager(TestCase): -- GitLab From 4bff1929772754db96b6f6b0993cb6971c8a9e88 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 15:03:59 +0200 Subject: [PATCH 3/6] Fixed lint --- test/unit/test_variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 7c6b9c0a..b5072f42 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -184,7 +184,7 @@ class TestVariable(TestCase): self.assertEqual(table.date, 'December 2013') def test_get_table_added(self): - convention= Mock() + convention = Mock() convention.name = 'specs' var = Variable() var.domain = ModelingRealms.atmos -- GitLab From 06570981f96381e2c2604f27614fef68e2c9f25f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 15:14:18 +0200 Subject: [PATCH 4/6] Fixed py3 tests --- test/integration/test_cmorizer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index e7b0d129..abe27695 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -1,9 +1,10 @@ from earthdiagnostics.cmorizer import Cmorizer from earthdiagnostics.utils import TempFile, Utils +from earthdiagnostics.data_convention import DataConvention from bscearth.utils import log from unittest import TestCase -from mock import Mock +from mock import Mock, create_autospec import os import tempfile import shutil @@ -49,7 +50,8 @@ class TestCmorizer(TestCase): self.data_manager.config.data_dir = os.path.join(self.tmp_dir, 'data') self.data_manager.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') TempFile.scratch_folder = self.data_manager.config.scratch_dir - self.data_manager.config.data_convention = 'data_convention' + self.data_manager.config.data_convention = create_autospec(DataConvention) + self.data_manager.config.data_convention.name = 'data_convention' self.data_manager.config.var_manager.get_variable_and_alias = self._get_variable_and_alias self.data_manager.config.var_manager.get_variable = self._get_variable -- GitLab From abf8153816dd3163465ee4a55245659183ebe9b7 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 15:37:36 +0200 Subject: [PATCH 5/6] Fixed cmorization --- earthdiagnostics/cmorizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 9f7a9e10..efdadcfe 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -705,13 +705,13 @@ class Cmorizer(object): handler.physics_description = cmor.physics_description handler.model_id = experiment.model handler.associated_model = cmor.associated_model - handler.project_id = self.config.data_convention.upper() + handler.project_id = self.config.data_convention.name.upper() handler.realization = str(self.member + 1) handler.source = cmor.source handler.startdate = 'S{0}'.format(self.startdate) handler.tracking_id = str(uuid.uuid1()) handler.title = "{0} model output prepared for {2} {1}".format(experiment.model, experiment.experiment_name, - self.config.data_convention.upper()) + self.config.data_convention.name.upper()) def _gribfiles_available(self): grb_path = os.path.join(self.original_files_path, '*.grb') -- GitLab From 6501c4b88aac143ac9b21fb7a6f0f219a4757eef Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 15:46:10 +0200 Subject: [PATCH 6/6] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6f9b87cf..78238d72 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc6 +3.0.0rc7 -- GitLab