From 57dbe97aee28c0cba6c9f120168e774c45d54bf7 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 29 May 2018 11:35:04 +0200 Subject: [PATCH 01/12] Improved tests --- earthdiagnostics/config.py | 5 +-- earthdiagnostics/data_convention.py | 8 ++-- .../data_convention/test_data_convention.py | 39 ++++++++++--------- test/unit/data_convention/test_specs.py | 20 ++++++++++ test/unit/test_config.py | 38 +++++++++++++++++- test/unit/test_modelling_realm.py | 1 + 6 files changed, 86 insertions(+), 25 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index ecd00031..5321e160 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -158,9 +158,8 @@ class Config(object): namelist_file = os.path.join(os.path.dirname(__file__), 'CDFTOOLS_{0}.namlist'.format(self.data_convention.name)) Log.debug(namelist_file) - if os.path.isfile(namelist_file): - Log.debug('Setting namelist {0}', namelist_file) - os.environ['NAM_CDF_NAMES'] = namelist_file + Log.debug('Setting namelist {0}', namelist_file) + os.environ['NAM_CDF_NAMES'] = namelist_file self.var_manager = VariableManager() self.var_manager.load_variables(data_convention) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 4207f844..373a9ce1 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -24,6 +24,9 @@ class DataConvention(object): self.lock = threading.Lock() self._checked_vars = list() + def get_scratch_masks(self, scratch_masks): + return scratch_masks + def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, grid=None, year=None, date_str=None): """ @@ -262,6 +265,8 @@ class DataConvention(object): self.lock.release() def _get_time_component(self, chunk, date_str, frequency, startdate, year): + if len([x for x in (chunk, date_str, year) if x is not None]) > 1: + raise ValueError('Only one of the parameters chunk, year or date_str may be provided') if chunk is not None: time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency) elif year: @@ -302,9 +307,6 @@ class DataConvention(object): class Cmor2Convention(DataConvention): """Base class for CMOR2-based conventions""" - def get_scratch_masks(self, scratch_masks): - return scratch_masks - def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): if cmor_var is None: cmor_table = domain.get_table(frequency, self.config.data_convention) diff --git a/test/unit/data_convention/test_data_convention.py b/test/unit/data_convention/test_data_convention.py index 4ce42ac0..1c1e999a 100644 --- a/test/unit/data_convention/test_data_convention.py +++ b/test/unit/data_convention/test_data_convention.py @@ -17,6 +17,7 @@ class TestDataConvention(TestCase): os.mkdir(os.path.join(self.tmp_dir, 'expid')) self.config = Mock() + self.config.frequency = 'mon' self.config.experiment.experiment_name = 'expname' self.config.data_dir = self.tmp_dir self.config.experiment.expid = 'expid' @@ -60,29 +61,31 @@ class TestDataConvention(TestCase): def test_get_member_str(self): with self.assertRaises(NotImplementedError): self.convention.get_member_str(None) + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_cmor_folder_path', autospec=True) + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_file_name', autospec=True) + def test_get_file_path(self, mock_filename, mock_cmor_path): + mock_cmor_path.return_value = 'path' + mock_filename.return_value = 'filename' + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon - def test_get_file_path_raise_incompatible_date_info(self): + filepath = self.convention.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 1, 'mon') + self.assertEqual('path/filename', filepath) + + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_cmor_folder_path', autospec=True) + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_file_name', autospec=True) + def test_get_file_path_no_freq(self, mock_filename, mock_cmor_path): + mock_cmor_path.return_value = 'path' + mock_filename.return_value = 'filename' cmor_var = Mock() omon = Mock() omon.name = 'Omon' cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.frequency = 'monthly' - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' - - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, None, frequency, year='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, 1, frequency, year='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, 1, frequency, date_str='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, None, frequency, year='1998', date_str='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, 1, frequency, year='1998', date_str='1998') - self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', - cmor_var, None, frequency) + + filepath = self.convention.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 1, None) + self.assertEqual('path/filename', filepath) def test_create_link(self): file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) diff --git a/test/unit/data_convention/test_specs.py b/test/unit/data_convention/test_specs.py index 82e7e982..c4b2b6ed 100644 --- a/test/unit/data_convention/test_specs.py +++ b/test/unit/data_convention/test_specs.py @@ -177,6 +177,26 @@ class TestSpecsConvention(TestCase): self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, Frequencies.monthly, None, None, None, None) + def test_get_file_path_raise_incompatible_date_info(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = 'mon' + + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency, year='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, year='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency) + @mock.patch('os.path.isfile') def test_is_cmorized(self, mock_is_file): mock_is_file.return_value = True diff --git a/test/unit/test_config.py b/test/unit/test_config.py index b588881a..4a0e51a7 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -7,7 +7,8 @@ import os from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig, Config from earthdiagnostics.frequency import Frequencies from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.data_convention import SPECSConvention, PrimaveraConvention +from earthdiagnostics.data_convention import SPECSConvention, PrimaveraConvention, PrefaceConvention, \ + MeteoFranceConvention, CMIP6Convention class VariableMock(object): @@ -507,6 +508,11 @@ class TestConfig(TestCase): self.assertEqual(config.con_files, '') self.assertEqual(config.data_adaptor, 'CMOR') self.assertEqual(config.get_commands(), (['diag1', 'diag2'])) + self.assertIsInstance(config.data_convention, SPECSConvention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_specs.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) def test_alias(self): config = Config() @@ -531,3 +537,33 @@ class TestConfig(TestCase): namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'earthdiagnostics/CDFTOOLS_primavera.namlist')) self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) + + def test_data_convention_cmip6(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'cmip6') + self._parse(config) + self.assertIsInstance(config.data_convention, CMIP6Convention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks/cmip6') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_cmip6.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) + + def test_data_convention_meteofrance(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'meteofrance') + self._parse(config) + self.assertIsInstance(config.data_convention, MeteoFranceConvention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_meteofrance.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) + + def test_data_convention_preface(self): + config = Config() + self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'preface') + self._parse(config) + self.assertIsInstance(config.data_convention, PrefaceConvention) + self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks') + namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', + 'earthdiagnostics/CDFTOOLS_preface.namlist')) + self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 99c74fdf..8340ffa7 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -12,6 +12,7 @@ class TestModellingRealms(TestCase): self.assertEqual(ModelingRealms.parse('atmos'), ModelingRealms.atmos) self.assertEqual(ModelingRealms.parse('atmoschem'), ModelingRealms.atmosChem) self.assertEqual(ModelingRealms.parse('atmoSChem'), ModelingRealms.atmosChem) + self.assertIsNone(ModelingRealms.parse('')) with self.assertRaises(ValueError): ModelingRealms.parse('badrealm') -- GitLab From f6997792db49c20ea22f4c0ca185d263564cc01c Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 29 May 2018 11:50:07 +0200 Subject: [PATCH 02/12] Improved tests --- .../data_convention/test_data_convention.py | 5 +++++ test/unit/data_convention/test_primavera.py | 17 +++++++++++++++++ test/unit/data_convention/test_specs.py | 19 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/test/unit/data_convention/test_data_convention.py b/test/unit/data_convention/test_data_convention.py index 1c1e999a..152198ec 100644 --- a/test/unit/data_convention/test_data_convention.py +++ b/test/unit/data_convention/test_data_convention.py @@ -61,6 +61,11 @@ class TestDataConvention(TestCase): def test_get_member_str(self): with self.assertRaises(NotImplementedError): self.convention.get_member_str(None) + + def test_is_cmorized(self): + with self.assertRaises(NotImplementedError): + self.convention.is_cmorized(None, None, None, None) + @mock.patch('earthdiagnostics.data_convention.DataConvention.get_cmor_folder_path', autospec=True) @mock.patch('earthdiagnostics.data_convention.DataConvention.get_file_name', autospec=True) def test_get_file_path(self, mock_filename, mock_cmor_path): diff --git a/test/unit/data_convention/test_primavera.py b/test/unit/data_convention/test_primavera.py index 271f8db2..2eb75af9 100644 --- a/test/unit/data_convention/test_primavera.py +++ b/test/unit/data_convention/test_primavera.py @@ -63,6 +63,13 @@ class TestPrimaveraConvention(TestCase): os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' 'r2i1p1f1/Omon/var/ocean_grid/version')) + def test_get_cmor_folder_path_no_cmor_var(self): + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' + 'r2i1p1f1/Omon/var/ocean_grid/version')) + def test_get_cmor_folder_path_atmos(self): cmor_var = Mock() omon = Mock() @@ -115,6 +122,16 @@ class TestPrimaveraConvention(TestCase): self.assertEqual(file_path, 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_no_cmor_var(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', None, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_daily(self): cmor_var = Mock() omon = Mock() diff --git a/test/unit/data_convention/test_specs.py b/test/unit/data_convention/test_specs.py index c4b2b6ed..8e054d10 100644 --- a/test/unit/data_convention/test_specs.py +++ b/test/unit/data_convention/test_specs.py @@ -116,6 +116,12 @@ class TestSpecsConvention(TestCase): self.assertEqual(file_path, 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + def test_get_filename_no_cmor_var(self): + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', None, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + def test_get_filename_daily(self): cmor_var = Mock() omon = Mock() @@ -225,6 +231,19 @@ class TestSpecsConvention(TestCase): 'r2i1p1/version')) self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + @mock.patch('os.path.isfile') + def test_is_cmorized_no_startdate_path(self, mock_is_file): + mock_is_file.return_value = False + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + def test_is_cmorized_false_not_member_folder(self): cmor_var = Mock() omon = Mock() -- GitLab From 7f2b81c5e20d182a19afc669f0d7cb9bf8291c04 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 29 May 2018 11:56:50 +0200 Subject: [PATCH 03/12] Fixed some codacy warnings --- earthdiagnostics/data_convention.py | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 373a9ce1..4382f346 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -1,3 +1,4 @@ +"""Module to manage the different data conventions supported by EarthDiagnostics""" import os import shutil import re @@ -25,6 +26,18 @@ class DataConvention(object): self._checked_vars = list() def get_scratch_masks(self, scratch_masks): + """ + Get the final scratch_masks path + + Parameters + ---------- + scratch_masks: str + + Returns + ------- + str + + """ return scratch_masks def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, @@ -301,6 +314,26 @@ class DataConvention(object): return current_count def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + Raises + ------ + NotImplementedError: + If not implemented by the derived classes + + """ raise NotImplementedError @@ -356,6 +389,21 @@ class Cmor2Convention(DataConvention): "", False, vartype=VariableType.MEAN) def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + """ startdate_path = self.get_startdate_path(startdate) if not os.path.isdir(startdate_path): return False @@ -388,6 +436,7 @@ class PrefaceConvention(Cmor2Convention): config: Config """ + def __init__(self, name, config): super(PrefaceConvention, self).__init__(name, config) self.time_separator = '_' @@ -408,6 +457,7 @@ class Cmor3Convention(DataConvention): config: Config """ + def __init__(self, name, config): super(Cmor3Convention, self).__init__(name, config) self.lat_name = 'latitude' @@ -504,6 +554,21 @@ class Cmor3Convention(DataConvention): self.config.cmor.initialization_number) def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + """ startdate_path = self.get_startdate_path(startdate) if not os.path.isdir(startdate_path): return False @@ -524,11 +589,13 @@ class Cmor3Convention(DataConvention): class CMIP6Convention(Cmor3Convention): """Class managing CMIP6 file conventions""" + pass class PrimaveraConvention(Cmor3Convention): """Class managing Primavera file conventions""" + pass @@ -563,10 +630,57 @@ class MeteoFranceConvention(DataConvention): return time_bound def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + """ + Create file link + + In this convention, it does nothing + + Parameters + ---------- + domain: ModelingRealm + filepath: str + frequency: Frequency + var: str + grid: str + move_old: bool + vartype: VariableType + + """ pass def create_links(self, startdate, member=None): + """ + Create links for a given startdate or member + + In this convention, it does nothing + + Parameters + ---------- + startdate: str + member: int or None + + """ pass def is_cmorized(self, startdate, member, chunk, domain): + """ + Check if a given chunk is cmorized for a given domain + + Parameters + ---------- + startdate: str + member: str + chunk: int + domain: ModelingRealm + + Returns + ------- + bool + + Raises + ------ + NotImplementedError: + If not implemented by the derived classes + + """ return True -- GitLab From cb99d12cb6be02460d4a07005b5aeeafb39226cd Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 29 May 2018 12:05:22 +0200 Subject: [PATCH 04/12] Fixed some codacy warnings --- earthdiagnostics/data_convention.py | 142 ++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 4382f346..4572a7b2 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -341,6 +341,27 @@ class Cmor2Convention(DataConvention): """Base class for CMOR2-based conventions""" def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + """ if cmor_var is None: cmor_table = domain.get_table(frequency, self.config.data_convention) else: @@ -364,6 +385,23 @@ class Cmor2Convention(DataConvention): return folder_path def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ template = 'r{0}i{1}p1' return template.format(member + 1 - self.config.experiment.member_count_start, self.config.cmor.initialization_number) @@ -421,6 +459,18 @@ class SPECSConvention(Cmor2Convention): """Base class for CMOR2-based conventions""" def get_startdate_path(self, startdate): + """ + Return the path to the startdate's CMOR folder + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.experiment.institute, self.config.experiment.model, self.experiment_name(startdate), 'S' + startdate) @@ -442,6 +492,18 @@ class PrefaceConvention(Cmor2Convention): self.time_separator = '_' def get_startdate_path(self, startdate): + """ + Return the path to the startdate's CMOR folder + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.experiment.institute, self.experiment_name(startdate), 'S' + startdate) @@ -464,9 +526,44 @@ class Cmor3Convention(DataConvention): self.lon_name = 'longitude' def get_scratch_masks(self, scratch_masks): + """ + Get the final scratch_masks path + + Adds a folder matching the convention name to the configured path + + Parameters + ---------- + scratch_masks: str + + Returns + ------- + str + + """ return os.path.join(scratch_masks, self.name) def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + """ if cmor_var is None: cmor_table = domain.get_table(frequency, self.config.data_convention) else: @@ -549,6 +646,18 @@ class Cmor3Convention(DataConvention): False, vartype=VariableType.MEAN) def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + """ template = 'r{0}i{1}p1f1' return template.format(member + 1 - self.config.experiment.member_count_start, self.config.cmor.initialization_number) @@ -603,6 +712,27 @@ class MeteoFranceConvention(DataConvention): """Class managing MeteoFrance file conventions""" def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + """ if year is not None: raise ValueError('Year not supported with MeteoFrance convention') if date_str is not None: @@ -620,6 +750,18 @@ class MeteoFranceConvention(DataConvention): return folder_path def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + """ return '{0:02d}'.format(member) def _get_chunk_time_bounds(self, startdate, chunk, frequency): -- GitLab From 38c25740e15516a416b2f5579a217a5e6ef7061b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 29 May 2018 12:18:14 +0200 Subject: [PATCH 05/12] Refactored complex methods --- earthdiagnostics/cmorizer.py | 24 ++++++++------ earthdiagnostics/config.py | 62 +++++++++++++++++++----------------- earthdiagnostics/datafile.py | 23 +++++++------ earthdiagnostics/utils.py | 28 +++++++++------- 4 files changed, 77 insertions(+), 60 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index efdadcfe..c4b8af62 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -452,16 +452,7 @@ class Cmorizer(object): if 'leadtime' in handler.variables.keys(): coords.append('leadtime') - if var_cmor.domain == ModelingRealms.ocean: - lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', - 'depth': 'lev'} - elif var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: - lev_dimensions = {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', - 'depth_4': 'sdepth'} - elif var_cmor.domain == ModelingRealms.atmos: - lev_dimensions = {'depth': 'plev'} - else: - lev_dimensions = {} + lev_dimensions = self._get_lev_dimensions(var_cmor) for lev_dim in lev_dimensions.keys(): if lev_dim in handler.variables[variable].dimensions: @@ -516,6 +507,19 @@ class Cmorizer(object): region_str = '' Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str) + def _get_lev_dimensions(self, var_cmor): + if var_cmor.domain == ModelingRealms.ocean: + lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', + 'depth': 'lev'} + elif var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: + lev_dimensions = {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', + 'depth_4': 'sdepth'} + elif var_cmor.domain == ModelingRealms.atmos: + lev_dimensions = {'depth': 'plev'} + else: + lev_dimensions = {} + return lev_dimensions + def _get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') valid_starts = (self.experiment.expid, 'MMA', 'MMASH', 'MMAGG', 'MMO') diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 5321e160..572897c4 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -138,31 +138,10 @@ class Config(object): self.mask_regions = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS', '') self.mask_regions_3d = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS_3D', '') - data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', - ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), - 'specs', - ignore_case=True) - - if data_convention == 'specs': - self.data_convention = SPECSConvention(data_convention, self) - elif data_convention == 'primavera': - self.data_convention = PrimaveraConvention(data_convention, self) - elif data_convention == 'cmip6': - self.data_convention = CMIP6Convention(data_convention, self) - elif data_convention == 'preface': - self.data_convention = PrefaceConvention(data_convention, self) - elif data_convention == 'meteofrance': - self.data_convention = MeteoFranceConvention(data_convention, self) - - self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) - namelist_file = os.path.join(os.path.dirname(__file__), - 'CDFTOOLS_{0}.namlist'.format(self.data_convention.name)) - Log.debug(namelist_file) - Log.debug('Setting namelist {0}', namelist_file) - os.environ['NAM_CDF_NAMES'] = namelist_file + self._parse_dataconvention(parser) self.var_manager = VariableManager() - self.var_manager.load_variables(data_convention) + self.var_manager.load_variables(self.data_convention) self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') self.skip_diags_done = parser.get_bool_option('DIAGNOSTICS', 'SKIP_DIAGS_DONE', True) self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) @@ -184,6 +163,16 @@ class Config(object): Log.debug('Preparing command list') commands = self._diags.split() self._real_commands = list() + self._apply_aliases(commands) + Log.debug('Command list ready ') + + self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid) + + self.cmor = CMORConfig(parser, self.var_manager) + self.thredds = THREDDSConfig(parser) + self.report = ReportConfig(parser) + + def _apply_aliases(self, commands): for command in commands: command = command.strip() if command.startswith('#'): @@ -195,13 +184,28 @@ class Config(object): self._real_commands.append(add_command) else: self._real_commands.append(command) - Log.debug('Command list ready ') - self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid) - - self.cmor = CMORConfig(parser, self.var_manager) - self.thredds = THREDDSConfig(parser) - self.report = ReportConfig(parser) + def _parse_dataconvention(self, parser): + data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', + ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), + 'specs', + ignore_case=True) + if data_convention == 'specs': + self.data_convention = SPECSConvention(data_convention, self) + elif data_convention == 'primavera': + self.data_convention = PrimaveraConvention(data_convention, self) + elif data_convention == 'cmip6': + self.data_convention = CMIP6Convention(data_convention, self) + elif data_convention == 'preface': + self.data_convention = PrefaceConvention(data_convention, self) + elif data_convention == 'meteofrance': + self.data_convention = MeteoFranceConvention(data_convention, self) + self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) + namelist_file = os.path.join(os.path.dirname(__file__), + 'CDFTOOLS_{0}.namlist'.format(self.data_convention.name)) + Log.debug(namelist_file) + Log.debug('Setting namelist {0}', namelist_file) + os.environ['NAM_CDF_NAMES'] = namelist_file def get_commands(self): """ diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 852dab7a..07981fc6 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -660,16 +660,7 @@ class NetCDFFile(DataFile): if self.region: try: cubes = iris.load(self.remote_file) - for cube in cubes: - try: - if isinstance(self.region, six.string_types): - regions = {self.region.name} - else: - regions = {basin.name for basin in self.region} - if regions.issubset(set(cube.coord('region').points)): - self.storage_status = StorageStatus.READY - except iris.exceptions.CoordinateNotFoundError: - pass + self._check_regions(cubes) except iris.exceptions.TranslationError as ex: # If the check goes wrong, we must execute everything os.remove(self.remote_file) @@ -678,6 +669,18 @@ class NetCDFFile(DataFile): else: self.storage_status = StorageStatus.READY + def _check_regions(self, cubes): + for cube in cubes: + try: + if isinstance(self.region, six.string_types): + regions = {self.region.name} + else: + regions = {basin.name for basin in self.region} + if regions.issubset(set(cube.coord('region').points)): + self.storage_status = StorageStatus.READY + except iris.exceptions.CoordinateNotFoundError: + pass + def create_link(self): """Create a link from the original data in the _ folder""" try: diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 9f343a6a..2e12e466 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -611,6 +611,22 @@ class Utils(object): if new_name in destiny.variables.keys(): return + translated_dimensions = Utils._copy_dimensions(add_dimensions, destiny, must_exist, new_names, rename_dimension, + source, variable) + if new_name in destiny.variables.keys(): + # Just in case the variable we are copying match a dimension name + return + original_var = source.variables[variable] + new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) + Utils.copy_attributes(new_var, original_var) + if hasattr(new_var, 'coordinates'): + coords = [new_names[coord] if coord in new_names else coord for coord in new_var.coordinates.split(' ')] + new_var.coordinates = Utils.convert_to_ascii_if_possible(' '.join(coords)) + + new_var[:] = original_var[:] + + @staticmethod + def _copy_dimensions(add_dimensions, destiny, must_exist, new_names, rename_dimension, source, variable): if rename_dimension: translated_dimensions = Utils._translate(source.variables[variable].dimensions, new_names) else: @@ -621,17 +637,7 @@ class Utils(object): '{1} {2}'.format(variable, translated_dimensions, destiny.dimensions)) for dimension in source.variables[variable].dimensions: Utils.copy_dimension(source, destiny, dimension, must_exist, new_names, rename_dimension) - if new_name in destiny.variables.keys(): - # Just in case the variable we are copying match a dimension name - return - original_var = source.variables[variable] - new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) - Utils.copy_attributes(new_var, original_var) - if hasattr(new_var, 'coordinates'): - coords = [new_names[coord] if coord in new_names else coord for coord in new_var.coordinates.split(' ')] - new_var.coordinates = Utils.convert_to_ascii_if_possible(' '.join(coords)) - - new_var[:] = original_var[:] + return translated_dimensions @staticmethod def copy_attributes(new_var, original_var, omitted_attributtes=None): -- GitLab From a32d1c25ae191e00a3aad24c0f19de2b8ce318fd Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 30 May 2018 13:17:42 +0200 Subject: [PATCH 06/12] Improved tests --- earthdiagnostics/variable.py | 4 +++- test/unit/test_diagnostic.py | 26 ++++++++++++++++++++++++++ test/unit/test_variable.py | 4 ++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 597c2747..705aea3b 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -167,7 +167,9 @@ class VariableManager(object): if 'variable_entry' in data: Log.debug('Parsing file {0}'.format(json_path)) table_id = data['Header']['table_id'][6:] - table = CMORTable(table_id, Frequency(data['Header']['frequency']), data['Header']['table_date']) + table = CMORTable(table_id, + Frequency(data['variable_entry'].values()[0]['frequency']), + data['Header']['table_date']) self.tables[table_id] = table self._load_json_variables(data['variable_entry'], table) diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index 2ee05480..f425083b 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -367,6 +367,32 @@ class TestDiagnostic(TestCase): def test_empty_process_options(self): self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) + def test_diagnostic_can_be_skipped(self): + diag = self._get_diag_for_skipping() + self.assertTrue(diag.can_skip_run()) + + def _get_diag_for_skipping(self, modified=False, status=StorageStatus.READY): + mock = Mock() + datafile_mock = Mock() + datafile_mock.storage_status = status + datafile_mock.has_modifiers.return_value = modified + mock.declare_chunk.return_value = datafile_mock + diag = Diagnostic(mock) + diag.declare_chunk(None, None, None, None, None) + return diag + + def test_diagnostic_can_not_be_skipped_do_not_generate_files(self): + diag = Diagnostic(None) + self.assertFalse(diag.can_skip_run()) + + def test_diagnostic_can_not_be_skipped_modified(self): + diag = self._get_diag_for_skipping(modified=True) + self.assertFalse(diag.can_skip_run()) + + def test_diagnostic_can_not_be_skipped_data_not_ready(self): + diag = self._get_diag_for_skipping(status=StorageStatus.PENDING) + self.assertFalse(diag.can_skip_run()) + class TestDiagnosticBasinOption(TestCase): diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index b5072f42..f6c86e2a 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -223,15 +223,19 @@ class TestVariableManager(TestCase): def test_load_primavera(self): self.var_manager.load_variables('primavera') + self.assertTrue(self.var_manager.get_all_variables()) def test_load_cmip6(self): self.var_manager.load_variables('cmip6') + self.assertTrue(self.var_manager.get_all_variables()) def test_load_specs(self): self.var_manager.load_variables('specs') + self.assertTrue(self.var_manager.get_all_variables()) def test_load_preface(self): self.var_manager.load_variables('preface') + self.assertTrue(self.var_manager.get_all_variables()) def test_bad_load(self): with self.assertRaises(Exception): -- GitLab From 1eb33a6f7b6898aac6d9b30648f8591aff31b46a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 30 May 2018 13:20:03 +0200 Subject: [PATCH 07/12] Refactored extract_variable --- earthdiagnostics/cmorizer.py | 46 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index c4b8af62..50ffd5f7 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -447,28 +447,8 @@ class Cmorizer(object): return temp = TempFile.get() - handler = Utils.open_cdf(file_path) - coords = [self.lon_name, self.lat_name, 'time'] - if 'leadtime' in handler.variables.keys(): - coords.append('leadtime') - - lev_dimensions = self._get_lev_dimensions(var_cmor) - - for lev_dim in lev_dimensions.keys(): - if lev_dim in handler.variables[variable].dimensions: - coords.append(lev_dim) - - handler.variables[variable].coordinates = ' '.join(set(coords)) - handler.close() - - cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable)) - for lev_original, lev_target in six.iteritems(lev_dimensions): - try: - cube.coord(lev_original).var_name = lev_target - except iris.exceptions.CoordinateNotFoundError: - pass - - iris.save(cube, temp, zlib=True) + lev_dimensions = self.set_coordinates_attribute(file_path, var_cmor, variable) + self._rename_level_coords(file_path, lev_dimensions, temp) if alias.basin is None: region = None @@ -507,6 +487,28 @@ class Cmorizer(object): region_str = '' Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str) + def _rename_level_coords(self, file_path, lev_dimensions, temp): + cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable)) + for lev_original, lev_target in six.iteritems(lev_dimensions): + try: + cube.coord(lev_original).var_name = lev_target + except iris.exceptions.CoordinateNotFoundError: + pass + iris.save(cube, temp, zlib=True) + + def set_coordinates_attribute(self, file_path, var_cmor, variable): + handler = Utils.open_cdf(file_path) + coords = [self.lon_name, self.lat_name, 'time'] + if 'leadtime' in handler.variables.keys(): + coords.append('leadtime') + lev_dimensions = self._get_lev_dimensions(var_cmor) + for lev_dim in lev_dimensions.keys(): + if lev_dim in handler.variables[variable].dimensions: + coords.append(lev_dim) + handler.variables[variable].coordinates = ' '.join(set(coords)) + handler.close() + return lev_dimensions + def _get_lev_dimensions(self, var_cmor): if var_cmor.domain == ModelingRealms.ocean: lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', -- GitLab From 1766561452b96d551b317dd91c3b05cda500ae9d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 30 May 2018 15:38:14 +0200 Subject: [PATCH 08/12] Added more tests --- earthdiagnostics/cmorizer.py | 8 +++---- test/unit/test_constants.py | 43 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 50ffd5f7..87c2169a 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -447,8 +447,8 @@ class Cmorizer(object): return temp = TempFile.get() - lev_dimensions = self.set_coordinates_attribute(file_path, var_cmor, variable) - self._rename_level_coords(file_path, lev_dimensions, temp) + lev_dimensions = self._set_coordinates_attribute(file_path, var_cmor, variable) + self._rename_level_coords(file_path, lev_dimensions, temp, variable) if alias.basin is None: region = None @@ -487,7 +487,7 @@ class Cmorizer(object): region_str = '' Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str) - def _rename_level_coords(self, file_path, lev_dimensions, temp): + def _rename_level_coords(self, file_path, lev_dimensions, temp, variable): cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable)) for lev_original, lev_target in six.iteritems(lev_dimensions): try: @@ -496,7 +496,7 @@ class Cmorizer(object): pass iris.save(cube, temp, zlib=True) - def set_coordinates_attribute(self, file_path, var_cmor, variable): + def _set_coordinates_attribute(self, file_path, var_cmor, variable): handler = Utils.open_cdf(file_path) coords = [self.lon_name, self.lat_name, 'time'] if 'leadtime' in handler.variables.keys(): diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index a3d7983e..1a5b96fb 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -1,5 +1,6 @@ # coding=utf-8 from unittest import TestCase +from mock import Mock from earthdiagnostics.constants import Basin, Basins @@ -16,9 +17,36 @@ class TestBasin(TestCase): self.assertTrue(Basin('Basin') == self.basin) self.assertFalse(Basin('bas') == self.basin) + def test__neq__(self): + self.assertFalse(Basin('Basin') != self.basin) + self.assertTrue(Basin('bas') != self.basin) + + def test__greater(self): + self.assertFalse(Basin('Aasin') > self.basin) + self.assertFalse(Basin('Basin') > self.basin) + self.assertTrue(Basin('Casin') > self.basin) + + def test__lower(self): + self.assertTrue(Basin('Aasin') < self.basin) + self.assertFalse(Basin('Basin') < self.basin) + self.assertFalse(Basin('Casin') < self.basin) + + def test_ge(self): + self.assertFalse(Basin('Aasin') >= self.basin) + self.assertTrue(Basin('Basin') >= self.basin) + self.assertTrue(Basin('Casin') >= self.basin) + + def test__le(self): + self.assertTrue(Basin('Aasin') <= self.basin) + self.assertTrue(Basin('Basin') <= self.basin) + self.assertFalse(Basin('Casin') <= self.basin) + def test__str__(self): self.assertEqual(str(self.basin), 'Basin') + def test__repr__(self): + self.assertEqual(repr(self.basin), 'Basin') + def test_order(self): self.assertTrue(self.basin < Basin('Vasin')) self.assertTrue(self.basin > Basin('Asin')) @@ -28,3 +56,18 @@ class TestBasins(TestCase): def test_singleton(self): self.assertIs(Basins(), Basins()) + + def test_get_basins(self): + mock_handler = Mock() + mock_handler.variables.keys.return_value = ('var1', 'lat') + Basins().get_available_basins(mock_handler) + self.assertTrue(hasattr(Basins(), 'var1')) + self.assertFalse(hasattr(Basins(), 'lat')) + + def test_get_basins_with_alias(self): + mock_handler = Mock() + mock_handler.variables.keys.return_value = ('Kara_Sea', 'lat') + basins = Basins() + Basins().get_available_basins(mock_handler) + self.assertTrue(hasattr(basins, 'Kara_Sea')) + self.assertFalse(hasattr(basins, 'lat')) -- GitLab From 9e1a3edc6f89c17b29dd770e98c3164651dd38cf Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Jun 2018 09:11:13 +0200 Subject: [PATCH 09/12] Fix extent bug --- earthdiagnostics/config.py | 2 +- earthdiagnostics/data_convention.py | 6 +++--- earthdiagnostics/ocean/siasiesiv.py | 12 +++++++----- environment.yml | 1 + test/unit/general/test_monthlymean.py | 23 ++++++++++++++++++++--- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 572897c4..781a199a 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -141,7 +141,7 @@ class Config(object): self._parse_dataconvention(parser) self.var_manager = VariableManager() - self.var_manager.load_variables(self.data_convention) + self.var_manager.load_variables(self.data_convention.name) self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') self.skip_diags_done = parser.get_bool_option('DIAGNOSTICS', 'SKIP_DIAGS_DONE', True) self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 4572a7b2..43122007 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -419,12 +419,12 @@ class Cmor2Convention(DataConvention): for name in os.listdir(os.path.join(path, freq, domain, var, member)): filepath = os.path.join(path, freq, domain, var, member, name) if os.path.isfile(filepath): - self.create_link(domain, filepath, frequency, var, "", False, + self.create_link(ModelingRealms.parse(domain), filepath, frequency, var, "", False, vartype=VariableType.MEAN) else: for filename in os.listdir(filepath): - self.create_link(domain, os.path.join(filepath, filename), frequency, var, - "", False, vartype=VariableType.MEAN) + self.create_link(ModelingRealms.parse(domain), os.path.join(filepath, filename), + frequency, var, "", False, vartype=VariableType.MEAN) def is_cmorized(self, startdate, member, chunk, domain): """ diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 6c5d3385..42b16f41 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -3,6 +3,8 @@ import os import six +import numpy as np + import iris import iris.analysis import iris.coords @@ -135,9 +137,10 @@ class Siasiesiv(Diagnostic): sic = iris.load_cube(self.sic.local_file) if sic.units.origin == '%' and sic.data.max() < 2: sic.units = '1.0' - sic.convert_units('1.0') + + extent = sic.copy((sic.data >= 0.15).astype(np.int8)) * Siasiesiv.area.data sic *= Siasiesiv.area.data - extent = sic.data > 0.15 + if not self.omit_volume: handler = Utils.open_cdf(self.sit.local_file) @@ -154,9 +157,8 @@ class Siasiesiv(Diagnostic): self.results['sivoln'][basin] = self.sum(volume, mask, north=True) self.results['sivols'][basin] = self.sum(volume, mask, north=False) - extent_mask = mask * extent - self.results['siextentn'][basin] = self.sum(sic, extent_mask, north=True) - self.results['siextents'][basin] = self.sum(sic, extent_mask, north=False) + self.results['siextentn'][basin] = self.sum(extent, mask, north=True) + self.results['siextents'][basin] = self.sum(extent, mask, north=False) self.save() diff --git a/environment.yml b/environment.yml index 42dcdbbe..c64b1ce5 100644 --- a/environment.yml +++ b/environment.yml @@ -31,3 +31,4 @@ dependencies: - exrex - xxhash - pytest-profiling + - dummydata diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py index 5672c462..0e89916b 100644 --- a/test/unit/general/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -1,7 +1,10 @@ # coding=utf-8 +import os from unittest import TestCase +from tempfile import mktemp from mock import Mock, patch +import dummydata.model3 from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption @@ -13,6 +16,7 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestMonthlyMean(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() @@ -23,7 +27,11 @@ class TestMonthlyMean(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + self.monmean = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'ta', 'freq', '') + + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -51,6 +59,15 @@ class TestMonthlyMean(TestCase): MonthlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), + self.assertEqual(str(self.monmean), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + 'Variable: atmos:ta Original frequency: freq Grid: ') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, start_year=2000, stop_year=2000, method='constant', constant=1) + + self.monmean.variable_file = Mock() + self.monmean.variable_file.local_file = self.var_file + self.monmean.mean_file = Mock() + self.monmean.compute() + self.monmean.mean_file.set_local_file.assert_called_once() -- GitLab From 24ca86c440f52d8246605e1d3f7e4c94d4b8d5ee Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Jun 2018 10:02:57 +0200 Subject: [PATCH 10/12] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index fa1beb5e..e2859adc 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc8 +3.0.0rc9 -- GitLab From 03ff462e31838428c15a77814a1730f9f6743b8b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Jun 2018 10:31:00 +0200 Subject: [PATCH 11/12] Added more tests --- earthdiagnostics/ocean/siasiesiv.py | 1 - test/unit/general/test_dailymean.py | 27 ++++++++++++++++++++------- test/unit/general/test_monthlymean.py | 5 ----- test/unit/general/test_yearlymean.py | 25 ++++++++++++++++++------- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 42b16f41..c1df586b 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -141,7 +141,6 @@ class Siasiesiv(Diagnostic): extent = sic.copy((sic.data >= 0.15).astype(np.int8)) * Siasiesiv.area.data sic *= Siasiesiv.area.data - if not self.omit_volume: handler = Utils.open_cdf(self.sit.local_file) handler.variables[self.sit_varname].coordinates = coordinates diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py index 426e8931..741779a4 100644 --- a/test/unit/general/test_dailymean.py +++ b/test/unit/general/test_dailymean.py @@ -1,9 +1,11 @@ # coding=utf-8 from unittest import TestCase - +from tempfile import mktemp from mock import Mock, patch +import os + +import dummydata -from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.timemean import DailyMean @@ -13,15 +15,18 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestDailyMean(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() self.diags.model_version = 'model_version' self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 + self.daymean = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -50,7 +55,15 @@ class TestDailyMean(TestCase): DailyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - mixed = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') - self.assertEqual(str(mixed), + self.assertEqual(str(self.daymean), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Original frequency: freq Grid: ') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, start_year=2000, stop_year=2000, method='constant', constant=1) + + self.daymean.variable_file = Mock() + self.daymean.variable_file.local_file = self.var_file + self.daymean.mean_file = Mock() + self.daymean.compute() + self.daymean.mean_file.set_local_file.assert_called_once() diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py index 0e89916b..0ee4dee8 100644 --- a/test/unit/general/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -6,7 +6,6 @@ from tempfile import mktemp from mock import Mock, patch import dummydata.model3 -from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.timemean import MonthlyMean @@ -23,10 +22,6 @@ class TestMonthlyMean(TestCase): self.diags.model_version = 'model_version' self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 - self.monmean = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'ta', 'freq', '') def tearDown(self): diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py index 5c0d7c03..a37b70aa 100644 --- a/test/unit/general/test_yearlymean.py +++ b/test/unit/general/test_yearlymean.py @@ -1,9 +1,10 @@ # coding=utf-8 from unittest import TestCase - +from tempfile import mktemp from mock import Mock, patch +import os +import dummydata -from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.timemean import YearlyMean @@ -13,17 +14,18 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestYearlyMean(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.diags.config.frequency = Frequencies.monthly - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 + self.yearly_mean = YearlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') - self.mixed = YearlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -51,6 +53,15 @@ class TestYearlyMean(TestCase): YearlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), + self.assertEqual(str(self.yearly_mean), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' 'Variable: ocean:var Original frequency: freq Grid: ') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, start_year=2000, stop_year=2000, method='constant', constant=1) + + self.yearly_mean.variable_file = Mock() + self.yearly_mean.variable_file.local_file = self.var_file + self.yearly_mean.mean_file = Mock() + self.yearly_mean.compute() + self.yearly_mean.mean_file.set_local_file.assert_called_once() -- GitLab From c8e5966fea1a5937d6befdbcc0f6b8f06eb84053 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Jun 2018 11:15:51 +0200 Subject: [PATCH 12/12] Added test for vertical mean meters --- .../general/verticalmeanmetersiris.py | 4 ++-- .../general/test_verticalmeanmetersiris.py | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index 602fc5cd..80f01390 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -105,7 +105,7 @@ class VerticalMeanMetersIris(Diagnostic): var_cube = iris.load_cube(self.variable_file.local_file) - lev_names = ('lev', 'depth') + lev_names = ('lev', 'depth', 'air_pressure') coord = None for coord_name in lev_names: try: @@ -122,7 +122,7 @@ class VerticalMeanMetersIris(Diagnostic): lev_max = coord.points[-1] else: lev_max = self.box.max_depth - lev_constraint = iris.Constraint(coord_values={coord.var_name: lambda cell: lev_min <= cell <= lev_max}) + lev_constraint = iris.Constraint(coord_values={coord.name(): lambda cell: lev_min <= cell <= lev_max}) var_cube = var_cube.extract(lev_constraint) var_cube = var_cube.collapsed(coord, iris.analysis.MEAN) temp = TempFile.get() diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py index 49f6a1b5..99dd5766 100644 --- a/test/unit/general/test_verticalmeanmetersiris.py +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -1,6 +1,9 @@ # coding=utf-8 from unittest import TestCase +import os +from tempfile import mktemp +import dummydata from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError from earthdiagnostics.box import Box from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris @@ -13,6 +16,7 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestVerticalMeanMetersIris(TestCase): def setUp(self): + self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() @@ -24,6 +28,10 @@ class TestVerticalMeanMetersIris(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) + def fake_parse(self, value): if not value: raise DiagnosticOptionError @@ -74,3 +82,18 @@ class TestVerticalMeanMetersIris(TestCase): self.assertEqual(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Box: 0-100m') + + def test_compute(self): + box = Box() + box.min_depth = 1 + box.max_depth = 50000 + + dummydata.model3.Model3(oname=self.var_file, var='ta', start_year=2000, stop_year=2000, method='constant', + constant=1) + + diag = VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', box) + + diag.variable_file = Mock() + diag.variable_file.local_file = self.var_file + diag.results = Mock() + diag.compute() -- GitLab