diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index e32c5cdfbd2a6cfd2570099412f2b0e3b7e70ba7..f43b83b0b754b9755728c93ff49a081dab3e6be1 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -462,12 +462,19 @@ class DataFile(Publisher): final_cube = cube_list.merge_cube() temp = TempFile.get() iris.save(final_cube, temp, zlib=True) + handler = Utils.open_cdf(temp) + renames = {} + for dim in handler.dimensions: + if dim.startswith('dim'): + renames[dim] = 'region' + if dim.startswith('string'): + renames[dim] = 'region_length' if '-' in final_cube.var_name: - Utils.rename_variable( - temp, - final_cube.var_name.replace('-', '_'), - final_cube.var_name, must_exist=False - ) + renames[final_cube.var_name.replace('-', '_')] = \ + final_cube.var_name + + Utils.rename_variables( + temp, renames, must_exist=False, rename_dimension=True) Utils.move_file(temp, self.local_file) self._correct_metadata() diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 2eb41416a4fa957271d4234e4658d8b98fc1bdb5..fe9cc8131246e3d531f5edd01a69c4814f8e293f 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -68,7 +68,6 @@ class RegionMean(Diagnostic): variable, box, save3d, - variance, basins, grid_point, frequency, @@ -81,7 +80,6 @@ class RegionMean(Diagnostic): self.variable = variable self.box = box self.save3d = save3d - self.variance = variance self.basins = basins self.grid_point = grid_point self.frequency = frequency @@ -106,7 +104,7 @@ class RegionMean(Diagnostic): return ( "Region mean Startdate: {0.startdate} Member: {0.member} " "Chunk: {0.chunk} Variable: {0.variable} " - "Box: {0.box} Save 3D: {0.save3d} Save variance: {0.variance} " + "Box: {0.box} Save 3D: {0.save3d} " "Grid point: {0.grid_point}".format(self) ) @@ -134,7 +132,6 @@ class RegionMean(Diagnostic): DiagnosticIntOption("min_depth", -1), DiagnosticIntOption("max_depth", -1), DiagnosticBoolOption("save3D", True), - DiagnosticBoolOption("variance", False), DiagnosticOption("grid", ""), DiagnosticFrequencyOption("frequency", diags.config.frequency), ) @@ -165,7 +162,6 @@ class RegionMean(Diagnostic): var, box, options["save3D"], - options["variance"], options["basins"], options["grid_point"].lower(), options["frequency"], @@ -206,7 +202,7 @@ class RegionMean(Diagnostic): def _mean_2d_var(self, data, mesh, masks): areacello = mesh.get_areacello(cell_point=self.grid_point) - mean = regmean.compute_regmean_2D(data.data, masks, areacello) + mean = regmean.compute_regmean_2d(data.data, masks, areacello) self._save_result_2d("mean", mean, data) def _meand_3d_variable(self, data, mesh, masks): @@ -221,7 +217,7 @@ class RegionMean(Diagnostic): e3 = e3.extract(depth_constraint) data = data.extract(depth_constraint) volcello = areacello * e3.data.astype(np.float32) - mean = regmean.compute_regmean_3D(data.data, masks, volcello) + mean = regmean.compute_regmean_3d(data.data, masks, volcello) self._save_result_2d("mean", mean, data) if self.save3d: mean3d = regmean.compute_regmean_levels(data.data, masks, volcello) @@ -328,6 +324,7 @@ class RegionMean(Diagnostic): self.chunk, box=box_save, region=self.basins, + frequency=self.frequency, ) def _save_result_2d(self, var, result, data): diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index adadf3036dc50c8980a43a9bdb517dbd807288ba..ded849f673b5971e82fde3d057ff981842496d7b 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -19,7 +19,8 @@ from earthdiagnostics.diagnostic import ( DiagnosticDomainOption, DiagnosticBoolOption, DiagnosticBasinListOption, - DiagnosticVariableOption, + DiagnosticVariableListOption, + DiagnosticFrequencyOption ) from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile @@ -70,6 +71,7 @@ class RegionSum(Diagnostic): save3d, basins, grid, + frequency, ): Diagnostic.__init__(self, data_manager) self.startdate = startdate @@ -82,11 +84,15 @@ class RegionSum(Diagnostic): self.save3d = save3d self.basins = basins self.grid = grid + self.frequency = frequency + self.declared = {} self.lat_name = "lat" self.lon_name = "lon" + self._hash = None + def __eq__(self, other): if self._different_type(other): return False @@ -99,14 +105,14 @@ class RegionSum(Diagnostic): and self.variable == other.variable and self.grid_point == other.grid_point and self.grid == other.grid - and self.basin == other.basin + and self.basins == other.basins ) def __str__(self): return ( - "Region sum Startdate: {0.startdate} Member: {0.member}" + "Region sum Startdate: {0.startdate} Member: {0.member} " "Chunk: {0.chunk} Variable: {0.variable} " - "Grid point: {0.grid_point} Box: {0.box} Save 3D: {0.save3d}" + "Grid point: {0.grid_point} Box: {0.box} Save 3D: {0.save3d} " "Original grid: {0.grid} Basin: {0.basins}".format(self) ) @@ -126,54 +132,52 @@ class RegionSum(Diagnostic): """ options_available = ( DiagnosticDomainOption(), - DiagnosticVariableOption(diags.data_manager.config.var_manager), - DiagnosticOption("grid_point", "T"), + DiagnosticVariableListOption( + diags.data_manager.config.var_manager, "variable" + ), DiagnosticBasinListOption("basins", "global"), + DiagnosticOption("grid_point", "T"), DiagnosticIntOption("min_depth", -1), DiagnosticIntOption("max_depth", -1), - DiagnosticIntOption("min_lat", -1), - DiagnosticIntOption("max_lat", -1), - DiagnosticIntOption("min_lon", -1), - DiagnosticIntOption("max_lon", -1), DiagnosticBoolOption("save3D", True), DiagnosticOption("grid", ""), + DiagnosticFrequencyOption("frequency", diags.config.frequency), ) options = cls.process_options(options, options_available) box = Box() box.min_depth = options["min_depth"] box.max_depth = options["max_depth"] - box.min_lat = options["min_lat"] - box.max_lat = options["max_lat"] - box.min_lon = options["min_lon"] - box.max_lon = options["max_lon"] basins = options["basins"] if not basins: Log.error("Basins not recognized") return () + basins.sort() job_list = list() - for ( - startdate, - member, - chunk, - ) in diags.config.experiment.get_chunk_list(): - job_list.append( - RegionSum( - diags.data_manager, - startdate, - member, - chunk, - options["domain"], - options["variable"], - options["grid_point"].lower(), - box, - options["save3D"], - options["basins"], - options["grid"], + for var in options["variable"]: + for ( + startdate, + member, + chunk, + ) in diags.config.experiment.get_chunk_list(): + job_list.append( + RegionSum( + diags.data_manager, + startdate, + member, + chunk, + options["domain"], + var, + options["grid_point"].lower(), + box, + options["save3D"], + options["basins"], + options["grid"], + options["frequency"], + ) ) - ) return job_list def request_data(self): @@ -185,6 +189,7 @@ class RegionSum(Diagnostic): self.member, self.chunk, grid=self.grid, + frequency=self.frequency ) def declare_data_generated(self): @@ -202,27 +207,10 @@ class RegionSum(Diagnostic): has_levels = self._fix_file_metadata() data = self._load_data() masks = {} - self.basins.sort() for basin in self.basins: masks[basin] = Utils.get_mask(basin) mesh = Nemo("mesh_hgr.nc", "mask_regions.nc") - if ( - self.box.min_lat is not -1 - and self.box.max_lat is not -1 - and self.box.min_lon is not -1 - and self.box.max_lat is not -1 - ): - name = "{0}_{1}".format( - Box.get_lat_str(self.box), Box.get_lon_str(self.box) - ) - - masks[name] = mesh.get_region_mask( - self.box.min_lat, - self.box.max_lat, - self.box.min_lon, - self.box.max_lon, - ) if has_levels: self._sum_3d_variable(data, mesh, masks) else: @@ -230,7 +218,7 @@ class RegionSum(Diagnostic): def _sum_2d_var(self, data, mesh, masks): areacello = mesh.get_areacello(cell_point=self.grid_point) - varsum = regsum.compute_regsum_2D(data.data, masks, areacello) + varsum = regsum.compute_regsum_2d(data.data, masks, areacello) self._save_result_2d("sum", varsum, data) def _sum_3d_variable(self, data, mesh, masks): @@ -249,13 +237,13 @@ class RegionSum(Diagnostic): data = data.extract(depth_constraint) tmask = tmask.extract(depth_constraint) volcello = areacello * e3.data.astype(np.float32) - varsum = regsum.compute_regsum_3D( + varsum = regsum.compute_regsum_3d( data.data, masks, volcello, tmask.data ) self._save_result_2d("sum", varsum, data) if self.save3d: varsum3d = regsum.compute_regsum_levels( - data.data, masks, volcello, tmask + data.data, masks, volcello, tmask.data ) self._save_result_3d("sum", varsum3d, data) @@ -350,7 +338,7 @@ class RegionSum(Diagnostic): var = "sum" if threed: if not self.save3d: - return False + return original_name = "{0}_{1}".format(var, self.variable) final_name = "{1}3d{0}".format(var, self.variable) levels = ",lev" @@ -386,7 +374,7 @@ class RegionSum(Diagnostic): def _declare_var(self, var, threed, box_save): if threed: if not self.save3d: - return False + return final_name = "{1}3d{0}".format(var, self.variable) else: final_name = "{1}{0}".format(var, self.variable) @@ -400,6 +388,7 @@ class RegionSum(Diagnostic): box=box_save, region=self.basins, grid=self.grid, + frequency=self.frequency, ) def _save_result_2d(self, var, result, data): diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py index 4f80873111afed92245640f96184041a9a63b6a2..695c9ca212a7ea2ce8ae5026902f847f4efdfaef 100644 --- a/earthdiagnostics/publisher.py +++ b/earthdiagnostics/publisher.py @@ -30,7 +30,7 @@ class Publisher(object): :param who: suscriber to remove :type who: object """ - del self._subscribers[who] + self._subscribers.pop(who) def dispatch(self, *args): """ diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index e62c5f0d12a2f4e3b7199dabaa4d64ce0206751a..44a5ad014b157935c21117a6098eb9bf908a6eff 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -56,7 +56,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "t", Frequencies.monthly, @@ -73,7 +72,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "t", Frequencies.monthly, @@ -95,7 +93,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "u", Frequencies.monthly, @@ -112,7 +109,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "u", Frequencies.monthly, @@ -134,7 +130,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "u", Frequencies.monthly, @@ -151,7 +146,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "u", Frequencies.monthly, @@ -178,7 +172,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "u", Frequencies.monthly, @@ -195,7 +188,6 @@ class TestRegionMean(TestCase): "var", box, True, - False, Basins().Global, "u", Frequencies.monthly, @@ -227,7 +219,6 @@ class TestRegionMean(TestCase): "var", box, False, - False, Basins().Global, "u", Frequencies.monthly, @@ -244,7 +235,6 @@ class TestRegionMean(TestCase): "var", box, False, - False, Basins().Global, "u", Frequencies.monthly, @@ -262,57 +252,6 @@ class TestRegionMean(TestCase): "1", "10", "false", - "True", - ], - ) - self.assertEqual(len(jobs), 2) - self.assertEqual( - jobs[0], - RegionMean( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.ocean, - "var", - box, - False, - True, - Basins().Global, - "u", - Frequencies.monthly, - ), - ) - self.assertEqual( - jobs[1], - RegionMean( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.ocean, - "var", - box, - False, - True, - Basins().Global, - "u", - Frequencies.monthly, - ), - ) - - jobs = RegionMean.generate_jobs( - self.diags, - [ - "diagnostic", - "ocean", - ["var"], - "global", - "U", - "1", - "10", - "false", - "True", "grid", ], ) @@ -329,7 +268,6 @@ class TestRegionMean(TestCase): box, False, Basins().Global, - True, "grid", Frequencies.monthly, ), @@ -346,7 +284,6 @@ class TestRegionMean(TestCase): box, False, Basins().Global, - True, "grid", Frequencies.monthly, ), @@ -363,7 +300,6 @@ class TestRegionMean(TestCase): "1", "10", "false", - "True", "grid", "day", ], @@ -381,7 +317,6 @@ class TestRegionMean(TestCase): box, False, Basins().Global, - True, "grid", Frequencies.daily, ), @@ -398,7 +333,6 @@ class TestRegionMean(TestCase): box, False, Basins().Global, - True, "grid", Frequencies.daily, ), @@ -419,7 +353,6 @@ class TestRegionMean(TestCase): "1", "10", "false", - "True", "grid", "day", "extra", @@ -439,13 +372,12 @@ class TestRegionMean(TestCase): "var", box, False, - True, - Basins().Global, + [Basins().Global], "T", Frequencies.monthly, ) self.assertEqual( str(diag), "Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var " - "Box: 1-10 Save 3D: False Save variance: True Grid point: T", + "Box: 1-10 Save 3D: False Grid point: T", ) diff --git a/test/unit/test_lint.py b/test/unit/test_lint.py deleted file mode 100644 index f4d8da73ee013208cc8a1d6824abe769f4e553cb..0000000000000000000000000000000000000000 --- a/test/unit/test_lint.py +++ /dev/null @@ -1,32 +0,0 @@ -""" Lint tests """ -import os -import unittest - -import pycodestyle # formerly known as pep8 - - -class TestLint(unittest.TestCase): - def test_pep8_conformance(self): - """Test that we conform to PEP-8.""" - - check_paths = [ - "earthdiagnostics", - "test", - ] - exclude_paths = ["earthdiagnostics/cmor_tables/primavera"] - - print("PEP8 check of directories: {}\n".format(", ".join(check_paths))) - - # Get paths wrt package root - package_root = os.path.dirname( - os.path.dirname(os.path.dirname(__file__)) - ) - for paths in (check_paths, exclude_paths): - for i, path in enumerate(paths): - paths[i] = os.path.join(package_root, path) - - style = pycodestyle.StyleGuide() - style.options.exclude.extend(exclude_paths) - style.options.max_line_length = 120 - - self.assertEqual(style.check_files(check_paths).total_errors, 0)