From c4af905739d6c48a5155ea557f6dedf0bc31352e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 14 Oct 2020 11:39:55 +0200 Subject: [PATCH 1/5] Do not use fixed region length --- earthdiagnostics/ocean/regionmean.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index b1a48300..db285f4d 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -89,6 +89,9 @@ class RegionMean(Diagnostic): self.lat_name = "lat" self.lon_name = "lon" + self._length = max(len(basin.name) for basin in self.basins) + Log.debug(f'Max basin length: {self._length}') + def __eq__(self, other): if self._different_type(other): return False @@ -315,22 +318,23 @@ class RegionMean(Diagnostic): def _save_result_2d(self, var, result, data): final_name = "{1}{0}".format(var, self.variable) + temp = TempFile.get() handler_source = Utils.open_cdf(self.variable_file.local_file) handler_temp = Utils.open_cdf(temp, "w") Utils.copy_variable(handler_source, handler_temp, "time", True, True) handler_temp.createDimension("region", len(result)) - handler_temp.createDimension("region_length", 50) + handler_temp.createDimension("region_length", self._length) var_region = handler_temp.createVariable( "region", "S1", ("region", "region_length") ) var = handler_temp.createVariable( - "{1}{0}".format(var, self.variable), float, ("time", "region",), + "{1}{0}".format(var, self.variable), float, ("region", "time"), ) var.units = "{0}".format(data.units) for i, basin in enumerate(result): - var_region[i, ...] = netCDF4.stringtoarr(str(basin), 50) - var[..., i] = result[basin] + var_region[i, ...] = netCDF4.stringtoarr(str(basin), self._length) + var[i, ...] = result[basin] handler_temp.close() self.declared[final_name].set_local_file(temp, diagnostic=self) @@ -341,7 +345,7 @@ class RegionMean(Diagnostic): handler_temp = Utils.open_cdf(temp, "w") Utils.copy_variable(handler_source, handler_temp, "time", True, True) handler_temp.createDimension("region", len(result)) - handler_temp.createDimension("region_length", 50) + handler_temp.createDimension("region_length", self._length) handler_temp.createDimension("lev", data.shape[1]) var_level = handler_temp.createVariable("lev", float, "lev") var_level[...] = data.coord("depth").points @@ -362,7 +366,7 @@ class RegionMean(Diagnostic): var.units = "{0}".format(data.units) for i, basin in enumerate(result): - var_region[i, ...] = netCDF4.stringtoarr(str(basin), 50) + var_region[i, ...] = netCDF4.stringtoarr(str(basin), self._length) var[..., i] = result[basin] handler_temp.close() self.declared[final_name].set_local_file(temp, diagnostic=self) -- GitLab From 9159a514088fbd485a7eae11e1492c9a21bb56b1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 3 Nov 2020 09:58:34 +0100 Subject: [PATCH 2/5] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6ce49394..6361e20a 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ REQUIREMENTS = { "cdo>=1.3.4", "cfgrib", "dask[array]", - "diagonals>=0.3.1", + "diagonals>=0.3.2", "netCDF4", "nco>=0.0.3", "numba", -- GitLab From 6b577064d07477b72875b4caa1cf822c1f513249 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 3 Nov 2020 10:03:01 +0100 Subject: [PATCH 3/5] Fix sivol --- earthdiagnostics/ocean/sivolume.py | 93 ++++++++++++++++-------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/earthdiagnostics/ocean/sivolume.py b/earthdiagnostics/ocean/sivolume.py index 75ba01cb..2f1fe3d0 100644 --- a/earthdiagnostics/ocean/sivolume.py +++ b/earthdiagnostics/ocean/sivolume.py @@ -60,6 +60,7 @@ class Sivolume(Diagnostic): self.sivoln = {} self.sivols = {} + def __str__(self): return ( "Sivolume Startdate: {0.startdate} Member: {0.member} " @@ -83,7 +84,7 @@ class Sivolume(Diagnostic): :return: """ options_available = ( - DiagnosticBasinListOption("basins", [Basins().Global]), + DiagnosticBasinListOption("basins", Basins().Global.name), ) options = cls.process_options(options, options_available) @@ -200,48 +201,52 @@ class Sivolume(Diagnostic): def _save_file(self, data, var): generated_file = self.generated[var] temp = TempFile.get() - region = data.coord("region").points - data.remove_coord("region") + data.remove_coord('latitude') + data.remove_coord('longitude') + data.remove_coord(data.coord(var_name='i')) + data.remove_coord(data.coord(var_name='j')) iris.save(data, temp, zlib=True) - if len(region) > 1: - Utils.rename_variable(temp, "dim0", "region", False) - handler = Utils.open_cdf(temp) - var = handler.createVariable("region2", str, ("region",)) - var[...] = region - handler.close() - Utils.rename_variable(temp, "region2", "region", True) - else: - handler = Utils.open_cdf(temp) - if "region" not in handler.dimensions: - new_file = TempFile.get() - new_handler = Utils.open_cdf(new_file, "w") - - new_handler.createDimension("region", 1) - for dimension in handler.dimensions: - Utils.copy_dimension(handler, new_handler, dimension) - - for variable in handler.variables.keys(): - if variable in (var, "region"): - continue - Utils.copy_variable(handler, new_handler, variable) - old_var = handler.variables[var] - new_var = new_handler.createVariable( - var, - old_var.dtype, - ("region",) + old_var.dimensions, - zlib=True, - fill_value=1.0e20, - ) - Utils.copy_attributes(new_var, old_var) - new_var[0, :] = old_var[:] - - new_var = new_handler.createVariable( - "region", str, ("region",) - ) - new_var[0] = region[0] - - new_handler.close() - os.remove(temp) - temp = new_file - handler.close() + # if len(region) > 1: + # Utils.rename_variable(temp, "dim0", "region", False) + # handler = Utils.open_cdf(temp) + # handler.createDimension("region_lenth", region_length) + # var = handler.createVariable("region2", "S1", ("region", "region_length")) + # var[...] = region + # handler.close() + # Utils.rename_variable(temp, "region2", "region", True) + # else: + # handler = Utils.open_cdf(temp) + # if "region" not in handler.dimensions: + # new_file = TempFile.get() + # new_handler = Utils.open_cdf(new_file, "w") + + # new_handler.createDimension("region", 1) + # new_handler.createDimension("region_lenth", region_length) + # for dimension in handler.dimensions: + # Utils.copy_dimension(handler, new_handler, dimension) + + # for variable in handler.variables.keys(): + # if variable in (var, "region"): + # continue + # Utils.copy_variable(handler, new_handler, variable) + # old_var = handler.variables[var] + # new_var = new_handler.createVariable( + # var, + # old_var.dtype, + # ("region",) + old_var.dimensions, + # zlib=True, + # fill_value=1.0e20, + # ) + # Utils.copy_attributes(new_var, old_var) + # new_var[0, :] = old_var[:] + + # new_var = new_handler.createVariable( + # "region", "S1", ("region", "region_length") + # ) + # new_var[0, :] = region[0] + + # new_handler.close() + # os.remove(temp) + # temp = new_file + # handler.close() generated_file.set_local_file(temp) -- GitLab From 033e825c49df9ac54cd9a064cc005141527d80f7 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 3 Nov 2020 11:33:39 +0100 Subject: [PATCH 4/5] Fix tests --- test/unit/ocean/test_region_mean.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index 44a5ad01..d51dc68f 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -56,7 +56,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "t", Frequencies.monthly, ), @@ -72,7 +72,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "t", Frequencies.monthly, ), @@ -93,7 +93,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -109,7 +109,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -130,7 +130,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -146,7 +146,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -172,7 +172,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -188,7 +188,7 @@ class TestRegionMean(TestCase): "var", box, True, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -219,7 +219,7 @@ class TestRegionMean(TestCase): "var", box, False, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -235,7 +235,7 @@ class TestRegionMean(TestCase): "var", box, False, - Basins().Global, + [Basins().Global], "u", Frequencies.monthly, ), @@ -267,7 +267,7 @@ class TestRegionMean(TestCase): "var", box, False, - Basins().Global, + [Basins().Global], "grid", Frequencies.monthly, ), @@ -283,7 +283,7 @@ class TestRegionMean(TestCase): "var", box, False, - Basins().Global, + [Basins().Global], "grid", Frequencies.monthly, ), @@ -316,7 +316,7 @@ class TestRegionMean(TestCase): "var", box, False, - Basins().Global, + [Basins().Global], "grid", Frequencies.daily, ), @@ -332,7 +332,7 @@ class TestRegionMean(TestCase): "var", box, False, - Basins().Global, + [Basins().Global], "grid", Frequencies.daily, ), -- GitLab From d87798cc9dcd7b3e84bc06a5367411b9c26fe933 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 3 Nov 2020 11:53:48 +0100 Subject: [PATCH 5/5] Fix tests and format --- earthdiagnostics/ocean/psi.py | 2 +- earthdiagnostics/ocean/sivolume.py | 45 ------------------------------ test/unit/ocean/test_psi.py | 2 +- 3 files changed, 2 insertions(+), 47 deletions(-) diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index e1624b6a..b9faea87 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -58,7 +58,7 @@ class Psi(Diagnostic): ) def __str__(self): - return "PSI Startdate: {0} Member: {1} Chunk: {2}".format( + return "PSI Startdate: {0} Member: {1} Chunk: {2} Basins:{3}".format( self.startdate, self.member, self.chunk, diff --git a/earthdiagnostics/ocean/sivolume.py b/earthdiagnostics/ocean/sivolume.py index 2f1fe3d0..40b6227d 100644 --- a/earthdiagnostics/ocean/sivolume.py +++ b/earthdiagnostics/ocean/sivolume.py @@ -1,6 +1,5 @@ # coding=utf-8 """Computes the seaice extent, area and volume""" -import os import six import iris @@ -60,7 +59,6 @@ class Sivolume(Diagnostic): self.sivoln = {} self.sivols = {} - def __str__(self): return ( "Sivolume Startdate: {0.startdate} Member: {0.member} " @@ -206,47 +204,4 @@ class Sivolume(Diagnostic): data.remove_coord(data.coord(var_name='i')) data.remove_coord(data.coord(var_name='j')) iris.save(data, temp, zlib=True) - # if len(region) > 1: - # Utils.rename_variable(temp, "dim0", "region", False) - # handler = Utils.open_cdf(temp) - # handler.createDimension("region_lenth", region_length) - # var = handler.createVariable("region2", "S1", ("region", "region_length")) - # var[...] = region - # handler.close() - # Utils.rename_variable(temp, "region2", "region", True) - # else: - # handler = Utils.open_cdf(temp) - # if "region" not in handler.dimensions: - # new_file = TempFile.get() - # new_handler = Utils.open_cdf(new_file, "w") - - # new_handler.createDimension("region", 1) - # new_handler.createDimension("region_lenth", region_length) - # for dimension in handler.dimensions: - # Utils.copy_dimension(handler, new_handler, dimension) - - # for variable in handler.variables.keys(): - # if variable in (var, "region"): - # continue - # Utils.copy_variable(handler, new_handler, variable) - # old_var = handler.variables[var] - # new_var = new_handler.createVariable( - # var, - # old_var.dtype, - # ("region",) + old_var.dimensions, - # zlib=True, - # fill_value=1.0e20, - # ) - # Utils.copy_attributes(new_var, old_var) - # new_var[0, :] = old_var[:] - - # new_var = new_handler.createVariable( - # "region", "S1", ("region", "region_length") - # ) - # new_var[0, :] = region[0] - - # new_handler.close() - # os.remove(temp) - # temp = new_file - # handler.close() generated_file.set_local_file(temp) diff --git a/test/unit/ocean/test_psi.py b/test/unit/ocean/test_psi.py index c94ff3ea..d3038151 100644 --- a/test/unit/ocean/test_psi.py +++ b/test/unit/ocean/test_psi.py @@ -40,5 +40,5 @@ class TestPsi(TestCase): psi = Psi(self.data_manager, "20000101", 1, 1, {Basins().Global: None}) self.assertEqual( str(psi), - "PSI Startdate: 20000101 Member: 1 Chunk: 1", + "PSI Startdate: 20000101 Member: 1 Chunk: 1 Basins:Global", ) -- GitLab