From c57c2ea298fc35cbb53c65f9325ef9717b6130d2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 26 Feb 2021 12:38:10 +0100 Subject: [PATCH 1/9] Fix dim names in density --- earthdiagnostics/ocean/density.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/ocean/density.py b/earthdiagnostics/ocean/density.py index 69b070c4..cb144610 100644 --- a/earthdiagnostics/ocean/density.py +++ b/earthdiagnostics/ocean/density.py @@ -5,7 +5,7 @@ import numpy as np import iris from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.utils import TempFile +from earthdiagnostics.utils import TempFile, Utils from earthdiagnostics.diagnostic import Diagnostic @@ -147,4 +147,12 @@ class Density(Diagnostic): iris.save(sigma_cube, temp, zlib=True, fill_value=1e20) del sigma_cube del sigma_values + Utils.rename_variables( + temp, + { + "dim2": "j", + "dim3": "i", + }, + must_exist=False, + ) self.sigma[sigma].set_local_file(temp) -- GitLab From 993c3c8a82590fd68c0c78961d435bebe56cea0e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 12 Mar 2021 13:02:24 +0100 Subject: [PATCH 2/9] Add nested alias support --- earthdiagnostics/config.py | 47 +++++++++++++++++++++++--------------- test/unit/test_config.py | 8 +++++++ 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 8d373efa..cb21d9b6 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -213,9 +213,11 @@ class Config(object): ) Log.debug("Preparing command list") commands = self._diags.split() - self._real_commands = list() - self._apply_aliases(commands) - Log.debug("Command list ready ") + while self._apply_aliases(commands): + pass + Log.debug("Command list ready: ") + for command in self._real_commands: + Log.debug(command) self.scratch_dir = os.path.join( self.scratch_dir, "diags", self.experiment.expid @@ -226,22 +228,29 @@ class Config(object): self.report = ReportConfig(parser) def _apply_aliases(self, commands): - for command in commands: - command = command.strip() - if command.startswith("#"): - break - if command.lower() in self._aliases: - added_commands = self._aliases[command.lower()] - Log.info( - "Changing alias {0} for {1}", - command, - " ".join(added_commands), - ) - for add_command in added_commands: - if add_command: - self._real_commands.append(add_command) - else: - self._real_commands.append(command) + applied_alias = True + while applied_alias: + real_commands = [] + applied_alias = False + for command in commands: + command = command.strip() + if command.startswith("#"): + break + if command.lower() in self._aliases: + added_commands = self._aliases[command.lower()] + Log.info( + "Changing alias {0} for {1}", + command, + " ".join(added_commands), + ) + for add_command in added_commands: + if add_command: + real_commands.append(add_command) + applied_alias = True + else: + real_commands.append(command) + commands = real_commands + self._real_commands = commands def _parse_dataconvention(self, parser): data_convention = parser.get_choice_option( diff --git a/test/unit/test_config.py b/test/unit/test_config.py index b8bc4cc0..6d2bbca8 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -752,6 +752,14 @@ class TestConfig(TestCase): self._parse(config) self.assertEqual(config.get_commands(), ["diag2"]) + def test_nested_alias(self): + """Test alias parsing""" + config = Config() + self.mock_parser.add_value("ALIAS", "nested", "diag3") + self.mock_parser.add_value("ALIAS", "diag2", "nested diag4") + self._parse(config) + self.assertEqual(config.get_commands(), ["diag1", "diag3", "diag4"]) + def test_auto_clean_ram_disk(self): """Test that USE_RAMDISK forces AUTO_CLEAN to true""" config = Config() -- GitLab From 00f23cc8fd9d7d2a4988690a84147b495091af47 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 12 Mar 2021 13:07:03 +0100 Subject: [PATCH 3/9] Initialize CDO on the main thread interpolation --- earthdiagnostics/ocean/interpolatecdo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 42dd17a9..dcf8dd53 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -126,6 +126,8 @@ class InterpolateCDO(Diagnostic): :type options: list[str] :return: """ + # Initialize CDO on the main thread + Utils.cdo() options_available = ( DiagnosticDomainOption(default_value=ModelingRealms.ocean), DiagnosticVariableListOption( -- GitLab From a978bf578768f57b8462bce1629b7f8da9753602 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 23 Mar 2021 11:57:57 +0100 Subject: [PATCH 4/9] Fix clean exit if all diags were wrongly configured --- earthdiagnostics/work_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 9192b58c..ad272789 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -94,7 +94,7 @@ class WorkManager(object): and not self.jobs[DiagnosticStatus.READY] ): Log.result("No diagnostics to run") - return True + return not self.had_errors start_time = datetime.datetime.now() Log.info("Starting to compute at {0}", start_time) -- GitLab From be7e53b2de4314f51e0324517434ad05f87b3d73 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 7 Apr 2021 21:45:51 +0200 Subject: [PATCH 5/9] Fix package versions --- environment.yml | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index f6657b17..2808ec2d 100644 --- a/environment.yml +++ b/environment.yml @@ -5,9 +5,9 @@ channels: - conda-forge dependencies: -- python=3.7.4 +- python<3.9 - cdo - nco - eccodes - six -- iris>=2.4 +- iris>=3.0.1 diff --git a/setup.py b/setup.py index 6361e20a..074466cb 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ REQUIREMENTS = { ], "install": [ "bscearth.utils", - "cdo>=1.3.4", + "cdo==1.5.3", "cfgrib", "dask[array]", "diagonals>=0.3.2", @@ -31,7 +31,7 @@ REQUIREMENTS = { "numpy", "psutil", "openpyxl", - "scitools-iris>=2.4", + "scitools-iris>=3.0.1", "six", "xxhash", ], -- GitLab From 29753ddfcdd2ca1197d7a1ea60e2a96390f586ae Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Apr 2021 11:15:47 +0200 Subject: [PATCH 6/9] Fix call to equalise_attributes --- earthdiagnostics/datafile.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 54a0c4b1..c6e47284 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -8,7 +8,6 @@ import iris import iris.cube import iris.coords import iris.exceptions -import iris.experimental.equalise_cubes from bscearth.utils.log import Log from earthdiagnostics.modelingrealm import ModelingRealms @@ -473,7 +472,7 @@ class DataFile(Publisher): ) if len(cube_list) == 1: return - iris.experimental.equalise_cubes.equalise_attributes(cube_list) + iris.util.equalise_attributes(cube_list) iris.util.unify_time_units(cube_list) final_cube = cube_list.merge_cube() temp = TempFile.get() -- GitLab From e453ce6595f922be2821b5c209a685703ba889c3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 4 Jun 2021 13:24:05 +0000 Subject: [PATCH 7/9] Fix sivolume issues in HR experiments --- earthdiagnostics/ocean/sivolume.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/earthdiagnostics/ocean/sivolume.py b/earthdiagnostics/ocean/sivolume.py index 40b6227d..39c4a38c 100644 --- a/earthdiagnostics/ocean/sivolume.py +++ b/earthdiagnostics/ocean/sivolume.py @@ -98,7 +98,7 @@ class Sivolume(Diagnostic): area = e1t * e2t for basin in basins: - masks[basin] = Utils.get_mask(basin) * area.data + masks[basin] = Utils.get_mask(basin) * area.core_data() job_list = list() for ( @@ -175,19 +175,19 @@ class Sivolume(Diagnostic): condition = data.coord("latitude").points > 0 else: condition = data.coord("latitude").points < 0 - weights = ( - iris.util.broadcast_to_shape( - condition, data.shape, data.coord_dims("latitude") - ) - * mask - ) + new_mask = condition * mask + if new_mask.max() == 0: + return None + data = data * new_mask return data.collapsed( - ("latitude", "longitude"), iris.analysis.SUM, weights=weights + ("latitude", "longitude"), iris.analysis.SUM ) def save(self, var, results): cubes = iris.cube.CubeList() for basin, result in six.iteritems(results): + if result is None: + continue result.var_name = var result.units = "m^3" result.add_aux_coord( -- GitLab From af349fa3126db63a72b75f1ea1395df943303e1e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 4 Jun 2021 15:34:02 +0200 Subject: [PATCH 8/9] Bump version --- VERSION | 2 +- doc/source/conf.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 444877d4..65afb3b8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.5.3 +3.5.4 diff --git a/doc/source/conf.py b/doc/source/conf.py index 9e05183a..77faeee9 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -65,7 +65,7 @@ copyright = u"2020, BSC-CNS Earth Sciences Department" # The short X.Y version. version = "3.5" # The full version, including alpha/beta/rc tags. -release = "3.5.3" +release = "3.5.4" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 074466cb..83633e79 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ REQUIREMENTS = { "cdo==1.5.3", "cfgrib", "dask[array]", - "diagonals>=0.3.2", + "diagonals>=0.3.4", "netCDF4", "nco>=0.0.3", "numba", -- GitLab From e2b6b9ac662028ec4238b394a3ddb801c1d9bc87 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 8 Jun 2021 11:17:28 +0000 Subject: [PATCH 9/9] Use dask in regmean --- earthdiagnostics/ocean/regionmean.py | 34 +++++++--------------------- earthdiagnostics/ocean/regionsum.py | 2 +- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index e01b5a0a..e2d2db10 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -205,7 +205,7 @@ class RegionMean(Diagnostic): def _mean_2d_var(self, data, mesh, masks): areacello = mesh.get_areacello(cell_point=self.grid_point) - mean = regmean.compute_regmean_2d(data.data, masks, areacello) + mean = regmean.compute_regmean_2d(data.core_data(), masks, areacello) self._save_result_2d("mean", mean, data) def _meand_3d_variable(self, data, mesh, masks): @@ -213,20 +213,20 @@ class RegionMean(Diagnostic): e3 = self._try_load_cube(3) e3 = self._rename_depth(e3) e3.coord("depth").bounds = data.coord("depth").bounds - if self.box.min_depth is not -1 and self.box.max_depth is not -1: + if self.box.min_depth != -1 and self.box.max_depth != -1: depth_constraint = iris.Constraint( depth=lambda c: self.box.min_depth <= c <= self.box.max_depth ) e3 = e3.extract(depth_constraint) data = data.extract(depth_constraint) - if self.box.min_depth is -1 and self.box.max_depth is not -1: + if self.box.min_depth == -1 and self.box.max_depth != -1: self.box.min_depth = 0 depth_constraint = iris.Constraint( depth=lambda c: self.box.min_depth <= c <= self.box.max_depth ) e3 = e3.extract(depth_constraint) data = data.extract(depth_constraint) - if self.box.min_depth is not -1 and self.box.max_depth is -1: + if self.box.min_depth != -1 and self.box.max_depth == -1: self.box.max_depth = 6000 depth_constraint = iris.Constraint( depth=lambda c: self.box.min_depth <= c <= self.box.max_depth @@ -234,10 +234,12 @@ class RegionMean(Diagnostic): e3 = e3.extract(depth_constraint) data = data.extract(depth_constraint) volcello = areacello * e3.data.astype(np.float32) - mean = regmean.compute_regmean_3d(data.data, masks, volcello) + mean = regmean.compute_regmean_3d(data.core_data(), masks, volcello) self._save_result_2d("mean", mean, data) if self.save3d: - mean3d = regmean.compute_regmean_levels(data.data, masks, volcello) + mean3d = regmean.compute_regmean_levels( + data.core_data(), masks, volcello + ) self._save_result_3d("mean", mean3d, data) def _try_load_cube(self, number): @@ -270,26 +272,6 @@ class RegionMean(Diagnostic): return cube def _load_data(self): - # handler = Utils.open_cdf(self.variable_file.local_file) - # coords = set(handler.variables[self.variable].coordinates.split(' ')) - # for variable in handler.variables: - # if variable in ( - # "time", - # "lev", - # "lat", - # "lon", - # "latitude", - # "longitude", - # "leadtime", - # "time_centered", - # ): - # coords.add(variable) - # if variable == "time_centered": - # handler.variables[variable].standard_name = "" - - # handler.variables[self.variable].coordinates = " ".join(coords) - # handler.close() - data = iris.load_cube(self.variable_file.local_file) return self._rename_depth(data) diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index ded849f6..b89cc195 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -229,7 +229,7 @@ class RegionSum(Diagnostic): e3 = self._try_load_cube(3) e3 = self._rename_depth(e3) e3.coord("depth").bounds = data.coord("depth").bounds - if self.box.min_depth is not -1 and self.box.max_depth is not -1: + if self.box.min_depth != -1 and self.box.max_depth != -1: depth_constraint = iris.Constraint( depth=lambda c: self.box.min_depth <= c <= self.box.max_depth ) -- GitLab