From c40caa505eeac402c1f061d3149895c5e6799ec2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 25 Jun 2020 14:46:39 +0200 Subject: [PATCH 1/2] Support yearly and remove auto data fix from regmean --- earthdiagnostics/data_convention.py | 12 +++++- earthdiagnostics/datafile.py | 1 + earthdiagnostics/ocean/interpolatecdo.py | 8 ++++ earthdiagnostics/ocean/moc.py | 11 +++-- earthdiagnostics/ocean/regionmean.py | 51 +++++++++--------------- 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index eb137782..ee6bf7ca 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -812,8 +812,16 @@ class Cmor3Convention(DataConvention): "month", self.config.experiment.calendar, ) - - if frequency == Frequencies.monthly: + if frequency == Frequencies.yearly: + chunk_end = previous_day( + chunk_end, self.config.experiment.calendar + ) + time_bound = "{0:04}{2}{1:04}".format( + chunk_start.year, + chunk_end.year, + self.time_separator, + ) + elif frequency == Frequencies.monthly: chunk_end = previous_day( chunk_end, self.config.experiment.calendar ) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 1390a973..54a0c4b1 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -474,6 +474,7 @@ class DataFile(Publisher): if len(cube_list) == 1: return iris.experimental.equalise_cubes.equalise_attributes(cube_list) + iris.util.unify_time_units(cube_list) final_cube = cube_list.merge_cube() temp = TempFile.get() iris.save(final_cube, temp, zlib=True) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index dd514e1d..2a23075f 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -12,6 +12,7 @@ from earthdiagnostics.diagnostic import ( DiagnosticChoiceOption, DiagnosticBoolOption, DiagnosticOption, + DiagnosticFrequencyOption, ) from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile @@ -66,6 +67,7 @@ class InterpolateCDO(Diagnostic): mask_oceans, original_grid, weights, + frequency, ): Diagnostic.__init__(self, data_manager) self.startdate = startdate @@ -81,6 +83,7 @@ class InterpolateCDO(Diagnostic): self.mask_oceans = mask_oceans self.original_grid = original_grid self.weights = weights + self.frequency = frequency def __eq__(self, other): if self._different_type(other): @@ -134,6 +137,7 @@ class InterpolateCDO(Diagnostic): DiagnosticBoolOption("mask_oceans", True), DiagnosticOption("original_grid", ""), DiagnosticBoolOption("weights_from_mask", True), + DiagnosticFrequencyOption(), ) options = cls.process_options(options, options_available) target_grid = cls._translate_ifs_grids_to_cdo_names( @@ -166,6 +170,7 @@ class InterpolateCDO(Diagnostic): options["original_grid"], weights, options["method"], + options["frequency"], ) for var in options["variables"]: @@ -186,6 +191,7 @@ class InterpolateCDO(Diagnostic): options["mask_oceans"], options["original_grid"], weights, + options["frequency"] ) if weights_job is not None: job.add_subjob(weights_job) @@ -326,6 +332,7 @@ class InterpolateCDO(Diagnostic): self.member, self.chunk, grid=self.original_grid, + frequency=self.frequency, ) def declare_data_generated(self): @@ -337,6 +344,7 @@ class InterpolateCDO(Diagnostic): self.member, self.chunk, grid=self.grid, + frequency=self.frequency, ) def compute(self): diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index f5931fc8..4f3f3c5d 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -45,6 +45,7 @@ class Moc(Diagnostic): def __init__(self, data_manager, startdate, member, chunk, basins): Diagnostic.__init__(self, data_manager) + self.data_convention = data_manager.convention self.startdate = startdate self.member = member self.chunk = chunk @@ -161,21 +162,25 @@ class Moc(Diagnostic): "region", "S1", ("region", "region_length") ) - lat = handler_temp.createVariable("lat", float, ("j", "i")) + lat = handler_temp.createVariable( + self.data_convention.lat_name, float, ("j", "i")) lat[...] = gphiv[:, max_gphiv] lat.units = "degrees_north" lat.long_name = "Latitude" + lat.standard_name = "latitude" - lon = handler_temp.createVariable("lon", float, ("j", "i")) + lon = handler_temp.createVariable( + self.data_convention.lon_name, float, ("j", "i")) lon[...] = 0 lon.units = "degrees_east" lon.long_name = "Longitude" + lon.standard_name = "longitude" var = handler_temp.createVariable( "vsftmyz", float, ("time", "lev", "i", "j", "region") ) var.units = "Sverdrup" - var.coordinates = "lev time" + var.coordinates = "lev time latitude longitude" var.long_name = "Ocean meridional overturning volume streamfunction" var.missing_value = 1e20 var.fill_value = 1e20 diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index fe9cc813..81f2b674 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -253,25 +253,25 @@ class RegionMean(Diagnostic): return cube def _load_data(self): - coords = [] - handler = Utils.open_cdf(self.variable_file.local_file) - for variable in handler.variables: - if variable in ( - "time", - "lev", - "lat", - "lon", - "latitude", - "longitude", - "leadtime", - "time_centered", - ): - coords.append(variable) - if variable == "time_centered": - handler.variables[variable].standard_name = "" - - handler.variables[self.variable].coordinates = " ".join(coords) - handler.close() + # handler = Utils.open_cdf(self.variable_file.local_file) + # coords = set(handler.variables[self.variable].coordinates.split(' ')) + # for variable in handler.variables: + # if variable in ( + # "time", + # "lev", + # "lat", + # "lon", + # "latitude", + # "longitude", + # "leadtime", + # "time_centered", + # ): + # coords.add(variable) + # if variable == "time_centered": + # handler.variables[variable].standard_name = "" + + # handler.variables[self.variable].coordinates = " ".join(coords) + # handler.close() data = iris.load_cube(self.variable_file.local_file) return self._rename_depth(data) @@ -288,23 +288,10 @@ class RegionMean(Diagnostic): def _fix_file_metadata(self): handler = Utils.open_cdf(self.variable_file.local_file) var = handler.variables[self.variable] - coordinates = "" has_levels = False for dimension in handler.variables.keys(): - if dimension in [ - "time", - "lev", - "lat", - "latitude", - "lon", - "longitude", - "i", - "j", - ]: - coordinates += " {0}".format(dimension) if dimension == "lev": has_levels = True - var.coordinates = coordinates handler.close() return has_levels -- GitLab From 378905b5ffa318160b004927ca31829067517e67 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 25 Jun 2020 19:57:16 +0200 Subject: [PATCH 2/2] Fix tests --- earthdiagnostics/ocean/interpolatecdo.py | 8 ++- earthdiagnostics/ocean/regionmean.py | 1 - test/unit/ocean/test_interpolatecdo.py | 68 +++++++++++++++++++++++- 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 2a23075f..3de0f28a 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -99,6 +99,7 @@ class InterpolateCDO(Diagnostic): and self.mask_oceans == other.mask_oceans and self.grid == other.grid and self.original_grid == other.original_grid + and self.frequency == other.frequency ) def __hash__(self): @@ -108,6 +109,7 @@ class InterpolateCDO(Diagnostic): return ( "Interpolate with CDO Startdate: {0.startdate} Member: {0.member} " "Chunk: {0.chunk} Variable: {0.domain}:{0.variable} " + "Frequency: {0.frequency} " "Target grid: {0.grid} Original grid: {0.original_grid} " "Mask ocean: {0.mask_oceans} Model: {0.model_version}".format(self) ) @@ -137,7 +139,8 @@ class InterpolateCDO(Diagnostic): DiagnosticBoolOption("mask_oceans", True), DiagnosticOption("original_grid", ""), DiagnosticBoolOption("weights_from_mask", True), - DiagnosticFrequencyOption(), + DiagnosticFrequencyOption( + default_value=diags.config.frequency), ) options = cls.process_options(options, options_available) target_grid = cls._translate_ifs_grids_to_cdo_names( @@ -459,6 +462,7 @@ class ComputeWeights(Diagnostic): original_grid, weights_file, method, + frequency ): Diagnostic.__init__(self, data_manager) self.startdate = startdate @@ -470,6 +474,7 @@ class ComputeWeights(Diagnostic): self.original_grid = original_grid self.weights_file = weights_file self.method = method + self.frequency = frequency def __str__(self): return ( @@ -495,6 +500,7 @@ class ComputeWeights(Diagnostic): self.member, self.chunk, grid=self.original_grid, + frequency=self.frequency, ) def declare_data_generated(self): diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 81f2b674..b1a48300 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -287,7 +287,6 @@ class RegionMean(Diagnostic): def _fix_file_metadata(self): handler = Utils.open_cdf(self.variable_file.local_file) - var = handler.variables[self.variable] has_levels = False for dimension in handler.variables.keys(): if dimension == "lev": diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index cc2d86ae..5b546dd9 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -5,6 +5,7 @@ from earthdiagnostics.ocean.interpolatecdo import InterpolateCDO from mock import Mock, patch from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.frequency import Frequencies from earthdiagnostics.diagnostic import ( DiagnosticVariableListOption, DiagnosticOptionError, @@ -23,6 +24,7 @@ class TestInterpolate(TestCase): ) self.diags.config.experiment.model_version = "model_version" self.diags.config.experiment.atmos_grid = "atmos_grid" + self.diags.config.frequency = Frequencies.daily def fake_parse(self, value): if not value: @@ -63,6 +65,7 @@ class TestInterpolate(TestCase): True, "", None, + Frequencies.daily, ), ) self.assertEqual( @@ -79,6 +82,7 @@ class TestInterpolate(TestCase): True, "", None, + Frequencies.daily, ), ) @@ -100,6 +104,7 @@ class TestInterpolate(TestCase): True, "", None, + Frequencies.daily, ), ) self.assertEqual( @@ -116,6 +121,7 @@ class TestInterpolate(TestCase): True, "", None, + Frequencies.daily, ), ) @@ -137,6 +143,7 @@ class TestInterpolate(TestCase): True, "", None, + Frequencies.daily, ), ) self.assertEqual( @@ -153,6 +160,7 @@ class TestInterpolate(TestCase): True, "", None, + Frequencies.daily, ), ) @@ -175,6 +183,7 @@ class TestInterpolate(TestCase): False, "", None, + Frequencies.daily, ), ) self.assertEqual( @@ -191,6 +200,7 @@ class TestInterpolate(TestCase): False, "", None, + Frequencies.daily, ), ) @@ -221,6 +231,7 @@ class TestInterpolate(TestCase): False, "orig", None, + Frequencies.daily, ), ) self.assertEqual( @@ -237,6 +248,7 @@ class TestInterpolate(TestCase): False, "orig", None, + Frequencies.daily, ), ) @@ -268,6 +280,7 @@ class TestInterpolate(TestCase): False, "orig", None, + Frequencies.daily, ), ) self.assertEqual( @@ -284,6 +297,57 @@ class TestInterpolate(TestCase): False, "orig", None, + Frequencies.daily, + ), + ) + + jobs = InterpolateCDO.generate_jobs( + self.diags, + [ + "interpcdo", + "ocean", + "var", + "target_grid", + "bicubic", + "false", + "orig", + "false", + "mon" + ], + ) + self.assertEqual(len(jobs), 2) + self.assertEqual( + jobs[0], + InterpolateCDO( + self.data_manager, + "20010101", + 0, + 0, + ModelingRealms.ocean, + "var", + "target_grid", + "model_version", + False, + "orig", + None, + Frequencies.monthly, + ), + ) + self.assertEqual( + jobs[1], + InterpolateCDO( + self.data_manager, + "20010101", + 0, + 1, + ModelingRealms.ocean, + "var", + "target_grid", + "model_version", + False, + "orig", + None, + Frequencies.monthly, ), ) @@ -301,6 +365,7 @@ class TestInterpolate(TestCase): "false", "orig", "false", + "daily," "extra", ], ) @@ -318,11 +383,12 @@ class TestInterpolate(TestCase): False, "orig", None, + Frequencies.monthly, ) self.assertEqual( str(diag), "Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 " - "Variable: ocean:var " + "Variable: ocean:var Frequency: mon " "Target grid: atmos_grid Original grid: orig Mask ocean: False " "Model: model_version", ) -- GitLab