From 983cdff05e222356a11cf0bc9bfbc131acee35ba Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 15:57:26 +0200 Subject: [PATCH 1/4] Clean code --- earthdiagnostics/cmormanager.py | 2 +- earthdiagnostics/ocean/siasiesiv.py | 1 - earthdiagnostics/work_manager.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 4142ab09..5ae6f881 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -11,7 +11,7 @@ from earthdiagnostics.datafile import StorageStatus from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.cmorizer import Cmorizer from earthdiagnostics.datamanager import DataManager -from earthdiagnostics.frequency import Frequencies, Frequency +from earthdiagnostics.frequency import Frequencies from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import TempFile, Utils from earthdiagnostics.variable import VariableType diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 99979bb2..1771332a 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,7 +1,6 @@ # coding=utf-8 """Compute the sea ice extent , area and volume in both hemispheres or a specified region""" import six -import numpy as np import iris import iris.analysis diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index a6aff46c..7bc60427 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -5,7 +5,6 @@ import datetime import operator import sys import threading -import time import traceback # noinspection PyCompatibility from concurrent.futures import ThreadPoolExecutor -- GitLab From 9d1dfb5f9ffb7b7ef13a7e956d96ffdb6d6c0409 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 28 May 2018 13:28:39 +0200 Subject: [PATCH 2/4] Fixed siasiesiv for one region case --- earthdiagnostics/ocean/siasiesiv.py | 48 +++++++++++++++++++++++------ earthdiagnostics/utils.py | 30 ++++++++++-------- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 1771332a..6c5d3385 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,5 +1,6 @@ # coding=utf-8 """Compute the sea ice extent , area and volume in both hemispheres or a specified region""" +import os import six import iris @@ -132,7 +133,7 @@ class Siasiesiv(Diagnostic): handler.variables[self.sic_varname].coordinates = coordinates handler.close() sic = iris.load_cube(self.sic.local_file) - if sic.units.origin == '%' and sic.data.max < 2: + if sic.units.origin == '%' and sic.data.max() < 2: sic.units = '1.0' sic.convert_units('1.0') sic *= Siasiesiv.area.data @@ -180,17 +181,46 @@ class Siasiesiv(Diagnostic): results.append(result) if not results: continue - self._save_file(results.merge_cube(), self.generated[var]) + self._save_file(results.merge_cube(), var) - def _save_file(self, data, generated_file): + def _save_file(self, data, var): + generated_file = self.generated[var] temp = TempFile.get() region = data.coord('region').points data.remove_coord('region') iris.save(data, temp, zlib=True) - Utils.rename_variable(temp, 'dim0', 'region', False) - handler = Utils.open_cdf(temp) - var = handler.createVariable('region2', str, ('region',)) - var[...] = region - handler.close() - Utils.rename_variable(temp, 'region2', 'region', True) + if len(region) > 1: + Utils.rename_variable(temp, 'dim0', 'region', False) + handler = Utils.open_cdf(temp) + var = handler.createVariable('region2', str, ('region',)) + var[...] = region + handler.close() + Utils.rename_variable(temp, 'region2', 'region', True) + else: + handler = Utils.open_cdf(temp) + if 'region' not in handler.dimensions: + new_file = TempFile.get() + new_handler = Utils.open_cdf(new_file, 'w') + + new_handler.createDimension('region', 1) + for dimension in handler.dimensions: + Utils.copy_dimension(handler, new_handler, dimension) + + for variable in handler.variables.keys(): + if variable in (var, 'region'): + continue + Utils.copy_variable(handler, new_handler, variable) + old_var = handler.variables[var] + new_var = new_handler.createVariable(var, old_var.dtype, ('region',) + old_var.dimensions, + zlib=True, fill_value=1.0e20) + Utils.copy_attributes(new_var, old_var) + new_var[0, :] = old_var[:] + + new_var = new_handler.createVariable('region', str, ('region',)) + new_var[0] = region[0] + + new_handler.close() + os.remove(temp) + temp = new_file + handler.close() generated_file.set_local_file(temp) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 632debe4..9f343a6a 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -109,7 +109,7 @@ class Utils(object): handler.close() @staticmethod - def rename_variable(filepath, old_name, new_name, must_exist=True): + def rename_variable(filepath, old_name, new_name, must_exist=True, rename_dimension=True): """ Rename variable from a NetCDF file @@ -127,10 +127,10 @@ class Utils(object): Utils.rename_variables """ - Utils.rename_variables(filepath, {old_name: new_name}, must_exist) + Utils.rename_variables(filepath, {old_name: new_name}, must_exist, rename_dimension) @staticmethod - def rename_variables(filepath, dic_names, must_exist=True): + def rename_variables(filepath, dic_names, must_exist=True, rename_dimension=True): """ Rename multiple variables from a NetCDF file @@ -166,9 +166,11 @@ class Utils(object): original = getattr(original_handler, attribute) setattr(new_handler, attribute, Utils.convert_to_ascii_if_possible(original)) for dimension in original_handler.dimensions.keys(): - Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) + Utils.copy_dimension(original_handler, new_handler, dimension, + new_names=dic_names, rename_dimension=rename_dimension) for variable in original_handler.variables.keys(): - Utils.copy_variable(original_handler, new_handler, variable, new_names=dic_names) + Utils.copy_variable(original_handler, new_handler, variable, + new_names=dic_names, rename_dimension=rename_dimension) original_handler.close() new_handler.close() @@ -577,7 +579,8 @@ class Utils(object): return netCDF4.num2date(nctime, units=units, calendar=cal_temps) @staticmethod - def copy_variable(source, destiny, variable, must_exist=True, add_dimensions=False, new_names=None): + def copy_variable(source, destiny, variable, must_exist=True, add_dimensions=False, new_names=None, + rename_dimension=True): """ Copy the given variable from source to destiny @@ -608,14 +611,16 @@ class Utils(object): if new_name in destiny.variables.keys(): return - - translated_dimensions = Utils._translate(source.variables[variable].dimensions, new_names) + if rename_dimension: + translated_dimensions = Utils._translate(source.variables[variable].dimensions, new_names) + else: + translated_dimensions = list(source.variables[variable].dimensions) if not set(translated_dimensions).issubset(destiny.dimensions): if not add_dimensions: raise Exception('Variable {0} can not be added because dimensions does not match: ' '{1} {2}'.format(variable, translated_dimensions, destiny.dimensions)) for dimension in source.variables[variable].dimensions: - Utils.copy_dimension(source, destiny, dimension, must_exist, new_names) + Utils.copy_dimension(source, destiny, dimension, must_exist, new_names, rename_dimension) if new_name in destiny.variables.keys(): # Just in case the variable we are copying match a dimension name return @@ -646,7 +651,7 @@ class Utils(object): for k in original_var.ncattrs() if k not in omitted_attributtes}) @staticmethod - def copy_dimension(source, destiny, dimension, must_exist=True, new_names=None): + def copy_dimension(source, destiny, dimension, must_exist=True, new_names=None, rename_dimension=False): """ Copy the given dimension from source to destiny, including dimension variables if present @@ -660,7 +665,7 @@ class Utils(object): """ if not must_exist and dimension not in source.dimensions.keys(): return - if not new_names: + if not new_names or not rename_dimension: new_names = dict() if dimension in new_names: new_name = new_names[dimension] @@ -672,7 +677,8 @@ class Utils(object): new_name = dimension destiny.createDimension(new_name, source.dimensions[dimension].size) if dimension in source.variables: - Utils.copy_variable(source, destiny, dimension, new_names=new_names) + Utils.copy_variable(source, destiny, dimension, + new_names=new_names, rename_dimension=rename_dimension) @staticmethod def concat_variables(source, destiny, remove_source=False): -- GitLab From 9ba58fbd3e1846160eb60e87c96b26f2df43f8c9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 28 May 2018 14:49:11 +0200 Subject: [PATCH 3/4] Bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 78238d72..fa1beb5e 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc7 +3.0.0rc8 -- GitLab From 4e617aeeeacc671607ae2ea8c58af6e9741ce6c3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 28 May 2018 15:03:35 +0200 Subject: [PATCH 4/4] Fixed tests --- test/unit/test_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 7e096de5..a277f7ca 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -55,5 +55,7 @@ class TestUtils(TestCase): with mock.patch('earthdiagnostics.utils.Utils.rename_variables') as rename_mock: Utils.rename_variable('file', 'old', 'new') Utils.rename_variable('file', 'old', 'new', False) - rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True), - mock.call('file', {'old': 'new'}, False))) + Utils.rename_variable('file', 'old', 'new', False, False) + rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True, True), + mock.call('file', {'old': 'new'}, False, True), + mock.call('file', {'old': 'new'}, False, False))) -- GitLab