diff --git a/VERSION b/VERSION index e3ab2e1647bd77cac63f130c36b9a7641126e11d..34a8f4f6e53bf4616f153fb322c38d2f9e9337c4 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0b45 +3.0.0b46 diff --git a/doc/source/conf.py b/doc/source/conf.py index 9cba69d08275cf054174283fe07fc5ef73c8872b..e93d11ef6d355b0ef1e50f194e951fbe444b5654 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b45' +release = '3.0.0b46' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst index d678b1260a2bbe3f3f48c733f811c5a8421983be..5c393d890190d9cc17930b09769f16f5957def9b 100644 --- a/doc/source/diagnostic_list.rst +++ b/doc/source/diagnostic_list.rst @@ -508,19 +508,19 @@ Options: 4. Basin = Global: Basin to compute -5. Min depth: +5. Save 3d = True: + If True, it also stores the average per level + +6. Min depth: Minimum depth to compute in levels. If -1, average from the surface -6. Max depth: +7. Max depth: Maximum depth to compute in levels. If -1, average to the bottom -7. Save 3d = True: - If True, it also stores the average per level - 8. Variance = False: If True, it also stores the variance -5. Original grid = '': +9. Original grid = '': Source grid to choose. By default this is the original data, but sometimes you will want to use another (for example, the 'rotated' one produced by the rotation diagnostic) diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index 7e31e14c4941733d1b52c318ca8e3476fec22e2c..3df515fab7606aaea1ecbb465bb139401c4a5041 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 4addc828ac4f1d655b2209ab4af92d19bed5c0cf..6ab160179198b43da919f77b252213db9cd2286a 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -111,15 +111,25 @@ class Cmorizer(object): def _correct_fluxes(self): fluxes_vars = [self.data_manager.variable_list.get_variable(cmor_var, True).short_name - for cmor_var in ("prsn", "rss", "rls", "rsscs", "rsds", "rlds", "hfss", 'hfls')] - + for cmor_var in ('prc', "prsn", "rss", "rls", "rsscs", "rsds", "rlds", "hfss", 'hfls')] + change_sign_vars = [self.data_manager.variable_list.get_variable(cmor_var, True).short_name + for cmor_var in ("hfss", 'hfls')] + total_seconds = (self.experiment.atmos_timestep * 3600) for filename in glob.glob(os.path.join(self.cmor_scratch, '*.nc')): handler = Utils.openCdf(filename) for varname in handler.variables.keys(): cmor_var = self.data_manager.variable_list.get_variable(varname, True) + if cmor_var is None or cmor_var.short_name not in fluxes_vars: continue - handler.variables[varname][:] = handler.variables[varname][:] / (self.experiment.atmos_timestep * 3600) + + if cmor_var.short_name in change_sign_vars: + sign = -1 + else: + sign = 1 + + handler.variables[varname][:] = sign * handler.variables[varname][:] / total_seconds + handler.units = '{0} {1}'.format(handler.units, 's-1') handler.close() def _unpack_tar_file(self, tarfile): diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index dfc2082a83f814f51582c2881887a6aeb48ba32c..2a235977af263291d1826624c2e4d179d9a6e990 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -248,5 +248,25 @@ class DiagnosticBoolOption(DiagnosticOption): return option_value.lower() in ('true', 't', 'yes') +class DiagnosticChoiceOption(DiagnosticOption): + def __init__(self, name, choices, default_value=None, ignore_case=True): + super(DiagnosticChoiceOption, self).__init__(name, default_value) + self.choices = choices + self.ignore_case = ignore_case + + def parse(self, value): + value = self.check_default(value) + if self.ignore_case: + value = value.lower() + for choice in self.choices: + if value == choice.lower(): + return choice + else: + if value in self.choices: + return value + raise DiagnosticOptionError('Value {1} in option {0} is not a valid choice. ' + 'Options are {2}'.format(self.name, value, self.choices)) + + class DiagnosticOptionError(Exception): pass diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 35bbcb4c450535bee90ea65f3dc8589de75acf78..7b0dcc847ead4471e6f9fc64e41a273a376999bc 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -87,7 +87,7 @@ class Interpolate(Diagnostic): DiagnosticVariableOption('variable'), DiagnosticDomainOption('domain', ModelingRealms.ocean), DiagnosticBoolOption('invert_lat', False), - DiagnosticOption('original_grid')) + DiagnosticOption('original_grid', '')) options = cls.process_options(options, options_available) job_list = list() diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index bff647adaed13df8f48f917a8b4772780864418f..9d4b4011b3a00b4f9a87cbf6348a54bb7964b7fd 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -78,7 +78,7 @@ class InterpolateCDO(Diagnostic): DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), DiagnosticDomainOption('domain', ModelingRealms.ocean), DiagnosticBoolOption('mask_oceans', True), - DiagnosticOption('original_grid')) + DiagnosticOption('original_grid', '')) options = cls.process_options(options, options_available) target_grid = cls._translate_ifs_grids_to_cdo_names(options['target_grid']) job_list = list() diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 3effb07c85a78f68a5cfd31abb2ef2d2f459e74e..f9e758a5c16966981d9696d96b267e5d739eaeac 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -1,4 +1,5 @@ # coding=utf-8 +import shutil from bscearth.utils.log import Log import os @@ -43,6 +44,7 @@ class Rotation(Diagnostic): self.variablev = variablev self.domain = domain self.executable = executable + self.tempTemplate = None def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ @@ -82,27 +84,68 @@ class Rotation(Diagnostic): """ Runs the diagnostic """ - ufile = self.data_manager.get_file(self.domain, self.variableu, self.startdate, self.member, self.chunk) - vfile = self.data_manager.get_file(self.domain, self.variablev, self.startdate, self.member, self.chunk) + self.ufile = self.data_manager.get_file(self.domain, self.variableu, self.startdate, self.member, self.chunk) + self.vfile = self.data_manager.get_file(self.domain, self.variablev, self.startdate, self.member, self.chunk) - urotated = TempFile.get() - vrotated = TempFile.get() + handler = Utils.openCdf(self.ufile) + if 'lev' in handler.dimensions: + self.num_levels = handler.dimensions['lev'].size + self.has_levels = True + else: + self.num_levels = 1 + self.has_levels = False + handler.close() - namelist_file = self._create_namelist(ufile, urotated, vfile, vrotated) + for lev in range(0, self.num_levels): + self._rotate_level(lev) - Utils.execute_shell_command('{0} {1}'.format(self.executable, namelist_file), Log.INFO) + urotated = self._merge_levels(self.variableu, 'u') + vrotated = self._merge_levels(self.variablev, 'v') - ufile_handler = Utils.openCdf(ufile) + ufile_handler = Utils.openCdf(self.ufile) self._add_metadata_and_vars(ufile_handler, urotated, self.variableu) - self._add_metadata_and_vars(ufile_handler, vrotated, self.variablev) ufile_handler.close() + os.remove(self.ufile) - os.remove(ufile) - os.remove(vfile) + vfile_handler = Utils.openCdf(self.vfile) + self._add_metadata_and_vars(vfile_handler, vrotated, self.variablev) + vfile_handler.close() + os.remove(self.vfile) self.send_file(urotated, self.domain, self.variableu, self.startdate, self.member, self.chunk, grid='rotated') self.send_file(vrotated, self.domain, self.variablev, self.startdate, self.member, self.chunk, grid='rotated') + def _merge_levels(self, var, direction): + temp = TempFile.get() + if self.has_levels: + Utils.nco.ncecat(input=self._get_level_file(0, direction), output=temp, + options="-n {0},2,1 -v '{1}'".format(self.num_levels, var)) + handler = Utils.openCdf(temp) + if 'record' in handler.dimensions: + handler.renameDimension('record', 'lev') + handler.close() + Utils.nco.ncpdq(input=temp, output=temp, options='-O -h -a time,lev') + Utils.rename_variables(temp, {'x': 'i', 'y': 'j'}, must_exist=False, rename_dimension=True) + else: + Utils.move_file(self._get_level_file(0, direction), temp) + return temp + + def _rotate_level(self, lev): + ufile = self._extract_level(self.ufile, self.variableu, lev) + vfile = self._extract_level(self.vfile, self.variablev, lev) + namelist_file = self._create_namelist(ufile, self._get_level_file(lev, 'u'), + vfile, self._get_level_file(lev, 'v')) + Utils.execute_shell_command('{0} {1}'.format(self.executable, namelist_file), Log.INFO) + + def _extract_level(self, input_file, var, level): + temp = TempFile.get() + if self.has_levels: + Utils.nco.ncks(input=input_file, output=temp, options='-O -d lev,{0} -v {1},lat,lon'.format(level, var)) + Utils.nco.ncwa(input=temp, output=temp, options='-O -h -a lev') + else: + shutil.copy(input_file, temp) + return temp + def _create_namelist(self, ufile, urotated, vfile, vrotated): namelist_file = TempFile.get(suffix='') rotate_namelist = open(namelist_file, 'w') @@ -121,7 +164,8 @@ class Rotation(Diagnostic): def _add_metadata_and_vars(self, reference_file_handler, rotaded_file, var_name): rotated_handler = Utils.openCdf(rotaded_file) self._copy_extra_variables(reference_file_handler, rotated_handler) - Utils.copy_attributes(rotated_handler.variables[var_name], reference_file_handler.variables[var_name]) + Utils.copy_attributes(rotated_handler.variables[var_name], reference_file_handler.variables[var_name], + ('_FillValue',)) rotated_handler.close() def _copy_extra_variables(self, reference_file_handler, rotated_handler): @@ -129,6 +173,12 @@ class Rotation(Diagnostic): if var not in rotated_handler.variables.keys() and var not in [self.variableu, self.variablev]: Utils.copy_variable(reference_file_handler, rotated_handler, var, True, True) + def _get_level_file(self, lev, direction): + if not self.tempTemplate: + self.tempTemplate = TempFile.get(suffix='_01.nc') + # self.tempTemplate = 'temp_01.nc' + return self.tempTemplate.replace('_01.nc', '_{1}_{0:02d}.nc'.format(lev + 1, direction)) + diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index bb39bf42a418b2a5d4809f32125d9563be246bcb..302e33e0e4160f2f46a1985d7c7d07600e15b459 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -475,9 +475,11 @@ class Utils(object): new_var[:] = original_var[:] @staticmethod - def copy_attributes(new_var, original_var): + def copy_attributes(new_var, original_var, omitted_attributtes=None): + if omitted_attributtes is None: + omitted_attributtes = [] new_var.setncatts({k: Utils.convert_to_ASCII_if_possible(original_var.getncattr(k)) - for k in original_var.ncattrs()}) + for k in original_var.ncattrs() if k not in omitted_attributtes}) @staticmethod def copy_dimension(source, destiny, dimension, must_exist=True, new_names=None): diff --git a/setup.py b/setup.py index 3e079b5cbf4aa49094340ad5758ed312f824cce8..e8c07f09ce0c3a4886dd059eb3ab7b3f4b7dacb3 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( description='EarthDiagnostics', author='BSC-CNS Earth Sciences Department', author_email='javier.vegas@bsc.es', - url='http://www.bsc.es/projects/earthscixºence/autosubmit/', + url='http://www.bsc.es/projects/earthsciences/autosubmit/', keywords=['climate', 'weather', 'diagnostic'], setup_requires=['pyproj'], install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo', 'nco', 'cfunits>=1.1.4', 'coverage',