From ab07b205820ee968083274d6493dcb4c76a280c3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 27 Jul 2018 12:50:20 +0200 Subject: [PATCH 01/11] Added support for startdates in CMOR3 using subexpid --- earthdiagnostics/data_convention.py | 35 ++++++++++++++++++--- test/unit/data_convention/test_primavera.py | 14 ++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 901d49d2..98259dba 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -578,10 +578,20 @@ class Cmor3Convention(DataConvention): grid = self.config.cmor.default_ocean_grid else: grid = self.config.cmor.default_atmos_grid - file_name = '{0}_{1}_{2}_{3}_{4}_{5}{6}'.format(var, cmor_table.name, self.config.experiment.model, - self.experiment_name(startdate), - self.get_member_str(member), - grid, time_bound) + if self.config.cmor.append_startdate: + subexp_id = 's{}-'.format(startdate) + else: + subexp_id = '' + + file_name = '{var_name}_{table}_{model}_{exp_id}_{subexp_id}{member}_{grid}{time}'.format( + var_name=var, + table=cmor_table.name, + model=self.config.experiment.model, + exp_id=self.experiment_name(startdate), + subexp_id=subexp_id, + member=self.get_member_str(member), + grid=grid, + time=time_bound) return file_name def _get_chunk_time_bounds(self, startdate, chunk, frequency): @@ -651,6 +661,23 @@ class Cmor3Convention(DataConvention): self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) + def experiment_name(self, startdate): + """ + Get experiment name. + + CMOR3 ignores append startdates as thery are added to the member str + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ + return self.config.experiment.experiment_name + def get_member_str(self, member): """ Transalate member number to member string diff --git a/test/unit/data_convention/test_primavera.py b/test/unit/data_convention/test_primavera.py index 42fe37cb..f2e6739b 100644 --- a/test/unit/data_convention/test_primavera.py +++ b/test/unit/data_convention/test_primavera.py @@ -57,7 +57,7 @@ class TestPrimaveraConvention(TestCase): """Test get expriment name when appending startdate""" self.config.cmor.append_startdate = True self.assertEqual(self.convention.experiment_name('19900101'), - 'experiment_nameS19900101') + 'experiment_name') def test_get_cmor_folder_path(self): """Test get cmor foilder path""" @@ -136,6 +136,18 @@ class TestPrimaveraConvention(TestCase): self.assertEqual(file_path, 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_with_startdate(self): + """Test get_filename with startdate""" + self.config.cmor.append_startdate = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_s19900101-r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_no_cmor_var(self): """Test get_filename not passing cmor_var""" cmor_var = Mock() -- GitLab From d41f0f78e4d48d78ffa87c5b6d7a7cb15bf40faf Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 27 Jul 2018 15:43:28 +0200 Subject: [PATCH 02/11] remove unused code --- earthdiagnostics/data_convention.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 98259dba..94742fc5 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -229,8 +229,6 @@ class DataConvention(object): if not grid: grid = 'original' - if domain is None: - pass variable_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, self.config.experiment.atmos_timestep) vargrid_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, -- GitLab From b5356658690c5f252e976924174b51a139c0800f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 21 Aug 2018 12:23:10 +0200 Subject: [PATCH 03/11] Added sleep call before rmtree --- earthdiagnostics/cmorizer.py | 25 +++++++++++++++---------- earthdiagnostics/cmormanager.py | 13 +++++++++---- earthdiagnostics/earthdiags.py | 3 +++ earthdiagnostics/utils.py | 2 ++ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 73da16ba..0dcd848c 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -6,6 +6,7 @@ import shutil import uuid import traceback import eccodes +import time from datetime import datetime, timedelta import six @@ -124,11 +125,14 @@ class Cmorizer(object): if any(f in filename for f in filters): filtered.append(file_path) else: - os.remove(file_path) + self._remove(file_path) if len(filtered) == 0: Log.warning('Filters {0} do not match any of the files', filters) return filtered + def _remove(self, file_path): + os.remove(file_path) + def _cmorize_nc_files(self): nc_files = glob.glob(os.path.join(self.cmor_scratch, '*.nc')) for filename in self._filter_files(nc_files): @@ -177,6 +181,7 @@ class Cmorizer(object): def _clean_cmor_scratch(self): if os.path.exists(self.cmor_scratch): + time.sleep(2) shutil.rmtree(self.cmor_scratch) def _merge_mma_files(self, tarfile): @@ -192,7 +197,7 @@ class Cmorizer(object): shutil.move(temp, filename) Utils.cdo.mergetime(input=files, output=merged) for filename in files: - os.remove(filename) + self._remove(filename) tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') filename = 'MMA{0}_1m_{1[0]}_{1[1]}.nc'.format(grid, tar_startdate) shutil.move(merged, os.path.join(self.cmor_scratch, filename)) @@ -275,17 +280,17 @@ class Cmorizer(object): self._obtain_atmos_timestep(gribfile) full_file = self._get_monthly_grib(current_date, gribfile, grid) if not self._unpack_grib(full_file, gribfile, grid, current_date.month): - os.remove(gribfile) + self._remove(gribfile) return next_gribfile = self._get_original_grib_path(add_months(current_date, 1, self.experiment.calendar), grid) if not os.path.exists(next_gribfile): - os.remove(gribfile) + self._remove(gribfile) self._ungrib_vars(gribfile, current_date.month) for splited_file in glob.glob('{0}_*.128.nc'.format(gribfile)): - os.remove(splited_file) + self._remove(splited_file) Log.result('Month {0}, {1} variables finished', date2str(current_date), grid) @@ -371,7 +376,7 @@ class Cmorizer(object): Log.info('Processing file {0}', filename) if not self._contains_requested_variables(filename): - os.remove(filename) + self._remove(filename) return # Utils.convert2netcdf4(filename) @@ -394,7 +399,7 @@ class Cmorizer(object): except Exception as ex: Log.error('Variable {0} can not be cmorized: {1}', variable, ex) Log.result('File {0} cmorized!', filename) - os.remove(filename) + self._remove(filename) @staticmethod def _remove_valid_limits(handler): @@ -560,8 +565,8 @@ class Cmorizer(object): Utils.cdo.selmon(current_month.month, input=prev_gribfile, output=temp) Utils.cdo.mergetime(input=[temp, gribfile], output=self.path_icm) - os.remove(prev_gribfile) - os.remove(temp) + self._remove(prev_gribfile) + self._remove(temp) def _ungrib_vars(self, gribfile, month): for var_code in self.cmor.get_requested_codes(): @@ -662,7 +667,7 @@ class Cmorizer(object): var_cube = var_cubes.concatenate_cube() iris.save(var_cube, merged_file, zlib=True) for var_file in var_files: - os.remove(var_file) + self._remove(var_file) self._cmorize_nc_file(merged_file) def _update_time_variables(self, handler): diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 5ae6f881..ff5cef99 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -367,12 +367,17 @@ class CMORManager(DataManager): self.experiment.get_member_str(member), self.experiment.get_chunk_start_str(startdate, chunk), extension, cmor_prefix) - filepaths += glob.glob(os.path.join(tar_path, file_name)) - filepaths += glob.glob(os.path.join(tar_path, 'outputs', file_name)) - filepaths += glob.glob(os.path.join(tar_original_files, file_name)) - filepaths += glob.glob(os.path.join(tar_original_files, 'outputs', file_name)) + filepaths += self._find_paths(tar_path, file_name) + filepaths += self._find_paths(tar_path, 'outputs', file_name) + filepaths += self._find_paths(tar_original_files, file_name) + filepaths += self._find_paths(tar_original_files, 'outputs', file_name) return filepaths + def _find_paths(self, *args): + pattern = os.path.join(*args) + Log.debug('Looking for pattern {0}', pattern) + return glob.glob(pattern) + def _correct_paths(self, startdate): self._remove_extra_output_folder() self._fix_model_as_experiment_error(startdate) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 23224c48..9717abc4 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -3,6 +3,7 @@ """Entry point for EarthDiagnostics""" import argparse import os +import time import sys import shutil import tempfile @@ -240,9 +241,11 @@ class EarthDiags(object): def _remove_scratch_dir(self): if os.path.islink(self.config.scratch_dir): + time.sleep(2) shutil.rmtree(os.path.realpath(self.config.scratch_dir)) os.remove(self.config.scratch_dir) elif os.path.isdir(self.config.scratch_dir): + time.sleep(2) shutil.rmtree(self.config.scratch_dir) def report(self): diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 2e12e466..7130981a 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -2,6 +2,7 @@ """Common utilities for multiple topics that are not big enough to have their own module""" import datetime import os +import time import re import shutil import stat @@ -403,6 +404,7 @@ class Utils(object): copy_tree """ Utils.copy_tree(source, destiny) + time.sleep(2) shutil.rmtree(source) @staticmethod -- GitLab From f7258b1b12efa39e10db6340dc3ca16df15f14e4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 21 Aug 2018 12:23:32 +0200 Subject: [PATCH 04/11] Bumped version --- VERSION | 2 +- doc/source/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 53f0501f..cf232b52 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc11 +3.0.0 diff --git a/doc/source/conf.py b/doc/source/conf.py index 436019f1..ae5e392f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -63,7 +63,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # built documents.source ~/vi # # The short X.Y version. -version = '3.0rc' +version = '3.0' # The full version, including alpha/beta/rc tags. release = '3.0.0' -- GitLab From 87403678d5159148382e289cdf18992cac34f451 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 21 Aug 2018 12:24:57 +0200 Subject: [PATCH 05/11] Bumped version --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index ae5e392f..4270241b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = u'Earth Diagnostics' -copyright = u'2016, BSC-CNS Earth Sciences Department' +copyright = u'2018, BSC-CNS Earth Sciences Department' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the -- GitLab From 9dbb4766078ad88d40ecefb9ba285829c583d578 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 21 Aug 2018 18:40:44 +0200 Subject: [PATCH 06/11] Add units to ohc2d --- earthdiagnostics/ocean/heatcontentlayer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index f2e02465..a07c2efa 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -171,7 +171,8 @@ class HeatContentLayer(Diagnostic): nco.ncks(input=thetao_file, output=results, options=('-O -v lon,lat,time',)) Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False) handler_results = Utils.open_cdf(results) - handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) + var = handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) + var.units = 'J m-2' handler_results.sync() handler_results.variables['heatc'][:] = heatc_sl handler_results.close() -- GitLab From 9319f0818da8c0b653b90e6c1c6aa71991792e21 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 23 Aug 2018 12:08:43 +0200 Subject: [PATCH 07/11] Fixes for missing units in interpolation and original grid in CMIP5 --- earthdiagnostics/data_convention.py | 2 +- earthdiagnostics/ocean/interpolatecdo.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 94742fc5..054a64d3 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -376,7 +376,7 @@ class Cmor2Convention(DataConvention): def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): folder_path = os.path.join(self.get_startdate_path(startdate), str(frequency), domain.name, var) - if grid: + if grid and grid != 'original': folder_path = os.path.join(folder_path, grid) folder_path = os.path.join(folder_path, self.get_member_str(member)) if self.config.cmor.version: diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 2bd5dcbd..5ab9ad31 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -263,7 +263,10 @@ class InterpolateCDO(Diagnostic): handler = Utils.open_cdf(variable_file) lat_name, lon_name = self._get_lat_lon_alias(handler) var = handler.variables[self.variable] - units = var.units + try: + units = var.units + except AttributeError: + units = None coordinates = list() for dim in var.dimensions: if dim == 'i': @@ -284,8 +287,8 @@ class InterpolateCDO(Diagnostic): Utils.cdo.remap(','.join((self.grid.split('_')[0], self.weights)), input=variable_file, output=temp) handler = Utils.open_cdf(temp) - - handler.variables[self.variable].units = units + if units: + handler.variables[self.variable].units = units handler.close() if lat_name != 'lat': -- GitLab From 9082dbca00e35af9f84111e34b8fc29ff96e19fb Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 23 Aug 2018 15:45:52 +0200 Subject: [PATCH 08/11] Increase sleep in rmtree --- earthdiagnostics/earthdiags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 9717abc4..8bd23e51 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -241,11 +241,11 @@ class EarthDiags(object): def _remove_scratch_dir(self): if os.path.islink(self.config.scratch_dir): - time.sleep(2) + time.sleep(4) shutil.rmtree(os.path.realpath(self.config.scratch_dir)) os.remove(self.config.scratch_dir) elif os.path.isdir(self.config.scratch_dir): - time.sleep(2) + time.sleep(4) shutil.rmtree(self.config.scratch_dir) def report(self): -- GitLab From fdb7cf9c9a40402cdc955e54c5f2852e78850964 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 23 Aug 2018 16:18:02 +0200 Subject: [PATCH 09/11] Use rm call to remove scratch_dir --- earthdiagnostics/earthdiags.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 8bd23e51..64e585a7 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -241,12 +241,14 @@ class EarthDiags(object): def _remove_scratch_dir(self): if os.path.islink(self.config.scratch_dir): - time.sleep(4) - shutil.rmtree(os.path.realpath(self.config.scratch_dir)) + # time.sleep(4) + # shutil.rmtree(os.path.realpath(self.config.scratch_dir)) + Utils.execute_shell_command('rm -r {0}'.format(os.path.realpath(self.config.scratch_dir))) os.remove(self.config.scratch_dir) elif os.path.isdir(self.config.scratch_dir): - time.sleep(4) - shutil.rmtree(self.config.scratch_dir) + # time.sleep(4) + # shutil.rmtree(self.config.scratch_dir) + Utils.execute_shell_command('rm -r {0}'.format(self.config.scratch_dir)) def report(self): """ -- GitLab From 7944ef2ed1ec27d1869f23ce33a9ea339228dc1d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 23 Aug 2018 17:56:36 +0200 Subject: [PATCH 10/11] Fix PA name --- earthdiagnostics/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7130981a..d67c51df 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -255,7 +255,7 @@ class Utils(object): return string.encode(encoding) except UnicodeEncodeError: if u'Bretonnière' in string: - string = string.replace(u'Bretonnière', 'Bretonnière') + string = string.replace(u'Bretonnière', 'Bretonniere') return Utils.convert_to_ascii_if_possible(string, encoding) return string -- GitLab From b2e0d2de6c30c33cf653c4bf87adf20abf3bafc2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 25 Oct 2018 12:37:48 +0200 Subject: [PATCH 11/11] Changed code to avoid issues with sic and sit having slightly different coordinates --- VERSION | 2 +- doc/source/conf.py | 2 +- earthdiagnostics/ocean/siasiesiv.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index cf232b52..d4c4b54b 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0 +3.0.1 diff --git a/doc/source/conf.py b/doc/source/conf.py index 4270241b..4d4b9935 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -65,7 +65,7 @@ copyright = u'2018, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0' # The full version, including alpha/beta/rc tags. -release = '3.0.0' +release = '3.0.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 18e84551..a8ebe7a9 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -152,7 +152,7 @@ class Siasiesiv(Diagnostic): self.results['siareas'][basin] = self.sum(sic, mask, north=False) if not self.omit_volume: - volume = sic * sit + volume = sic * sit.data self.results['sivoln'][basin] = self.sum(volume, mask, north=True) self.results['sivols'][basin] = self.sum(volume, mask, north=False) -- GitLab