diff --git a/VERSION b/VERSION index 53f0501f4c2ace879ebefbe4942c2977dd23603d..d4c4b54b7ccf30c6d9c6bcaba18851a613a0c1cf 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc11 +3.0.1 diff --git a/doc/source/conf.py b/doc/source/conf.py index 436019f12521accc66aa512561525c49c7df0434..4d4b99356817bc5dcb170b582f8ce9726aba15e1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -56,16 +56,16 @@ master_doc = 'index' # General information about the project. project = u'Earth Diagnostics' -copyright = u'2016, BSC-CNS Earth Sciences Department' +copyright = u'2018, BSC-CNS Earth Sciences Department' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents.source ~/vi # # The short X.Y version. -version = '3.0rc' +version = '3.0' # The full version, including alpha/beta/rc tags. -release = '3.0.0' +release = '3.0.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 73da16ba7f90f894f4c5a755d25ebda3b40944fe..0dcd848c26f8d9bfc09d4c5badb708e8ee402fdb 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -6,6 +6,7 @@ import shutil import uuid import traceback import eccodes +import time from datetime import datetime, timedelta import six @@ -124,11 +125,14 @@ class Cmorizer(object): if any(f in filename for f in filters): filtered.append(file_path) else: - os.remove(file_path) + self._remove(file_path) if len(filtered) == 0: Log.warning('Filters {0} do not match any of the files', filters) return filtered + def _remove(self, file_path): + os.remove(file_path) + def _cmorize_nc_files(self): nc_files = glob.glob(os.path.join(self.cmor_scratch, '*.nc')) for filename in self._filter_files(nc_files): @@ -177,6 +181,7 @@ class Cmorizer(object): def _clean_cmor_scratch(self): if os.path.exists(self.cmor_scratch): + time.sleep(2) shutil.rmtree(self.cmor_scratch) def _merge_mma_files(self, tarfile): @@ -192,7 +197,7 @@ class Cmorizer(object): shutil.move(temp, filename) Utils.cdo.mergetime(input=files, output=merged) for filename in files: - os.remove(filename) + self._remove(filename) tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') filename = 'MMA{0}_1m_{1[0]}_{1[1]}.nc'.format(grid, tar_startdate) shutil.move(merged, os.path.join(self.cmor_scratch, filename)) @@ -275,17 +280,17 @@ class Cmorizer(object): self._obtain_atmos_timestep(gribfile) full_file = self._get_monthly_grib(current_date, gribfile, grid) if not self._unpack_grib(full_file, gribfile, grid, current_date.month): - os.remove(gribfile) + self._remove(gribfile) return next_gribfile = self._get_original_grib_path(add_months(current_date, 1, self.experiment.calendar), grid) if not os.path.exists(next_gribfile): - os.remove(gribfile) + self._remove(gribfile) self._ungrib_vars(gribfile, current_date.month) for splited_file in glob.glob('{0}_*.128.nc'.format(gribfile)): - os.remove(splited_file) + self._remove(splited_file) Log.result('Month {0}, {1} variables finished', date2str(current_date), grid) @@ -371,7 +376,7 @@ class Cmorizer(object): Log.info('Processing file {0}', filename) if not self._contains_requested_variables(filename): - os.remove(filename) + self._remove(filename) return # Utils.convert2netcdf4(filename) @@ -394,7 +399,7 @@ class Cmorizer(object): except Exception as ex: Log.error('Variable {0} can not be cmorized: {1}', variable, ex) Log.result('File {0} cmorized!', filename) - os.remove(filename) + self._remove(filename) @staticmethod def _remove_valid_limits(handler): @@ -560,8 +565,8 @@ class Cmorizer(object): Utils.cdo.selmon(current_month.month, input=prev_gribfile, output=temp) Utils.cdo.mergetime(input=[temp, gribfile], output=self.path_icm) - os.remove(prev_gribfile) - os.remove(temp) + self._remove(prev_gribfile) + self._remove(temp) def _ungrib_vars(self, gribfile, month): for var_code in self.cmor.get_requested_codes(): @@ -662,7 +667,7 @@ class Cmorizer(object): var_cube = var_cubes.concatenate_cube() iris.save(var_cube, merged_file, zlib=True) for var_file in var_files: - os.remove(var_file) + self._remove(var_file) self._cmorize_nc_file(merged_file) def _update_time_variables(self, handler): diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 5ae6f8816beabffaa2f4d12e7c8f5f3d9b097be3..ff5cef99a40acf21c8822d0fe5dacfc1248c0d04 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -367,12 +367,17 @@ class CMORManager(DataManager): self.experiment.get_member_str(member), self.experiment.get_chunk_start_str(startdate, chunk), extension, cmor_prefix) - filepaths += glob.glob(os.path.join(tar_path, file_name)) - filepaths += glob.glob(os.path.join(tar_path, 'outputs', file_name)) - filepaths += glob.glob(os.path.join(tar_original_files, file_name)) - filepaths += glob.glob(os.path.join(tar_original_files, 'outputs', file_name)) + filepaths += self._find_paths(tar_path, file_name) + filepaths += self._find_paths(tar_path, 'outputs', file_name) + filepaths += self._find_paths(tar_original_files, file_name) + filepaths += self._find_paths(tar_original_files, 'outputs', file_name) return filepaths + def _find_paths(self, *args): + pattern = os.path.join(*args) + Log.debug('Looking for pattern {0}', pattern) + return glob.glob(pattern) + def _correct_paths(self, startdate): self._remove_extra_output_folder() self._fix_model_as_experiment_error(startdate) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 901d49d2726a680085647574d45b0851cbc6969d..054a64d3854f1f6452a224249f7e84e2cb44f528 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -229,8 +229,6 @@ class DataConvention(object): if not grid: grid = 'original' - if domain is None: - pass variable_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, self.config.experiment.atmos_timestep) vargrid_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, @@ -378,7 +376,7 @@ class Cmor2Convention(DataConvention): def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): folder_path = os.path.join(self.get_startdate_path(startdate), str(frequency), domain.name, var) - if grid: + if grid and grid != 'original': folder_path = os.path.join(folder_path, grid) folder_path = os.path.join(folder_path, self.get_member_str(member)) if self.config.cmor.version: @@ -578,10 +576,20 @@ class Cmor3Convention(DataConvention): grid = self.config.cmor.default_ocean_grid else: grid = self.config.cmor.default_atmos_grid - file_name = '{0}_{1}_{2}_{3}_{4}_{5}{6}'.format(var, cmor_table.name, self.config.experiment.model, - self.experiment_name(startdate), - self.get_member_str(member), - grid, time_bound) + if self.config.cmor.append_startdate: + subexp_id = 's{}-'.format(startdate) + else: + subexp_id = '' + + file_name = '{var_name}_{table}_{model}_{exp_id}_{subexp_id}{member}_{grid}{time}'.format( + var_name=var, + table=cmor_table.name, + model=self.config.experiment.model, + exp_id=self.experiment_name(startdate), + subexp_id=subexp_id, + member=self.get_member_str(member), + grid=grid, + time=time_bound) return file_name def _get_chunk_time_bounds(self, startdate, chunk, frequency): @@ -651,6 +659,23 @@ class Cmor3Convention(DataConvention): self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) + def experiment_name(self, startdate): + """ + Get experiment name. + + CMOR3 ignores append startdates as thery are added to the member str + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ + return self.config.experiment.experiment_name + def get_member_str(self, member): """ Transalate member number to member string diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 23224c48dc7d5e7f2f4afe89b23498696a5f231b..64e585a7e239ce9c847f9b182699a31f33913358 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -3,6 +3,7 @@ """Entry point for EarthDiagnostics""" import argparse import os +import time import sys import shutil import tempfile @@ -240,10 +241,14 @@ class EarthDiags(object): def _remove_scratch_dir(self): if os.path.islink(self.config.scratch_dir): - shutil.rmtree(os.path.realpath(self.config.scratch_dir)) + # time.sleep(4) + # shutil.rmtree(os.path.realpath(self.config.scratch_dir)) + Utils.execute_shell_command('rm -r {0}'.format(os.path.realpath(self.config.scratch_dir))) os.remove(self.config.scratch_dir) elif os.path.isdir(self.config.scratch_dir): - shutil.rmtree(self.config.scratch_dir) + # time.sleep(4) + # shutil.rmtree(self.config.scratch_dir) + Utils.execute_shell_command('rm -r {0}'.format(self.config.scratch_dir)) def report(self): """ diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index f2e0246570aae29b4998fd16ac6fec2bdcbfaffb..a07c2efa54a9f05b4b6d3c7852572ad978cce959 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -171,7 +171,8 @@ class HeatContentLayer(Diagnostic): nco.ncks(input=thetao_file, output=results, options=('-O -v lon,lat,time',)) Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False) handler_results = Utils.open_cdf(results) - handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) + var = handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) + var.units = 'J m-2' handler_results.sync() handler_results.variables['heatc'][:] = heatc_sl handler_results.close() diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 2bd5dcbd38580ab0f03070c7bfcef57613d2b795..5ab9ad31c3b31c5276b192322563f85dec4027ba 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -263,7 +263,10 @@ class InterpolateCDO(Diagnostic): handler = Utils.open_cdf(variable_file) lat_name, lon_name = self._get_lat_lon_alias(handler) var = handler.variables[self.variable] - units = var.units + try: + units = var.units + except AttributeError: + units = None coordinates = list() for dim in var.dimensions: if dim == 'i': @@ -284,8 +287,8 @@ class InterpolateCDO(Diagnostic): Utils.cdo.remap(','.join((self.grid.split('_')[0], self.weights)), input=variable_file, output=temp) handler = Utils.open_cdf(temp) - - handler.variables[self.variable].units = units + if units: + handler.variables[self.variable].units = units handler.close() if lat_name != 'lat': diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 18e8455116620399e40b7d11817f45a5ae033ded..a8ebe7a985d88f30958ac6bc5f7759587c148603 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -152,7 +152,7 @@ class Siasiesiv(Diagnostic): self.results['siareas'][basin] = self.sum(sic, mask, north=False) if not self.omit_volume: - volume = sic * sit + volume = sic * sit.data self.results['sivoln'][basin] = self.sum(volume, mask, north=True) self.results['sivols'][basin] = self.sum(volume, mask, north=False) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 2e12e4662d9474f49c07f8c789fbd6d5f8ca718e..d67c51df04df1b74768f9d4ec595ba28a86c9522 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -2,6 +2,7 @@ """Common utilities for multiple topics that are not big enough to have their own module""" import datetime import os +import time import re import shutil import stat @@ -254,7 +255,7 @@ class Utils(object): return string.encode(encoding) except UnicodeEncodeError: if u'Bretonnière' in string: - string = string.replace(u'Bretonnière', 'Bretonnière') + string = string.replace(u'Bretonnière', 'Bretonniere') return Utils.convert_to_ascii_if_possible(string, encoding) return string @@ -403,6 +404,7 @@ class Utils(object): copy_tree """ Utils.copy_tree(source, destiny) + time.sleep(2) shutil.rmtree(source) @staticmethod diff --git a/test/unit/data_convention/test_primavera.py b/test/unit/data_convention/test_primavera.py index 42fe37cb5627a60db0ee5e8ca3a873ceb4c71f04..f2e6739b04288f629c54d64ca6fed90b12dd59b8 100644 --- a/test/unit/data_convention/test_primavera.py +++ b/test/unit/data_convention/test_primavera.py @@ -57,7 +57,7 @@ class TestPrimaveraConvention(TestCase): """Test get expriment name when appending startdate""" self.config.cmor.append_startdate = True self.assertEqual(self.convention.experiment_name('19900101'), - 'experiment_nameS19900101') + 'experiment_name') def test_get_cmor_folder_path(self): """Test get cmor foilder path""" @@ -136,6 +136,18 @@ class TestPrimaveraConvention(TestCase): self.assertEqual(file_path, 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_with_startdate(self): + """Test get_filename with startdate""" + self.config.cmor.append_startdate = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_s19900101-r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_no_cmor_var(self): """Test get_filename not passing cmor_var""" cmor_var = Mock()