diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e7a4660d91d09fe1e621d0b8bec1266274149b3e..f3e8d8c5d078774ad1fd0a7eef79ba296be9dfc9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,6 +40,7 @@ test_python3: report_codacy: stage: report script: + - '[ -z "$CODACY_PROJECT_TOKEN" ] && exit 0' - source activate earthdiagnostics3 - pip install codacy-coverage --upgrade - python-codacy-coverage -r test/report/python3/coverage.xml diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 6bdf281a20c0b905aabbef2c9f9ea970b8acd5a4..838bb223d5badda52511a04f6488986fc1dbc817 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -250,6 +250,7 @@ class CMORConfig(object): self.activity = parser.get_option('CMOR', 'ACTIVITY', 'CMIP') self.min_cmorized_vars = parser.get_int_option('CMOR', 'MIN_CMORIZED_VARS', 10) self.append_startdate = parser.get_bool_option('CMOR', 'APPEND_STARTDATE', False) + self.append_startdate_year_only = parser.get_bool_option('CMOR', 'APPEND_STARTDATE_YEAR_ONLY', False) vars_string = parser.get_option('CMOR', 'VARIABLE_LIST', '') self.var_manager = var_manager diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 9e855dd16d880f645187ec661272fe9f4ec1695f..1c54d35cca874dc2430a1525eb285c4d09e5fa9b 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -585,6 +585,8 @@ class Cmor3Convention(DataConvention): else: grid = self.config.cmor.default_atmos_grid if self.config.cmor.append_startdate: + if self.config.cmor.append_startdate_year_only: + startdate = startdate[0:4] subexp_id = 's{}-'.format(startdate) else: subexp_id = '' diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 7732726d661aca8aa4ed059dee409c54cd8af957..a294b824ce3a581533613de18d6e3b6d883dc9f6 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -67,8 +67,10 @@ class Moc(Diagnostic): def __eq__(self, other): if self._different_type(other): return False - return self.startdate == other.startdate and \ - self.member == other.member and self.chunk == other.chunk + return ( + self.startdate == other.startdate and + self.member == other.member and self.chunk == other.chunk + ) @classmethod def generate_jobs(cls, diags, options): @@ -136,7 +138,6 @@ class Moc(Diagnostic): del vo, e1v, e3v self._save_result(moc_results, mesh) - def _save_result(self, result, mesh): temp = TempFile.get() handler_source = Utils.open_cdf(self.variable_file.local_file) @@ -151,7 +152,6 @@ class Moc(Diagnostic): handler_temp.createDimension('region', len(result)) handler_temp.createDimension('region_length', 50) - var_region = handler_temp.createVariable('region', 'S1', ('region', 'region_length')) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 792793a535e7f2b9c0d0743ee206541900717677..c70c950085fcef2dc3072608baebe6c67a5b348a 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -22,7 +22,6 @@ import diagonals.regmean as regmean from diagonals.mesh_helpers.nemo import Nemo - class RegionMean(Diagnostic): """ Computes the mean value of the field (3D, weighted). @@ -79,10 +78,10 @@ class RegionMean(Diagnostic): self.box == other.box and self.variable == other.variable def __str__(self): - return 'Region mean Startdate: {0.startdate} Member: {0.member}' \ - 'Chunk: {0.chunk} Variable: {0.variable} ' \ - 'Box: {0.box} Save 3D: {0.save3d} Save variance: {0.variance}' \ - 'Grid point: {0.grid_point}'.format(self) + return ('Region mean Startdate: {0.startdate} Member: {0.member} ' + 'Chunk: {0.chunk} Variable: {0.variable} ' + 'Box: {0.box} Save 3D: {0.save3d} Save variance: {0.variance} ' + 'Grid point: {0.grid_point}'.format(self)) def __hash__(self): return hash(str(self)) @@ -101,8 +100,7 @@ class RegionMean(Diagnostic): options_available = (DiagnosticDomainOption(), DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticOption('grid_point', 'T'), - DiagnosticBasinListOption('basins', - Basins().Global), + DiagnosticBasinListOption('basins', 'global'), DiagnosticIntOption('min_depth', -1), DiagnosticIntOption('max_depth', -1), DiagnosticIntOption('min_lat', -1), @@ -117,17 +115,18 @@ class RegionMean(Diagnostic): box = Box() box.min_depth = options['min_depth'] box.max_depth = options['max_depth'] - box.min_lat = options['min_lat'] - box.max_lat = options['max_lat'] - box.min_lon = options['min_lon'] - box.max_lon = options['max_lon'] + if options['min_lat'] != -1: + box.min_lat = options['min_lat'] + box.max_lat = options['max_lat'] + if options['min_lon'] != -1 or options['max_lon'] != -1: + box.min_lon = options['min_lon'] + box.max_lon = options['max_lon'] basins = options['basins'] if not basins: Log.error('Basins not recognized') return() - job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job = RegionMean(diags.data_manager, startdate, member, chunk, @@ -165,10 +164,13 @@ class RegionMean(Diagnostic): masks[basin] = Utils.get_mask(basin) mesh = Nemo('mesh_hgr.nc', 'mask_regions.nc') - if self.box.min_lat is not -1 and self.box.max_lat is not -1 and \ - self.box.min_lon is not -1 and self.box.max_lat is not -1: - name = '{0}_{1}'.format(Box.get_lat_str(self.box), - Box.get_lon_str(self.box)) + if ( + self.box.min_lat is not -1 and self.box.max_lat is not -1 and + self.box.min_lon is not -1 and self.box.max_lat is not -1 + ): + name = '{0}_{1}'.format( + Box.get_lat_str(self.box), Box.get_lon_str(self.box) + ) masks[name] = mesh.get_region_mask(self.box.min_lat, self.box.max_lat, @@ -184,7 +186,6 @@ class RegionMean(Diagnostic): mean = regmean.compute_regmean_2D(data.data, masks, areacello) self._save_result_2D('mean', mean, data) - def _meand_3d_variable(self, data, mesh, masks): areacello = mesh.get_areacello(cell_point=self.grid_point) e3 = self._try_load_cube(3) @@ -222,7 +223,6 @@ class RegionMean(Diagnostic): var_name='j'), dims - 2) return cube - def _load_data(self): coords = [] handler = Utils.open_cdf(self.variable_file.local_file) @@ -279,7 +279,6 @@ class RegionMean(Diagnostic): box=box_save, region=self.basins) - def _save_result_2D(self, var, result, data): final_name = '{1}{0}'.format(var, self.variable) temp = TempFile.get() @@ -299,7 +298,6 @@ class RegionMean(Diagnostic): handler_temp.close() self.declared[final_name].set_local_file(temp, diagnostic=self) - def _save_result_3D(self, var, result, data): final_name = '{1}3d{0}'.format(var, self.variable) temp = TempFile.get() @@ -321,11 +319,10 @@ class RegionMean(Diagnostic): var = handler_temp.createVariable('{1}3d{0}'.format(var, self.variable), float, ('time', 'lev', 'region',),) - + var.units = '{0}'.format(data.units) for i, basin in enumerate(result): var_region[i, ...] = netCDF4.stringtoarr(str(basin), 50) var[..., i] = result[basin] handler_temp.close() self.declared[final_name].set_local_file(temp, diagnostic=self) - diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index f9207e5999dcf6c015ac15c927ab43d366de8bb9..99b46ce19a28bccf74372b509098359e62a09776 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -24,6 +24,7 @@ from earthdiagnostics.utils import Utils, TempFile import diagonals.regsum as regsum from diagonals.mesh_helpers.nemo import Nemo + class RegionSum(Diagnostic): """ Computes the sum of the field (3D, weighted). @@ -88,11 +89,9 @@ class RegionSum(Diagnostic): 'Grid point: {0.grid_point} Box: {0.box} Save 3D: {0.save3d}' \ 'Original grid: {0.grid} Basin: {0.basins}'.format(self) - def __hash__(self): return hash(str(self)) - @classmethod def generate_jobs(cls, diags, options): """ @@ -167,7 +166,8 @@ class RegionSum(Diagnostic): masks[basin] = Utils.get_mask(basin) mesh = Nemo('mesh_hgr.nc', 'mask_regions.nc') - if self.box.min_lat is not -1 and self.box.max_lat is not -1 and self.box.min_lon is not -1 and self.box.max_lat is not -1: + if self.box.min_lat is not -1 and self.box.max_lat is not -1 and \ + self.box.min_lon is not -1 and self.box.max_lat is not -1: name = '{0}_{1}'.format(Box.get_lat_str(self.box), Box.get_lon_str(self.box)) masks[name] = mesh.get_region_mask(self.box.min_lat, @@ -179,13 +179,11 @@ class RegionSum(Diagnostic): else: self._sum_2d_var(data, mesh, masks) - def _sum_2d_var(self, data, mesh, masks): areacello = mesh.get_areacello(cell_point=self.grid_point) varsum = regsum.compute_regsum_2D(data.data, masks, areacello) self._save_result_2D('sum', varsum, data) - def _sum_3d_variable(self, data, mesh, masks): areacello = mesh.get_areacello(cell_point=self.grid_point) tmask = iris.load_cube('mesh_hgr.nc', '{0}mask'.format(self.grid_point)) @@ -206,7 +204,6 @@ class RegionSum(Diagnostic): volcello, tmask) self._save_result_3D('sum', varsum3d, data) - def _try_load_cube(self, number): try: cube = iris.load_cube('mesh_hgr.nc', 'e{0}{1}'.format(number, self.grid_point)) @@ -239,7 +236,6 @@ class RegionSum(Diagnostic): data = iris.load_cube(self.variable_file.local_file) return self._rename_depth(data) - def _rename_depth(self, data): for coord_name in ('model_level_number', 'Vertical T levels', 'lev'): if data.coords(coord_name): @@ -249,7 +245,6 @@ class RegionSum(Diagnostic): break return data - def _fix_file_metadata(self): handler = Utils.open_cdf(self.variable_file.local_file) var = handler.variables[self.variable] @@ -314,7 +309,6 @@ class RegionSum(Diagnostic): self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, box=box_save, region=self.basins, grid=self.grid) - def _save_result_2D(self, var, result, data): final_name = '{1}{0}'.format(var, self.variable) temp = TempFile.get() @@ -334,7 +328,6 @@ class RegionSum(Diagnostic): handler_temp.close() self.declared[final_name].set_local_file(temp, diagnostic=self) - def _save_result_3D(self, var, result, data): final_name = '{1}3d{0}'.format(var, self.variable) temp = TempFile.get() diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 5cf4a9050cf6e4fa6c905a20d953d7e387d4ccbe..f623f48361126d85c4c5e1894c98ac4f03bbd681 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -150,7 +150,8 @@ class Siasiesiv(Diagnostic): mesh = Nemo('mesh_hgr.nc', 'mask_regions.nc') areacello = mesh.get_areacello(cell_point='T') gphit = mesh.get_grid_latitude(cell_point='T') - self.results['siextentn'], self.results['siextents'], self.results['siarean'], self.results['siareas'] = siasie.compute(gphit, areacello, sic_slices, self.masks) + self.results['siextentn'], self.results['siextents'], self.results['siarean'], self.results['siareas'] = \ + siasie.compute(gphit, areacello, sic_slices, self.masks) self.save() @@ -165,7 +166,6 @@ class Siasiesiv(Diagnostic): ' '.join(set(coordinates) | add_coordinates) handler.close() - def save(self): for var in self.results.keys(): res = self.results[var] @@ -175,15 +175,15 @@ class Siasiesiv(Diagnostic): Utils.copy_variable(handler_source, handler_temp, 'time', True, True) handler_temp.createDimension('region', len(self.masks)) handler_temp.createDimension('region_length', 50) - var_region = handler_temp.createVariable('region', 'S1', - ('region', 'region_length')) - var_res = handler_temp.createVariable('{0}'.format(var), float, - ('time', 'region',)) + var_region = handler_temp.createVariable( + 'region', 'S1', ('region', 'region_length') + ) + var_res = handler_temp.createVariable( + '{0}'.format(var), float, ('time', 'region',) + ) var_res.units = 'm^2' for i, basin in enumerate(self.masks): var_region[i, ...] = netCDF4.stringtoarr(str(basin), 50) var_res[..., i] = res[i, ...] handler_temp.close() self.generated[var].set_local_file(temp, diagnostic=self) - - diff --git a/test/unit/data_convention/test_primavera.py b/test/unit/data_convention/test_primavera.py index f2e6739b04288f629c54d64ca6fed90b12dd59b8..52a06faadfdec95a4e5d6b817631ee7fd76521a0 100644 --- a/test/unit/data_convention/test_primavera.py +++ b/test/unit/data_convention/test_primavera.py @@ -32,6 +32,7 @@ class TestPrimaveraConvention(TestCase): self.config.cmor.activity = 'activity' self.config.cmor.append_startdate = False + self.config.cmor.append_startdate_year_only = False self.config.cmor.initialization_number = 1 self.config.cmor.version = 'version' self.config.cmor.default_ocean_grid = 'ocean_grid' @@ -148,6 +149,19 @@ class TestPrimaveraConvention(TestCase): self.assertEqual(file_path, 'var_Omon_model_experiment_name_s19900101-r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_with_startdate_year_only(self): + """Test get_filename with startdate""" + self.config.cmor.append_startdate = True + self.config.cmor.append_startdate_year_only = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_s1990-r2i1p1f1_ocean_grid_199001-199001.nc') + def test_get_filename_no_cmor_var(self): """Test get_filename not passing cmor_var""" cmor_var = Mock() diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index 966ed536c97807961da450d383dad07558c66a30..35f8a09cb38d5e64793fac60d43e1c912409b9ae 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -38,23 +38,23 @@ class TestRegionMean(TestCase): jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 't')) self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 't')) jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 'u')) self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 'u')) jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 'u')) self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 'u')) box = Box() box.min_depth = 1.0 @@ -63,27 +63,34 @@ class TestRegionMean(TestCase): jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 'u')) self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - box, True, 'weights', False, Basins().Global)) + box, True, False, Basins().Global, 'u')) - jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false']) + jobs = RegionMean.generate_jobs( + self.diags, + ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', '', '', '', '', 'false'] + ) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - box, False, Basins().Global, False, '')) + box, False, False, Basins().Global, 'u')) self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - box, False, Basins().Global, False, '')) + box, False, False, Basins().Global, 'u')) - jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', - 'True']) + jobs = RegionMean.generate_jobs( + self.diags, + ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', '', '', '', '', 'false', 'True'] + ) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - box, False, Basins().Global, True, '')) + box, False, True, Basins().Global, 'u')) self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - box, False, Basins().Global, True, '')) + box, False, True, Basins().Global, 'u')) - jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', - 'True', 'grid']) + jobs = RegionMean.generate_jobs( + self.diags, + ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', '', '', '', '', 'false', 'True', 'grid'] + ) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box, False, Basins().Global, True, 'grid')) @@ -94,8 +101,11 @@ class TestRegionMean(TestCase): RegionMean.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(DiagnosticOptionError): - RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', - 'True', 'grid', 'extra']) + RegionMean.generate_jobs( + self.diags, + ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', '', '', '', '', 'false', + 'True', 'grid', 'extra'] + ) def test_str(self): box = Box()