diff --git a/VERSION b/VERSION index 880f38c61c92bd49d96147f974c440f2ffbc641b..ea15350b27ab8756f6f8668b3d64d9ff70489af7 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0b52 +3.0.0b53 diff --git a/diags.conf b/diags.conf index 2e9c7c56e40f8903b20cc893f5c5b6085035af3e..7b966d685ad9ec84b1fd7fa9293279d228b862c4 100644 --- a/diags.conf +++ b/diags.conf @@ -15,11 +15,11 @@ CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = module,atmos,uas,vas,sfcwind monmean,sfcwind,atmos,6hr +DIAGS = maskland,ocean,thetao # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = 6hr +FREQUENCY = mon # Path to CDFTOOLS binaries CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir @@ -85,10 +85,10 @@ OCEAN_TIMESTEP = 6 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = a04f -STARTDATES = 20150201 -MEMBERS = fc00 -MEMBER_DIGITS = 2 +EXPID = t01f +STARTDATES = 19900101 +MEMBERS = fc0 +MEMBER_DIGITS = 1 CHUNK_SIZE = 1 CHUNKS = 1 # CHUNKS = 1 diff --git a/doc/source/codedoc/ocean.rst b/doc/source/codedoc/ocean.rst index cfa40e996173c8c5f98ae70c7ecca1da01900f13..52830bb9ccdc0e114c4506ed1f9d836aeda99834 100644 --- a/doc/source/codedoc/ocean.rst +++ b/doc/source/codedoc/ocean.rst @@ -55,6 +55,12 @@ earthdiagnostics.ocean.interpolatecdo :show-inheritance: :members: +earthdiagnostics.ocean.maskland +------------------------------- +.. automodule:: earthdiagnostics.ocean.maskland + :show-inheritance: + :members: + earthdiagnostics.ocean.maxmoc ----------------------------- .. automodule:: earthdiagnostics.ocean.maxmoc diff --git a/doc/source/conf.py b/doc/source/conf.py index 8a5867886b11f3a172628fc5a7ec5eac3663e375..eef27fe9c310774501b63ebc62b3fb52e33ca03f 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b52' +release = '3.0.0b53' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst index 39901f01e1c017f13c96f615015a32b62833c41a..6f42fbb5bcc184c673178657b08ee6853f982735 100644 --- a/doc/source/diagnostic_list.rst +++ b/doc/source/diagnostic_list.rst @@ -454,6 +454,27 @@ Options: Source grid to choose. By default this is the original data, but sometimes you will want to use another (for example, the 'rotated' one produced by the rotation diagnostic) +maskland +~~~~~~~~ + +Replaces all values excluded by the mask by NaN. See :class:`~earthdiagnostics.ocean.maskland.MaskLand` + +Options: +******** + +1. Domain: + Variable to mask domain + +2. Variable: + variable to mask + +3. Cell point = T: + Cell point where variable is stored. Options: T, U, V, W, F + +4. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) + maxmoc ~~~~~~ diff --git a/earthdiagnostics/cmor_tables/cmip6 b/earthdiagnostics/cmor_tables/cmip6 index 8415b26f6dda7b699501c6963a0ec6cb155eb1ab..78eb04bd32dcc398323b21b1cb0636b2f07ffc68 160000 --- a/earthdiagnostics/cmor_tables/cmip6 +++ b/earthdiagnostics/cmor_tables/cmip6 @@ -1 +1 @@ -Subproject commit 8415b26f6dda7b699501c6963a0ec6cb155eb1ab +Subproject commit 78eb04bd32dcc398323b21b1cb0636b2f07ffc68 diff --git a/earthdiagnostics/cmor_tables/default.csv b/earthdiagnostics/cmor_tables/default.csv index 3a49dc44f2e50f79ec60f0d5b1044560822a3b07..fd13cc4fa08ba52ef6288fb180cdf81cf86598d4 100644 --- a/earthdiagnostics/cmor_tables/default.csv +++ b/earthdiagnostics/cmor_tables/default.csv @@ -315,8 +315,9 @@ c-export,c-export,total_carbon_export_at_100m,total Carbon export at 100m,ocnBgc tintpp,tintpp,global_total_integrated_primary_production,global total integrated primary production,ocnBgchem,,,,,, tnfix,tnfix,global_total_nitrogen_fixation,global total nitrogen fixation,ocnBgchem,,,,,, tdenit,tdenit,total_denitrification,Total denitrification,ocnBgchem,,,,,, +intppnew,intppnew,new_primary_production,Vertically integrated new primary production,ocnBgchem,,,,,, inttpp,inttpp,total_primary_production_of_phyto,Total Primary production of phyto,ocnBgchem,,,,,, -intppnew,intppnew,new_primary_production_of_phyto,New Primary production of phyto,ocnBgchem,,,,,, +inttppnew,inttppnew,new_primary_production_of_phyto,New Primary production of phyto,ocnBgchem,,,,,, intppphy,intppphy,vertically_integrated_primary_production_by_nanophy,Vertically integrated primary production by nanophy,ocnBgchem,,,,,, ppphy,ppphy,primary_production_of_nanooplakton,Primary production of nanooplakton,ocnBgchem,,,,,, intpbcal,intpbcal,vertically_integrated_of_calcite_productdic_fluxion,Vertically integrated of calcite productDIC fluxion,ocnBgchem,,,,,, diff --git a/earthdiagnostics/cmor_tables/primavera b/earthdiagnostics/cmor_tables/primavera index ad4f256777265479b503bfa3e88a61ce05dd932f..10e46868e356ef3a217c38fe1e0b7d46f8d3158e 160000 --- a/earthdiagnostics/cmor_tables/primavera +++ b/earthdiagnostics/cmor_tables/primavera @@ -1 +1 @@ -Subproject commit ad4f256777265479b503bfa3e88a61ce05dd932f +Subproject commit 10e46868e356ef3a217c38fe1e0b7d46f8d3158e diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index b5590a6092b55a30ab9cc1bcf68fc7cf4f517d72..8307cb14838cb8eabc48c50988827867da082816 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -64,14 +64,19 @@ class Cmorizer(object): Log.info('Skipping ocean cmorization due to configuration') return Log.info('\nCMORizing ocean\n') - self._cmorize_ocean_files('MMO') - self._cmorize_ocean_files('PPO') - self._cmorize_ocean_files('diags') + self._cmorize_ocean_files('MMO', 'PPO', 'diags') - def _cmorize_ocean_files(self, prefix): - tar_folder = os.path.join(self.original_files_path, '{0}*'.format(prefix)) - tar_files = glob.glob(tar_folder) - tar_files.sort() + def _cmorize_ocean_files(self, *args): + tar_files = () + for prefix in args: + tar_folder = os.path.join(self.original_files_path, '{0}*'.format(prefix)) + tar_files = glob.glob(tar_folder) + tar_files.sort() + if len(tar_files) > 0: + break + + if not len(tar_files): + Log.error('No {1} files found in {0}'.format(self.original_files_path, args)) count = 1 for tarfile in tar_files: @@ -94,11 +99,12 @@ class Cmorizer(object): return file_list filtered = list() filters = self.cmor.filter_files.split(' ') - for filename in file_list: + for file_path in file_list: + filename = os.path.basename(file_path) if any(f in filename for f in filters): - filtered.append(filename) + filtered.append(file_path) else: - os.remove(filename) + os.remove(file_path) if len(filtered) == 0: Log.warning('Filters {0} do not match any of the files', filters) return filtered @@ -186,6 +192,8 @@ class Cmorizer(object): tar_files = glob.glob(os.path.join(self.original_files_path, 'MMA*')) tar_files.sort() count = 1 + if len(tar_files) == 0: + Log.error('MMA files not found in {0}'.format(self.original_files_path)) for tarfile in tar_files: if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), ModelingRealms.atmos): Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) @@ -302,11 +310,16 @@ class Cmorizer(object): def _get_atmos_timestep(self, gribfile): Log.info('Getting timestep...') grib_handler = pygrib.open(gribfile) - mes1 = grib_handler.message(1) - mes2 = grib_handler.readline() - while mes2.analDate == mes1.analDate: - mes2 = grib_handler.readline() - atmos_timestep = mes2.analDate - mes1.analDate + dates = set() + try: + while True: + mes = grib_handler.next() + dates.add(mes.analDate) + except StopIteration: + pass + dates = list(dates) + dates.sort() + atmos_timestep = dates[1] - dates[0] atmos_timestep = int(atmos_timestep.total_seconds() / 3600) self.experiment.atmos_timestep = atmos_timestep grib_handler.close() @@ -389,7 +402,7 @@ class Cmorizer(object): return temp = TempFile.get() - Utils.nco.ncks(input=file_path, output=temp, options='-v {0}'.format(variable)) + Utils.nco.ncks(input=file_path, output=temp, options=('-v {0}'.format(variable),)) self._rename_level_variables(temp, var_cmor) self._add_coordinate_variables(handler, temp) @@ -547,7 +560,7 @@ class Cmorizer(object): if var_name is not None: Utils.nco.ncks(input='{0}_{1}_1m.nc'.format(gribfile, var_code), output='{0}_{1}_1m.nc'.format(gribfile, var_code), - options='-O -v {0}'.format(var_name)) + options=('-O -v {0}'.format(var_name))) def _merge_and_cmorize_atmos(self, chunk_start, chunk_end, grid, frequency): merged_file = 'MMA_{0}_{1}_{2}_{3}.nc'.format(frequency, date2str(chunk_start), date2str(chunk_end), grid) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index a946dd328cd4126fb3415306fcb6313b25d3a72f..f572592bca90ed35c9e72887c0f9f99f47ed7d2b 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -304,12 +304,12 @@ class NetCDFFile(object): else: self._update_var_with_region_data() self._correct_metadata() - Utils.nco.ncks(input=self.local_file, output=self.local_file, options='-O --fix_rec_dmn region') + Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('-O --fix_rec_dmn region',)) def _update_var_with_region_data(self): temp = TempFile.get() shutil.copyfile(self.remote_file, temp) - Utils.nco.ncks(input=temp, output=temp, options='-O --mk_rec_dmn region') + Utils.nco.ncks(input=temp, output=temp, options=('-O --mk_rec_dmn region',)) handler = Utils.openCdf(temp) handler_send = Utils.openCdf(self.local_file) value = handler_send.variables[self.var][:] @@ -338,7 +338,7 @@ class NetCDFFile(object): value = original_var[:] new_var[..., 0] = value handler.close() - Utils.nco.ncks(input=self.local_file, output=self.local_file, options='-O -x -v {0}'.format(self.var)) + Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('-O -x -v {0}'.format(self.var),)) Utils.rename_variable(self.local_file, 'new_var', self.var) def _correct_metadata(self): @@ -364,18 +364,13 @@ class NetCDFFile(object): var_handler.short_name = self.cmor_var.short_name def _fix_values_metadata(self, var_type): + options = ['-O -a _FillValue,{0},o,{1},"1.e20"'.format(self.var, var_type.char), + '-a missingValue,{0},o,{1},"1.e20"'.format(self.var, var_type.char)] if self.cmor_var.valid_min != '': - valid_min = '-a valid_min,{0},o,{1},"{2}" '.format(self.var, var_type.char, self.cmor_var.valid_min) - else: - valid_min = '' + options.append('-a valid_min,{0},o,{1},"{2}" '.format(self.var, var_type.char, self.cmor_var.valid_min)) if self.cmor_var.valid_max != '': - valid_max = '-a valid_max,{0},o,{1},"{2}" '.format(self.var, var_type.char, self.cmor_var.valid_max) - else: - valid_max = '' - Utils.nco.ncatted(input=self.local_file, output=self.local_file, - options='-O -a _FillValue,{0},o,{1},"1.e20" ' - '-a missingValue,{0},o,{1},"1.e20" {2}{3}'.format(self.var, var_type.char, - valid_min, valid_max)) + options.append('-a valid_max,{0},o,{1},"{2}" '.format(self.var, var_type.char, self.cmor_var.valid_max)) + Utils.nco.ncatted(input=self.local_file, output=self.local_file, options=options) def _fix_coordinate_variables_metadata(self, handler): if 'lev' in handler.variables: diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index da6fe4538c34b3c294a6280b7a1bfdc4512a6023..1742e90d9d0393fc617169550bd4db4edbc3f78b 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -279,6 +279,7 @@ class EarthDiags(object): Diagnostic.register(Rotation) Diagnostic.register(Mxl) Diagnostic.register(VerticalGradient) + Diagnostic.register(MaskLand) def clean(self): Log.info('Removing scratch folder...') diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index 983609e146912820ce07b1178a2ff8c05e3d6e8b..cb4133130abf7d853ff08805584ed5c514773e85 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -91,7 +91,7 @@ class DailyMean(Diagnostic): handler = Utils.openCdf(variable_file) if 'region' in handler.variables: noregion = TempFile.get() - Utils.nco.ncks(input=variable_file, output=noregion, options='-O -C -x -v region') + Utils.nco.ncks(input=variable_file, output=noregion, options=('-O -C -x -v region',)) Utils.cdo.daymean(input=noregion, output=day_mean) monmean_handler = Utils.openCdf(day_mean) Utils.copy_variable(handler, monmean_handler, 'region') diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 3052a54a0191b56e9f586130e2da061af7eac3ab..ee3161791bb80891e4b4d75de83e96ea7ea636d6 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -89,7 +89,7 @@ class MonthlyMean(Diagnostic): handler = Utils.openCdf(variable_file) if 'region' in handler.variables: noregion = TempFile.get() - Utils.nco.ncks(input=variable_file, output=noregion, options='-O -C -x -v region') + Utils.nco.ncks(input=variable_file, output=noregion, options=('-O -C -x -v region',)) Utils.cdo.monmean(input=noregion, output=monmean) monmean_handler = Utils.openCdf(monmean) Utils.copy_variable(handler, monmean_handler, 'region') diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index a1fe6d0b01fe8d5193c0d7a0f0ff4a6f9168dc9a..dfd9d9a7d259d418988a065a42ef4d4fdf8f2d83 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -87,7 +87,8 @@ class SelectLevels(Diagnostic): variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) - Utils.nco.ncks(input=variable_file, output=variable_file, options='-O -d lev,{0.min_depth},{0.max_depth}'.format(self.box)) + Utils.nco.ncks(input=variable_file, output=variable_file, + options=('-O -d lev,{0.min_depth},{0.max_depth}'.format(self.box),)) self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 63444c1da8c24e7452b5c49534beb4667abfb182..1f645f8184f556143e6672ffb05aacd6570b8f77 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -97,12 +97,13 @@ class SimplifyDimensions(Diagnostic): temp = TempFile.get() new_file = Utils.openCdf(temp, 'w') for dim in handler.dimensions.keys(): - if dim in ('lat', 'lon', 'i', 'j'): + if dim in ('lat', 'lon', 'i', 'j', 'vertices'): continue Utils.copy_dimension(handler, new_file, dim, new_names={'i': 'lon', 'j': 'lat'}) new_file.createDimension('lon', handler.dimensions['i'].size) new_file.createDimension('lat', handler.dimensions['j'].size) + new_file.createDimension('vertices', 2) for var in handler.variables.keys(): if var in ('lat', 'lon', 'i', 'j', 'lat_vertices', 'lon_vertices'): @@ -128,9 +129,9 @@ class SimplifyDimensions(Diagnostic): if vertices_name in source.variables: var_vertices = source.variables[vertices_name] if var_name == 'lon': - vertices_values = var_vertices[0:1, ...] + vertices_values = var_vertices[0:1, :, 2:] else: - vertices_values = var_vertices[:, 0:1, :] + vertices_values = var_vertices[:, 0:1, 1:3] new_lat_vertices = destiny.createVariable(vertices_name, var_vertices.dtype, dimensions=(var_name, 'vertices')) new_lat_vertices[:] = vertices_values Utils.copy_attributes(new_lat_vertices, var_vertices) diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py index 99ee87d7c4323de39514960083cf97ea70441a81..07ae10022218b77b6aca5a86e03fa85f27dbcd67 100644 --- a/earthdiagnostics/general/yearlymean.py +++ b/earthdiagnostics/general/yearlymean.py @@ -91,7 +91,7 @@ class YearlyMean(Diagnostic): handler = Utils.openCdf(variable_file) if 'region' in handler.variables: noregion = TempFile.get() - Utils.nco.ncks(input=variable_file, output=noregion, options='-O -C -x -v region') + Utils.nco.ncks(input=variable_file, output=noregion, options=('-O -C -x -v region',)) Utils.cdo.yearmean(input=noregion, output=year_mean) monmean_handler = Utils.openCdf(year_mean) Utils.copy_variable(handler, monmean_handler, 'region') diff --git a/earthdiagnostics/ocean/__init__.py b/earthdiagnostics/ocean/__init__.py index 7512b3610c4c7db363a4fdf5e5ca1dd8fcc9b4bd..854d90d94bd5a3906a916bfe64798296cd774b55 100644 --- a/earthdiagnostics/ocean/__init__.py +++ b/earthdiagnostics/ocean/__init__.py @@ -23,3 +23,4 @@ from earthdiagnostics.ocean.regionmean import RegionMean from earthdiagnostics.ocean.rotation import Rotation from earthdiagnostics.ocean.mxl import Mxl from earthdiagnostics.ocean.verticalgradient import VerticalGradient +from earthdiagnostics.ocean.mask_land import MaskLand diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index e65fe622669435ca5f30abd12ec9e025be99cde0..a12fd7c6d83ceab594fbfa6ce227de2da7c381f1 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -99,7 +99,7 @@ class AreaMoc(Diagnostic): handler = Utils.openCdf(temp) if 'i' in handler.dimensions: handler.close() - nco.ncwa(input=temp, output=temp, options='-O -a i') + nco.ncwa(input=temp, output=temp, options=('-O -a i',)) handler = Utils.openCdf(temp) basin_index = np.where(handler.variables['basin'][:] == self.basin.fullname) @@ -114,7 +114,7 @@ class AreaMoc(Diagnostic): raise Exception('Basin {0} not defined in file') basin_index = basin_index[0][0] # To select basin and remove dimension - nco.ncwa(input=temp, output=temp, options='-O -d basin,{0} -a basin'.format(basin_index)) + nco.ncwa(input=temp, output=temp, options=('-O -d basin,{0} -a basin',).format(basin_index)) source = Utils.openCdf(temp) destiny = Utils.openCdf(temp2, 'w') diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index 282c0f5dded4d31b541400565439dda866d4f27b..4f2fb2e9efd554fea353bd85a98b69ad16a292e1 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -154,7 +154,7 @@ class CutSection(Diagnostic): mask_lev[:, :, listj[jpt], listi[jpt]]) new_coord[jpt] = old_coord[listj[jpt], listi[jpt]] - nco.ncks(input=temp, output=temp, options='-O -v lev,time') + nco.ncks(input=temp, output=temp, options=('-O -v lev,time',)) handler = Utils.openCdf(temp) if not self.zonal: diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 39729a9a62806d20bf98fc18dacfbf1ddaf44ac6..c94a3a5d8067d4e3e1fe31526646374087d1f8cb 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -124,7 +124,7 @@ class HeatContent(Diagnostic): if self.mxloption != 0: mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) - nco.ncks(input=mlotst_file, output=temperature_file, options='-A -v mlotst') + nco.ncks(input=mlotst_file, output=temperature_file, options=('-A -v mlotst',)) para = list() if self.box.min_depth != 0: @@ -155,7 +155,7 @@ class HeatContent(Diagnostic): results = Utils.openCdf(temp2) heatcsum_temp = TempFile.get() heatcvmean_temp = TempFile.get() - nco.ncks(input=temperature_file, output=heatcsum_temp, options='-O -v time') + nco.ncks(input=temperature_file, output=heatcsum_temp, options=('-O -v time',)) shutil.copy(heatcsum_temp, heatcvmean_temp) heatcsum_handler = Utils.openCdf(heatcsum_temp) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index cf398972bbcc26b89a42192c48746a57baa38187..003cee2df309093ded35e4990b81819e33d94360 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -166,7 +166,7 @@ class HeatContentLayer(Diagnostic): handler.renameVariable('thetao', 'heatc_sl') handler.close() - nco.ncks(input=thetao_file, output=results, options='-O -v lon,lat,time') + nco.ncks(input=thetao_file, output=results, options=('-O -v lon,lat,time',)) Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False, True) handler_results = Utils.openCdf(results) handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 7b0dcc847ead4471e6f9fc64e41a273a376999bc..10be14c5f11818cfc4436876615e01847922e85b 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -132,10 +132,10 @@ class Interpolate(Diagnostic): if 'record' in handler.dimensions: handler.renameDimension('record', 'lev') handler.close() - nco.ncpdq(input=temp, output=temp, options='-O -h -a time,lev') + nco.ncpdq(input=temp, output=temp, options=('-O -h -a time,lev',)) if has_levels: - nco.ncks(input=variable_file, output=temp, options='-A -v lev') + nco.ncks(input=variable_file, output=temp, options=('-A -v lev',)) for lev in range(0, num_levels): os.remove(self._get_level_file(lev)) temp2 = TempFile.get() @@ -145,7 +145,7 @@ class Interpolate(Diagnostic): cdo.invertlatdata(input=temp2, output=temp) shutil.move(temp, temp2) if not has_levels: - nco.ncks(input=temp2, output=temp2, options='-O -v {0},lat,lon,time'.format(self.variable)) + nco.ncks(input=temp2, output=temp2, options=('-O -v {0},lat,lon,time'.format(self.variable),)) self.send_file(temp2, self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) @@ -160,7 +160,7 @@ class Interpolate(Diagnostic): temp = TempFile.get() if has_levels: nco.ncks(input=input_file, output=temp, options='-O -d lev,{0} -v {1},lat,lon'.format(lev, self.variable)) - nco.ncwa(input=temp, output=temp, options='-O -h -a lev') + nco.ncwa(input=temp, output=temp, options=('-O -h -a lev',)) else: shutil.copy(input_file, temp) namelist_file = TempFile.get(suffix='') @@ -179,6 +179,6 @@ class Interpolate(Diagnostic): Utils.execute_shell_command('/home/Earth/jvegas/pyCharm/cfutools/interpolation/scrip_use ' '{0}'.format(namelist_file), Log.DEBUG) os.remove(namelist_file) - nco.ncecat(input=temp, output=temp, options="-O -h") + nco.ncecat(input=temp, output=temp, options=("-O -h",)) shutil.move(temp, self._get_level_file(lev)) Log.debug("Level {0} ready", lev) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 5a20154e150aebd06504c22bd7f0a97c44ab3f68..8bd9c40dfe85996330d655b30da7d622570a1595 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -112,7 +112,7 @@ class InterpolateCDO(Diagnostic): @classmethod def get_sample_grid_file(cls): temp = TempFile.get() - Utils.nco.ncks(input='mask.nc', output=temp, options='-O -v tmask,lat,lon,gphif,glamf') + Utils.nco.ncks(input='mask.nc', output=temp, options=('-O -v tmask,lat,lon,gphif,glamf',)) handler = Utils.openCdf(temp) lon = handler.variables['lon'] lon.units = "degrees_east" @@ -153,7 +153,7 @@ class InterpolateCDO(Diagnostic): handler.close() - Utils.nco.ncks(input=temp, output=temp, options='-O -x -v gphif,glamf') + Utils.nco.ncks(input=temp, output=temp, options=('-O -x -v gphif,glamf',)) return temp @classmethod diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py new file mode 100644 index 0000000000000000000000000000000000000000..9e51c041402a26c2612a456b351213aef490ed14 --- /dev/null +++ b/earthdiagnostics/ocean/mask_land.py @@ -0,0 +1,96 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, \ + DiagnosticDomainOption, DiagnosticChoiceOption, DiagnosticOption +from earthdiagnostics.utils import Utils +import numpy as np + + +class MaskLand(Diagnostic): + """ + Changes values present in the mask for NaNs + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + """ + + alias = 'maskland' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, mask, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.variable = variable + self.mask = mask + self.grid = grid + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable + + def __str__(self): + return 'Land mask Startdate: {0} Member: {1} Chunk: {2} Variable: {3}:{4} ' \ + 'Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, self.grid) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (level), maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption('domain'), + DiagnosticVariableOption('variable'), + DiagnosticChoiceOption('cell', ('t', 'u', 'v', 'f', 'w'), 't'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + + mask_file = Utils.openCdf('mask.nc') + cell_point = options['cell'] + # W and T share the same mask + if cell_point == 'w': + cell_point = 't' + mask = mask_file.variables['{0}mask'.format(cell_point)][:].astype(float) + mask[mask == 0] = np.nan + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MaskLand(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], mask, options['grid'])) + return job_list + + def compute(self): + """ + Runs the diagnostic + """ + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, + self.chunk, grid=self.grid) + + handler = Utils.openCdf(variable_file) + if not 'lev' in handler.dimensions: + mask = self.mask[:, 0, ...] + else: + mask =self.mask + handler.variables[self.variable][:] *= mask + handler.close() + + self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index 2717d413a9875703e443d0869f4e860ff29fbcf4..6d649edc72a12b9e4bac12cd6eabe44eb7c0f17d 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -102,7 +102,7 @@ class MaxMoc(Diagnostic): handler = Utils.openCdf(temp) if 'i' in handler.dimensions: handler.close() - nco.ncwa(input=temp, output=temp, options='-O -a i') + nco.ncwa(input=temp, output=temp, options=('-O -a i',)) else: handler.close() handler = Utils.openCdf(temp) diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index e880090106360a5d8ef90650e2436aad1fded493..cc7e3b5b3d9cd551eecde05f15658028381a78bf 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -72,7 +72,7 @@ class MixedLayerHeatContent(Diagnostic): mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) - Utils.nco.ncks(input=mlotst_file, output=temperature_file, options='-A -v mlotst') + Utils.nco.ncks(input=mlotst_file, output=temperature_file, options=('-A -v mlotst',)) temp = TempFile.get() cdftools.run('cdfmxlheatc', input=temperature_file, output=temp) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 0127fe64df69f8bc0078663bd472dbe0aa6913f9..0a0ea4782aca1e2a2b2e2f2ad6c98dea9bd15f41 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -69,7 +69,7 @@ class MixedLayerSaltContent(Diagnostic): mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) - Utils.nco.ncks(input=mlotst_file, output=salinity_file, options='-A -v mlotst') + Utils.nco.ncks(input=mlotst_file, output=salinity_file, options=('-A -v mlotst',)) temp = TempFile.get() cdftools.run('cdfmxlsaltc', input=salinity_file, output=temp) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 52bbc018ab9d3d0d93cf2bebc30d8faab02f32ec..1b0a463a351d1a1107727aa741a9d49cbe5fb123 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -74,7 +74,7 @@ class Moc(Diagnostic): Log.debug('Computing MOC') cdftools.run('cdfmoc', input=input_file, output=temp) - Utils.nco.ncks(input=input_file, output=temp, options='-A -v lev') + Utils.nco.ncks(input=input_file, output=temp, options=('-A -v lev',)) Utils.convert2netcdf4(temp) Log.debug('Reformatting variables') @@ -109,7 +109,7 @@ class Moc(Diagnostic): handler.close() Utils.nco.ncks(input=temp, output=temp, - options='-O -x -v zomsfglo,zomsfatl,zomsfpac,zomsfinp,zomsfind,zomsfinp0') + options=('-O -x -v zomsfglo,zomsfatl,zomsfpac,zomsfinp,zomsfind,zomsfinp0',)) Utils.setminmax(temp, 'vsftmyz') self.send_file(temp, ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 07954d7867c09d58b0c3c9c74514fa54e19c4bdf..630c2a1bd3e63a85fe239e863c78b4240a6a4cbb 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -147,7 +147,7 @@ class RegionMean(Diagnostic): levels = '' temp2 = TempFile.get() - Utils.nco.ncks(input=mean_file, output=temp2, options='-O -v {0},lat,lon{1}'.format(original_name, levels)) + Utils.nco.ncks(input=mean_file, output=temp2, options=('-O -v {0},lat,lon{1}'.format(original_name, levels),)) self.send_file(temp2, ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, box=box_save, rename_var=original_name, region=self.basin, grid=self.grid) diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index f9e758a5c16966981d9696d96b267e5d739eaeac..ed949cc6447508f66f713612e48d846d504f7a44 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -119,12 +119,12 @@ class Rotation(Diagnostic): temp = TempFile.get() if self.has_levels: Utils.nco.ncecat(input=self._get_level_file(0, direction), output=temp, - options="-n {0},2,1 -v '{1}'".format(self.num_levels, var)) + options=("-n {0},2,1 -v '{1}'".format(self.num_levels, var),)) handler = Utils.openCdf(temp) if 'record' in handler.dimensions: handler.renameDimension('record', 'lev') handler.close() - Utils.nco.ncpdq(input=temp, output=temp, options='-O -h -a time,lev') + Utils.nco.ncpdq(input=temp, output=temp, options=('-O -h -a time,lev',)) Utils.rename_variables(temp, {'x': 'i', 'y': 'j'}, must_exist=False, rename_dimension=True) else: Utils.move_file(self._get_level_file(0, direction), temp) @@ -140,8 +140,8 @@ class Rotation(Diagnostic): def _extract_level(self, input_file, var, level): temp = TempFile.get() if self.has_levels: - Utils.nco.ncks(input=input_file, output=temp, options='-O -d lev,{0} -v {1},lat,lon'.format(level, var)) - Utils.nco.ncwa(input=temp, output=temp, options='-O -h -a lev') + Utils.nco.ncks(input=input_file, output=temp, options=('-O -d lev,{0} -v {1},lat,lon'.format(level, var),)) + Utils.nco.ncwa(input=temp, output=temp, options=('-O -h -a lev',)) else: shutil.copy(input_file, temp) return temp diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 302e33e0e4160f2f46a1985d7c7d07600e15b459..2e32e689e77743881aaef346c92f86c5ba821232 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -81,9 +81,9 @@ class Utils(object): var = handler.variables[variable] values = [np.max(var), np.min(var)] Utils.nco.ncatted(input=filename, output=filename, - options='-h -a valid_max,{0},m,f,{1}'.format(variable, values[0])) + options=('-h -a valid_max,{0},m,f,{1}'.format(variable, values[0]),)) Utils.nco.ncatted(input=filename, output=filename, - options='-h -a valid_min,{0},m,f,{1}'.format(variable, values[1])) + options=('-h -a valid_min,{0},m,f,{1}'.format(variable, values[1]),)) handler.close() @staticmethod diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index d0bce08daeebed94870e7605a45bf0f268a24fbe..270878980351093c8e99b665049b01b2501aa78f 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -135,7 +135,10 @@ class VariableManager(object): if file_name in ('CMIP6_grids.json', 'CMIP6_formula_terms.json'): continue json_data = open(os.path.join(json_folder, file_name)).read() - data = json.loads(json_data) + try: + data = json.loads(json_data) + except ValueError: + continue if 'variable_entry' in data: Log.debug('Parsing file {0}'.format(file_name)) table = CMORTable(data['Header']['table_id'][6:], diff --git a/earthdiagnostics/variable_alias/cmip6.csv b/earthdiagnostics/variable_alias/cmip6.csv index abe2ddb8e24c1648546db92daa70e001e9a22359..d34dc918a1b934336c6c72762364f47aab6c8ac7 100644 --- a/earthdiagnostics/variable_alias/cmip6.csv +++ b/earthdiagnostics/variable_alias/cmip6.csv @@ -33,7 +33,8 @@ c-export,c-export,, tintpp,tintpp,, tnfix,tnfix,, tdenit,tdenit,, -intppnew:inttppnew,intppnew,, +intppnew,intppnew,, +inttppnew,inttppnew,, inttpbfe,pbfe,, intdic,intdic,, o2min,o2min,, @@ -43,7 +44,8 @@ intppphy,intppphy,, intppphy2,intppdiat,, ppphy ,ppphy ,, ppphy2 ,pdi,, -intpp:inttpp,intpp,, +intpp,intpp,, +inttpp,inttpp,, intpbfe,intpbfe,, intpbsi,intpbsi,, intpbcal,intpbcal,, diff --git a/setup.py b/setup.py index e8c07f09ce0c3a4886dd059eb3ab7b3f4b7dacb3..420db184d81c366160d2366d0b9615af0599007b 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( url='http://www.bsc.es/projects/earthsciences/autosubmit/', keywords=['climate', 'weather', 'diagnostic'], setup_requires=['pyproj'], - install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo', 'nco', 'cfunits>=1.1.4', 'coverage', + install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo', 'nco>=0.0.3', 'cfunits>=1.1.4', 'coverage', 'pygrib', 'openpyxl', 'mock'], packages=find_packages(), include_package_data=True,