diff --git a/diags.conf b/diags.conf index 6ff22c617e122a8098b8c871804aa2d89f7b7f30..ca5be109777afb51cf4eca7363996bda85db900f 100644 --- a/diags.conf +++ b/diags.conf @@ -16,13 +16,13 @@ CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = +DIAGS = regmean,ocean,thetao,T # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = 6hr +FREQUENCY = mon # Path to CDFTOOLS binaries -CDFTOOLS_PATH = +CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir RESTORE_MESHES = False # Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available) @@ -30,7 +30,7 @@ MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False -FORCE = True +FORCE = False # If true, CMORizes ocean files. Default = True OCEAN_FILES = True FILTER_FILES = 3h 6h diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst index 31fd63c5b5f50f7eeb1d7e29630c281c73467930..f8392b48d49bc1ee0072fc1481e0f8abccda9f16 100644 --- a/doc/source/diagnostic_list.rst +++ b/doc/source/diagnostic_list.rst @@ -193,9 +193,6 @@ averagesection Compute an average of a given zone. The variable MUST be in a regular grid See :class:`~earthdiagnostics.ocean.averagesection.AverageSection` -.. warning:: - This diagnostic is not finished - Options: ******** @@ -427,6 +424,41 @@ Options: This diagnostic has no options +regmean +~~~~~~~ + +Compute an average of a given zone using cdfmean from CDFTOOLS +See :class:`~earthdiagnostics.ocean.regionmean.RegionMean` + +.. warning:: + This diagnostic is a recent addition and needs more testing to be reliable + +Options: +******** + +1. Domain: + Variable domain + +2. Variable: + Variable to average + +3. Grid: + NEMO grid used to store the variable: T, U, V ... + +4. Basin = Global: + Basin to compute + +5. Save 3d = False: + If True, it also stores the average per level + +6. Min depth: + Minimum depth to compute in levels. If -1, average from the surface + +7. Max depth: + Maximum depth to compute in levels. If -1, average to the bottom + + + siasiesiv ~~~~~~~~~ diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index 32eabc34ed6d74d18beb3cab81e379e94ca886a7..792853ba2100f58f53db5961ef57ef7cbda5758f 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -16,7 +16,7 @@ class CDFTools(object): self.path = path # noinspection PyShadowingBuiltins - def run(self, command, input, output=None, options=None, log_level=Log.INFO): + def run(self, command, input, output=None, options=None, log_level=Log.INFO, input_option=None): """ Runs one of the CDFTools @@ -35,6 +35,8 @@ class CDFTools(object): line = [os.path.join(self.path, command)] self._check_command_existence(line[0]) + if input_option: + line.append(input_option) self._check_input(command, input, line) if options: if isinstance(options, basestring): diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 5970699faef891dc1d77ff708428ef027dff99cf..fc0a684938678df831c05c191c17fe5cd776623d 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -158,14 +158,19 @@ class CMORManager(DataManager): member + 1, time_bound) elif self.config.data_convention in ('primavera', 'cmip6'): - file_name = '{0}_{1}_{2}_{3}_S{4}-r{5}i1p1_{6}{7}'.format(var, - cmor_table.name, - self.experiment.experiment_name, - self.experiment.model, - startdate, - member + 1, - grid, - time_bound) + if grid is None: + grid = '' + else: + grid = '_{0}'.format(grid) + + file_name = '{0}_{1}_{2}_{3}_S{4}-r{5}i1p1{6}{7}'.format(var, + cmor_table.name, + self.experiment.experiment_name, + self.experiment.model, + startdate, + member + 1, + grid, + time_bound) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 83b1e44e001f8b538883b7d8ba90841ed944adb0..3f9a370dd32ab69538fa18b656c0bcaaab79b9a6 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,5 +1,5 @@ # coding=utf-8 -from earthdiagnostics.constants import Basins +from earthdiagnostics.constants import Basins, Basin from earthdiagnostics.frequency import Frequency from earthdiagnostics.variable_type import VariableType from earthdiagnostics.modelingrealm import ModelingRealms @@ -82,6 +82,8 @@ class Diagnostic(object): :type vartype: VariableType :return: """ + if isinstance(region, Basin): + region = region.fullname self.data_manager.send_file(filetosend, domain, var, startdate, member, chunk, grid, region, box, rename_var, frequency, year, date_str, move_old, diagnostic=self, vartype=vartype) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index b3a35c788d682616a457430723d30b2ba5c05b71..a4c40388c57b86991bd3cda9ac9d20cdca4d0adf 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -151,6 +151,7 @@ class EarthDiags(object): # Run diagnostics Log.info('Running diagnostics') list_jobs = self.prepare_job_list() + self._failed_jobs = [] time = datetime.datetime.now() Log.info("Starting to compute at {0}", time) @@ -170,6 +171,7 @@ class EarthDiags(object): finish_time = datetime.datetime.now() Log.result("Diagnostics finished at {0}", finish_time) Log.result("Elapsed time: {0}\n", finish_time - time) + self.print_errors() self.print_stats() return not self.had_errors @@ -193,6 +195,16 @@ class EarthDiags(object): for diag, time in sorted(total.items(), key=operator.itemgetter(1)): Log.info('{0:23} {1:}', diag.__name__, time) + def print_errors(self): + if len(self._failed_jobs) == 0: + return + self.had_errors = True + Log.error('Failed jobs') + Log.error('-----------') + for job in self._failed_jobs: + Log.error(str(job)) + Log.info('') + def prepare_job_list(self): list_jobs = Queue.Queue() for fulldiag in self.config.get_commands(): @@ -252,6 +264,7 @@ class EarthDiags(object): Diagnostic.register(MixedLayerHeatContent) Diagnostic.register(HeatContentLayer) Diagnostic.register(HeatContent) + Diagnostic.register(RegionMean) def clean(self): Log.info('Removing scratch folder...') @@ -345,9 +358,7 @@ class EarthDiags(object): else: Log.result('Thread {0} finished after running successfully {1} of {2} tasks', numthread, count, count + len(failed_jobs)) - self.had_errors = True - for job in failed_jobs: - Log.error('Job {0} could not be run', job) + self._failed_jobs += failed_jobs return def _prepare_mesh_files(self): diff --git a/earthdiagnostics/ocean/__init__.py b/earthdiagnostics/ocean/__init__.py index 0628fb68cbf6a22162783c241bd460736c9a3138..8443762a8ae4d32162b9ecf24f7b9f40a6d2e48d 100644 --- a/earthdiagnostics/ocean/__init__.py +++ b/earthdiagnostics/ocean/__init__.py @@ -19,3 +19,4 @@ from earthdiagnostics.ocean.mixedlayersaltcontent import MixedLayerSaltContent from earthdiagnostics.ocean.siasiesiv import Siasiesiv from earthdiagnostics.ocean.heatcontentlayer import HeatContentLayer from earthdiagnostics.ocean.mixedlayerheatcontent import MixedLayerHeatContent +from earthdiagnostics.ocean.regionmean import RegionMean diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 5c8464e0ebf320f50d30c5651eac633e936a6b62..e65fe622669435ca5f30abd12ec9e025be99cde0 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -54,8 +54,9 @@ class AreaMoc(Diagnostic): self.basin == other.basin and self.box == other.box def __str__(self): - return 'Area MOC Startdate: {0} Member: {1} Chunk: {2} Box: {3}'.format(self.startdate, self.member, - self.chunk, self.box) + return 'Area MOC Startdate: {0} Member: {1} Chunk: {2} Box: {3} Basin: {4}'.format(self.startdate, self.member, + self.chunk, self.box, + self.basin) @classmethod def generate_jobs(cls, diags, options): diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index acc6334d2c53bf4624cf180764d1865f22ac03f6..07f91ffd18e87c2dd54ebe39d579a9e51b115bba 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -98,12 +98,14 @@ class HeatContent(Diagnostic): nco.ncks(input=mlotst_file, output=temperature_file, options='-A -v mlotst') para = list() - para.append('0') - para.append('0') - para.append('0') - para.append('0') - para.append(self.box.min_depth) - para.append(self.box.max_depth) + if self.box.min_depth != 0: + para.append('-zoom') + para.append(0) + para.append(0) + para.append(0) + para.append(0) + para.append(self.box.min_depth) + para.append(self.box.max_depth) if self.mxloption != 0: para.append('-mxloption') para.append(str(self.mxloption)) @@ -113,13 +115,15 @@ class HeatContent(Diagnostic): raise Exception('Basin {0} is not defined on mask_regions.nc'.format(self.basin.fullname)) handler.close() - para.append('-maskfile') + para.append('-M') para.append('mask_regions.3d.nc') - para.append('-mask') para.append(self.basin.fullname) - shell_output = cdftools.run('cdfheatc', options=para, input=temperature_file) + temp2 = TempFile.get() + cdftools.run('cdfheatc', options=para, input=temperature_file, output=temp2, input_option='-f') + + results = Utils.openCdf(temp2) heatcsum_temp = TempFile.get() heatcvmean_temp = TempFile.get() nco.ncks(input=temperature_file, output=heatcsum_temp, options='-O -v time') @@ -130,36 +134,19 @@ class HeatContent(Diagnostic): thc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" thc.long_name = "Total heat content" thc.units = "J" + thc[:] = results.variables['heatc3d'][:, 0, 0] + heatcsum_handler.close() heatcvmean_handler = Utils.openCdf(heatcvmean_temp) uhc = heatcvmean_handler.createVariable('heatcvmean', float, 'time') uhc.standard_name = "integral_of_sea_water_potential_temperature_expressed_as_heat_content" uhc.long_name = "Heat content per unit volume" uhc.units = "J*m^-3" - - time = 0 - # noinspection PyUnboundLocalVariable - for lines in shell_output: - if not lines: - continue - - for line in lines.split('\n'): - line = line.lstrip() - if line.startswith("Heat Content at level"): - Log.info(line) - elif line.startswith("Total Heat content/volume"): - Log.user_warning(line) - uhc[time] = line[line.index(':') + 1: line.index('Joules')] - time += 1 - if line.startswith("Total Heat content "): - Log.result(line) - thc[time] = line[line.index(':') + 1: line.index('Joules')] - elif line.startswith('TIME : '): - Log.info(line) - - heatcsum_handler.close() + uhc[:] = results.variables['heatc3dpervol'][:, 0, 0] heatcvmean_handler.close() + results.close() + if self.box.min_depth == 0: # For cdftools, this is all levels box_save = None diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py new file mode 100644 index 0000000000000000000000000000000000000000..43e986da895f6f77a121cf7a053d404d573e9d6e --- /dev/null +++ b/earthdiagnostics/ocean/regionmean.py @@ -0,0 +1,118 @@ +# coding=utf-8 +from earthdiagnostics import cdftools +from earthdiagnostics.box import Box +from earthdiagnostics.constants import Basins +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption, DiagnosticDomainOption, \ + DiagnosticBoolOption, DiagnosticBasinOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class RegionMean(Diagnostic): + """ + Chooses vertical level in ocean, or vertically averages between + 2 or more ocean levels + + :original author: Javier Vegas-Regidor + + :created: January 2017 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + :param box: box used to restrict the vertical mean + :type box: Box + """ + + alias = 'regmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, box, save3d, basin): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.variable = variable + self.grid = grid.upper() + self.box = box + self.save3d = save3d + self.basin = basin + self.required_vars = [variable] + self.generated_vars = [variable + 'vmean'] + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.variable == other.variable + + def __str__(self): + return 'Vertical mean Startdate: {0} Member: {1} Chunk: {2} Variable: {3} ' \ + 'Box: {4}'.format(self.startdate, self.member, self.chunk, self.variable, self.box) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (level), maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption('domain'), + DiagnosticOption('variable'), + DiagnosticOption('grid'), + DiagnosticBasinOption('basin', Basins.Global), + DiagnosticBoolOption('save3D', False), + DiagnosticIntOption('min_depth', 0), + DiagnosticIntOption('max_depth', 0)) + options = cls.process_options(options, options_available) + + box = Box() + box.min_depth = options['min_depth'] + box.max_depth = options['max_depth'] + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(RegionMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'], box, options['save3D'], + options['basin'])) + return job_list + + def compute(self): + """ + Runs the diagnostic + """ + temp = TempFile.get() + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) + + cdfmean_options = [self.variable, self.grid, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] + if self.basin != Basins.Global: + cdfmean_options.append('-M') + cdfmean_options.append('mask_regions.3d.nc') + cdfmean_options.append(self.basin.shortname) + + cdftools.run('cdfmean', input=variable_file, output=temp, options=cdfmean_options) + Utils.setminmax(temp, 'mean_{0}'.format(self.variable)) + + if self.box.min_depth == 0: + # For cdftools, this is all levels + box_save = None + else: + box_save = self.box + + self.send_file(temp, ModelingRealms.ocean, self.variable + 'mean', self.startdate, self.member, self.chunk, + box=box_save, rename_var='mean_{0}'.format(self.variable), region=self.basin) + if self.save3d: + Utils.setminmax(temp, 'mean_3D{0}'.format(self.variable)) + self.send_file(temp, ModelingRealms.ocean, self.variable + '3dmean', self.startdate, self.member, + self.chunk, box=box_save, rename_var='mean_3D{0}'.format(self.variable), region=self.basin) + diff --git a/earthdiagnostics/variable_alias/default.csv b/earthdiagnostics/variable_alias/default.csv index 24683fe3fed92fba78ad1a7f9daedabc3db6ae72..2a4146d0d6986d2cfa178c98416c1eac1d7804d3 100644 --- a/earthdiagnostics/variable_alias/default.csv +++ b/earthdiagnostics/variable_alias/default.csv @@ -170,7 +170,7 @@ iice_hsd:snthicat,sndcat,, isnoheco,snheco,, sd,snld,, smlt,snm,, -isnowthi,snthic,, +isnowthi,snld,, sbgvoltot,snvolga,, snvolu,snvolu,, vosaline:mean_3Dsosaline,so,,