diff --git a/VERSION b/VERSION index 187366829bf28bb4094ebffa20e3e1959623e9de..996c69056fb9ede4b8ff8de2ef3916945a484432 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0b40 +3.0.0b44 diff --git a/doc/source/codedoc/general.rst b/doc/source/codedoc/general.rst index d7b2f88b41fc2cd0b539687ea8ece99701ff4fcc..d4fc8ea0828502b7a6869ed4f71b02a66e0be836 100644 --- a/doc/source/codedoc/general.rst +++ b/doc/source/codedoc/general.rst @@ -7,6 +7,12 @@ earthdiagnostics.general.attribute :show-inheritance: :members: +earthdiagnostics.general.dailymean +---------------------------------- +.. automodule:: earthdiagnostics.general.dailymean + :show-inheritance: + :members: + earthdiagnostics.general.monthlymean ------------------------------------ .. automodule:: earthdiagnostics.general.monthlymean @@ -37,3 +43,9 @@ earthdiagnostics.general.scale .. automodule:: earthdiagnostics.general.scale :show-inheritance: :members: + +earthdiagnostics.general.yearlymean +----------------------------------- +.. automodule:: earthdiagnostics.general.yearlymean + :show-inheritance: + :members: diff --git a/doc/source/codedoc/ocean.rst b/doc/source/codedoc/ocean.rst index 2da23cabf3372ccd784b4fb0980f812d4663cd7c..5ca11d71e4e36774561732b93ade7f2b714069a7 100644 --- a/doc/source/codedoc/ocean.rst +++ b/doc/source/codedoc/ocean.rst @@ -85,6 +85,12 @@ earthdiagnostics.ocean.psi :show-inheritance: :members: +earthdiagnostics.ocean.rotation +------------------------------- +.. automodule:: earthdiagnostics.ocean.rotation + :show-inheritance: + :members: + earthdiagnostics.ocean.siasiesiv -------------------------------- .. automodule:: earthdiagnostics.ocean.siasiesiv diff --git a/doc/source/conf.py b/doc/source/conf.py index 33a724e873d0e579779fcefcf15c954f643fe18f..d749566967f93f1f463d5de176753fb529985382 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b40' +release = '3.0.0b44' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/config_file.rst b/doc/source/config_file.rst index b4bd369813afe34a9155bd45422413ac5b959272..dde7685aa3d0f930fd55a310416f42075cca43bf 100644 --- a/doc/source/config_file.rst +++ b/doc/source/config_file.rst @@ -30,10 +30,21 @@ Mandatory configurations Default data frequency to be used by the diagnostics. Some diagnostics can override this configuration or even ignore it completely. +* DIAGS: + List of diagnostic to run, in the order you want them to run + Optional configurations ~~~~~~~~~~~~~~~~~~~~~~~ +* SCRATCH_MASKS + Common scratch folder for the ocean masks. This is useful to avoid replicating them for each run at the fat nodes. + By default is '/scratch/Earth/ocean_masks' + +* RESTORE_MESHES + By default, Earth Diagnostics only copies the mask files if they are not present in the scratch folder. If this + option is set to true, Earth Diagnostics will copy them regardless of existence. Default is False. + * DATA_ADAPTOR This is used to choose the mechanism for storing and retrieving data. Options are CMOR (for our own experiments) or THREDDS (for anything else). Default value is CMOR @@ -66,10 +77,10 @@ This sections contains options related to the experiment's definition or configu Model version. Used to get the correct mask and mesh files * ATMOS_TIMESTEP - Time between outputs from the atmosphere. This is not the model simulation timestep! + Time between outputs from the atmosphere. This is not the model simulation timestep! Default is 6. * OCEAN_TIMESTEP - Time between outputs from the ocean. This is not the model simulation timestep! + Time between outputs from the ocean. This is not the model simulation timestep! Default is 6. * ATMOS_GRID Atmospheric grid definition. Will be used as a default target for interpolation diagnostics. @@ -87,12 +98,19 @@ This sections contains options related to the experiment's definition or configu Startdates to run as a space separated list * MEMBER - Members to run as a space separated integer list + Members to run as a space separated list. You can just provide the number or also add the prefix * MEMBER_DIGITS Number of minimum digits to compose the member name. By default it is 1. For example, for member 1 member name will be fc1 if MEMBER_DIGITS is 1 or fc01 if MEMBER_DIGITS is 2 +* MEMBER_PREFIX + Prefix to use for the member names. By default is 'fc' + +* MEMBER_COUNT_START + Number corresponding to the first member. For example, if your first member is 'fc1', it should be 1. + If it is 'fc0', it should be 0. By default is 0 + * CHUNK_SIZE Length of the chunks in months diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst index 74c5b46a99b9958f9d4f00038ec2d413cfb35007..d678b1260a2bbe3f3f48c733f811c5a8421983be 100644 --- a/doc/source/diagnostic_list.rst +++ b/doc/source/diagnostic_list.rst @@ -43,6 +43,30 @@ Options: 5. Grid = '': Variable grid. Only required in case that you want to use interpolated data. +dailymean +~~~~~~~~~ +Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.general.dailymean.DailyMean` + +.. warning:: + + This diagnostic does not use the frequency configuration from the config file. You must specify the original + frequency when calling it. + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Original frequency: + Original frequency to use + +4. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + monmean ~~~~~~~ Calculates the monthly mean for a given variable. See :class:`~earthdiagnostics.general.monthlymean.MonthlyMean` @@ -151,6 +175,34 @@ Options: 7. Max limit = NaN: If there is any value above this threshold, scale will not be applied +8. Frequencies = [*Default_frequency*]: + List of frequencies ('-' separated) to apply the scale on. Default is the frequency defined globally for all the + diagnostics + +yearlymean +~~~~~~~~~~ +Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.general.yearlymean.YearlyMean` + +.. warning:: + + This diagnostic does not use the frequency configuration from the config file. You must specify the original + frequency when calling it. + +Options: +******** + +1. Variable: + Variable name + +2. Domain: + Variable domain + +3. Original frequency: + Original frequency to use + +4. Grid = '': + Variable grid. Only required in case that you want to use interpolated data. + Ocean ----- @@ -325,6 +377,10 @@ Options: 4. Invert latitude: If True, inverts the latitude in the output file. +5. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) + interpolateCDO ~~~~~~~~~~~~~~ @@ -350,6 +406,9 @@ Options: If True, replaces the values in the ocean by NaN. You must only set it to false if, for some reason, you are interpolating an atmospheric or land variable that is stored in the NEMO grid (yes, this can happen, i.e. with tas). +5. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) maxmoc ~~~~~~ @@ -428,11 +487,11 @@ This diagnostic has no options regmean ~~~~~~~ -Compute an average of a given zone using cdfmean from CDFTOOLS +Computes the mean value of the field (3D, weighted). For 3D fields, +a horizontal mean for each level is also given. If a spatial window +is specified, the mean value is computed only in this window. See :class:`~earthdiagnostics.ocean.regionmean.RegionMean` -.. warning:: - This diagnostic is a recent addition and needs more testing to be reliable Options: ******** @@ -443,22 +502,54 @@ Options: 2. Variable: Variable to average -3. Grid: - NEMO grid used to store the variable: T, U, V ... +3. Grid_point: + NEMO grid point used to store the variable: T, U, V ... 4. Basin = Global: Basin to compute -5. Save 3d = False: - If True, it also stores the average per level - -6. Min depth: +5. Min depth: Minimum depth to compute in levels. If -1, average from the surface -7. Max depth: +6. Max depth: Maximum depth to compute in levels. If -1, average to the bottom +7. Save 3d = True: + If True, it also stores the average per level + +8. Variance = False: + If True, it also stores the variance +5. Original grid = '': + Source grid to choose. By default this is the original data, but sometimes you will want to use another + (for example, the 'rotated' one produced by the rotation diagnostic) + + +rotate +~~~~~~ + +Rotates the given variables +See :class:`~earthdiagnostics.ocean.rotation.Rotation` + + +Options: +******** + +1. Variable u: + Variable's u component + +2. Variable v: + Variable's u component + +3. Domain = ocean: + Variable domain: + +4. Executable = /home/Earth/jvegas/pyCharm/cfutools/interpolation/rotateUVorca: + Path to the executable that will compute the rotation + +.. warning:: + This default executable has been compiled for ORCA1 experiments. For other resolutions you must use other + executables compiled ad-hoc for them siasiesiv ~~~~~~~~~ diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index cfc8af858302bbaee43ef698e7be55de1fca2018..ed0000d9001949b30945bae659f339eded7a885c 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cmor_tables/default.csv b/earthdiagnostics/cmor_tables/default.csv index 811422d90c4e40f3db21315432063028d98ec89c..fabfbfbb20c504562c5ffe82db00f5c1ea94d6ad 100644 --- a/earthdiagnostics/cmor_tables/default.csv +++ b/earthdiagnostics/cmor_tables/default.csv @@ -338,3 +338,8 @@ mud,mud,realized_growth_rate_for_diatomes,Realized growth rate for diatomes,ocnB ppnewn,ppnewn,new_primary_production_of_nanophyto,New Primary production of nanophyto,ocnBgchem,,,,,, ppnewd,ppnewd,new_primary_production_of_diatoms,New Primary production of diatoms,ocnBgchem,,,,,, dic,dic,disolved_inorganic_carbon,Disolved Inorganic Carbon,ocnBgchem,,,,,, +zqla,hflso,surface_downward_latent_heat_flux,Surface Downward Latent Heat Flux,ocean,,W m-2,,,, +zqsb,hfsso,surface_downward_sensible_heat_flux,Surface Downward Sensible Heat Flux,ocean,,W m-2,,,, +zqlw,rlntds,surface_net_downward_longwave_flux,Surface Net Downward Longwave Radiation,ocean,,W m-2,,,, +var78,tclw,total_column_liquid_water,Total column liquid water,atmos,,kg m-2,,,, +var79,tciw,total_column_ice_water,Total column ice water,atmos,,kg m-2,,,, \ No newline at end of file diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index d704f87f52ad5dad1fc3aa065d157d4caff710bf..6be46960233e9aaa5614c013c659008e247d63d3 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -29,7 +29,7 @@ class Cmorizer(object): """ - NON_DATA_VARIABLES = ('lon', 'lat', 'time', 'time_bnds', 'leadtime', 'lev', 'icethi', + NON_DATA_VARIABLES = ('lon', 'lat', 'time', 'time_bnds', 'leadtime', 'lev', 'lev_2', 'icethi', 'deptht', 'depthu', 'depthw', 'depthv', 'time_centered', 'time_centered_bounds', 'deptht_bounds', 'depthu_bounds', 'depthv_bounds', 'depthw_bounds', 'deptht_bnds', 'depthu_bnds', 'depthv_bnds', 'depthw_bnds', @@ -397,7 +397,11 @@ class Cmorizer(object): self.data_manager.send_file(temp, var_cmor.domain, var_cmor.short_name, self.startdate, self.member, frequency=frequency, rename_var=variable, date_str=date_str, region=region, move_old=True, grid=alias.grid, cmorized=True) - Log.info('Variable {0.domain}:{0.short_name} processed', var_cmor) + if region: + region_str = ' (Region {})'.format(region) + else: + region_str = '' + Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str) def get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index dcfed271f3e1607756959351bbc8d949d81fd21a..10c45421d1c8a360040da60b04aee4661e1d84b9 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -534,5 +534,5 @@ class CMORManager(DataManager): self.experiment.model, self.experiment.experiment_name, 'S' + startdate) def _get_member_str(self, member): - return 'r{0}i1p1'.format(member + 1) + return 'r{0}i1p1'.format(member + 1 - self.experiment.member_count_start) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 8e062a9b814d75a84578bd70bda234705dc5c12d..ba1efb7ad244a6f55248092570c2735d92b677fd 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -27,7 +27,7 @@ class Config(object): "Scratch folder path" self.scratch_dir = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_DIR') "Scratch folder path" - self.scratch_masks = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_MASKS', '') + self.scratch_masks = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_MASKS', '/scratch/Earth/ocean_masks') "Common scratch folder for masks" self.data_dir = parser.get_path_option('DIAGNOSTICS', 'DATA_DIR') "Root data folder path" @@ -201,18 +201,20 @@ class ExperimentConfig(object): self.institute = parser.get_option('EXPERIMENT', 'INSTITUTE') self.expid = parser.get_option('EXPERIMENT', 'EXPID') self.experiment_name = parser.get_option('EXPERIMENT', 'NAME', self.expid) - - self.members = parser.get_int_list_option('EXPERIMENT', 'MEMBERS') + self.members = parser.get_list_option('EXPERIMENT', 'MEMBERS') self.member_digits = parser.get_int_option('EXPERIMENT', 'MEMBER_DIGITS', 1) + self.member_prefix = parser.get_option('EXPERIMENT', 'MEMBER_PREFIX', 'fc') + self.member_count_start = parser.get_int_option('EXPERIMENT', 'MEMBER_COUNT_START', 0) + self.members = [int(mem) if mem.startswith(self.member_prefix) else int(mem) for mem in self.members] self.startdates = parser.get_option('EXPERIMENT', 'STARTDATES').split() self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') self.model = parser.get_option('EXPERIMENT', 'MODEL') - self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) - self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) self.model_version = parser.get_option('EXPERIMENT', 'MODEL_VERSION', '') self.atmos_grid = parser.get_option('EXPERIMENT', 'ATMOS_GRID', '') + self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) + self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) def get_chunk_list(self): """ @@ -308,5 +310,5 @@ class ExperimentConfig(object): :return: member's name :rtype: str """ - return 'fc{0}'.format(str(member).zfill(self.member_digits)) + return '{0}{1}'.format(self.member_prefix, str(member).zfill(self.member_digits)) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 9e4c7a369053812f086c63ceba58c5ce1579b9f3..40b30b77ab276f9858c6ce9e71f9e7a22fb9c4c4 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -391,6 +391,8 @@ class NetCDFFile(object): def _fix_units(self, var_handler): if 'units' not in var_handler.ncattrs(): return + if var_handler.units == '-': + var_handler.units = '1.0' if var_handler.units == 'PSU': var_handler.units = 'psu' if var_handler.units == 'C' and self.cmor_var.units == 'K': @@ -401,7 +403,7 @@ class NetCDFFile(object): def _convert_units(self, var_handler): try: - self._convert_using_cfunits(var_handler) + Utils.convert_units(var_handler, self.cmor_var.units) except ValueError: factor, offset = UnitConversion.get_conversion_factor_offset(var_handler.units, self.cmor_var.units) @@ -412,16 +414,7 @@ class NetCDFFile(object): if 'valid_max' in var_handler.ncattrs(): var_handler.valid_max = float(var_handler.valid_max) * factor + offset - def _convert_using_cfunits(self, var_handler): - new_unit = Units(self.cmor_var.units) - old_unit = Units(var_handler.units) - var_handler[:] = Units.conform(var_handler[:], old_unit, new_unit, inplace=True) - if 'valid_min' in var_handler.ncattrs(): - var_handler.valid_min = Units.conform(float(var_handler.valid_min), old_unit, new_unit, - inplace=True) - if 'valid_max' in var_handler.ncattrs(): - var_handler.valid_max = Units.conform(float(var_handler.valid_max), old_unit, new_unit, - inplace=True) + def _rename_coordinate_variables(self): variables = dict() @@ -452,7 +445,7 @@ class NetCDFFile(object): handler = Utils.openCdf(self.local_file) try: - history_line = handler.history + history_line + history_line = history_line + handler.history except AttributeError: history_line = history_line handler.history = Utils.convert_to_ASCII_if_possible(history_line) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 3548451d0e7e039979b8c3db6426c84d5f48b27b..dfc2082a83f814f51582c2881887a6aeb48ba32c 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -197,6 +197,19 @@ class DiagnosticListIntOption(DiagnosticOption): return values +class DiagnosticListFrequenciesOption(DiagnosticOption): + + def __init__(self, name, default_value=None): + super(DiagnosticListFrequenciesOption, self).__init__(name, default_value) + + def parse(self, option_value): + option_value = self.check_default(option_value) + if isinstance(option_value, tuple) or isinstance(option_value, list): + return option_value + values = [Frequency(i) for i in option_value.split('-')] + return values + + class DiagnosticVariableOption(DiagnosticOption): def parse(self, option_value): option_value = self.check_default(option_value) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 909854b407267b07922fc868b7d32e4979d57f5b..af81f2590a1a72e9b3913136538d38147bd384a3 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -245,7 +245,9 @@ class EarthDiags(object): @staticmethod def _register_general_diagnostics(): + Diagnostic.register(DailyMean) Diagnostic.register(MonthlyMean) + Diagnostic.register(YearlyMean) Diagnostic.register(Rewrite) Diagnostic.register(Relink) Diagnostic.register(RelinkAll) @@ -287,7 +289,9 @@ class EarthDiags(object): for startdate in self.config.experiment.startdates: for member in self.config.experiment.members: results = self._get_variable_report(startdate, member) - report_path = os.path.join(self.config.scratch_dir, '{0}_fc{1}.report'.format(startdate, member)) + report_path = os.path.join(self.config.scratch_dir, + '{0}_{1}.report'.format(startdate, + self.config.experiment.get_member_str(member))) self.create_report(report_path, results) Log.result('Report finished') @@ -380,24 +384,33 @@ class EarthDiags(object): mask_regions_3d = 'mask.regions.3d.{0}.nc'.format(model_version) if self.config.scratch_masks: Utils.create_folder_tree(self.config.scratch_masks) + Utils.give_group_write_permissions(self.config.scratch_masks) - if self._copy_file(os.path.join(con_files, mesh_mask), os.path.join(self.config.scratch_masks, mesh_mask), + mesh_mask_path = os.path.join(self.config.scratch_masks, mesh_mask) + if self._copy_file(os.path.join(con_files, mesh_mask), mesh_mask_path, restore_meshes): - self._link_file(os.path.join(self.config.scratch_masks, mesh_mask), 'mesh_hgr.nc') - self._link_file(os.path.join(self.config.scratch_masks, mesh_mask), 'mesh_zgr.nc') - self._link_file(os.path.join(self.config.scratch_masks, mesh_mask), 'mask.nc') + Utils.give_group_write_permissions(mesh_mask_path) + self._link_file(mesh_mask_path, 'mesh_hgr.nc') + self._link_file(mesh_mask_path, 'mesh_zgr.nc') + self._link_file(mesh_mask_path, 'mask.nc') + new_maskglo_scratch_path = os.path.join(self.config.scratch_masks, new_mask_glo) if self._copy_file(os.path.join(con_files, new_mask_glo), - os.path.join(self.config.scratch_masks, new_mask_glo), restore_meshes): - self._link_file(os.path.join(self.config.scratch_masks, new_mask_glo), 'new_maskglo.nc') + new_maskglo_scratch_path, restore_meshes): + Utils.give_group_write_permissions(new_maskglo_scratch_path) + self._link_file(new_maskglo_scratch_path, 'new_maskglo.nc') + mask_regions_scratch_path = os.path.join(self.config.scratch_masks, mask_regions) if self._copy_file(os.path.join(con_files, mask_regions), - os.path.join(self.config.scratch_masks, mask_regions), restore_meshes): - self._link_file(os.path.join(self.config.scratch_masks, mask_regions), 'mask_regions.nc') + mask_regions_scratch_path, restore_meshes): + Utils.give_group_write_permissions(mask_regions_scratch_path) + self._link_file(mask_regions_scratch_path, 'mask_regions.nc') + mask_regions3d_scratch_path = os.path.join(self.config.scratch_masks, mask_regions_3d) if self._copy_file(os.path.join(con_files, mask_regions_3d), - os.path.join(self.config.scratch_masks, mask_regions_3d), restore_meshes): - self._link_file(os.path.join(self.config.scratch_masks, mask_regions_3d), 'mask_regions.3d.nc') + mask_regions3d_scratch_path, restore_meshes): + Utils.give_group_write_permissions(mask_regions3d_scratch_path) + self._link_file(mask_regions3d_scratch_path, 'mask_regions.3d.nc') else: self._copy_file(os.path.join(con_files, mesh_mask), 'mesh_hgr.nc', restore_meshes) self._link_file('mesh_hgr.nc', 'mesh_zgr.nc') diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py index ff8b7e21e0e8bfbd7b24c54f19577eb9d33a46b3..35b19c2bfcb22c11a52de6accbcd8f645777b94a 100644 --- a/earthdiagnostics/frequency.py +++ b/earthdiagnostics/frequency.py @@ -8,8 +8,10 @@ class Frequency(object): 'c': 'clim', 'clim': 'clim', 'climatology': 'clim', 'monclim': 'clim', '1hrclimmon': 'clim', 'dec': 'dec', 'decadal': 'dec', 'y': 'year', 'yr': 'year', 'year': 'year', 'yearly': 'year', - 'm': 'mon', '1m': 'mon', 'mon': 'mon', 'monthly': 'mon', + 'm': 'mon', '1m': 'mon', 'mon': 'mon', 'monthly': 'mon', 'mm': 'mon', 'd': 'day', '1d': 'day', 'daily': 'day', 'day': 'day', + '15': '15hr', '15h': '15hr', '15hr': '15hr', '15_hourly': '15hr', '15hourly': '15hr', + '15 hourly': '15hr', '6': '6hr', '6h': '6hr', '6hr': '6hr', '6_hourly': '6hr', '6hourly': '6hr', '6 hourly': '6hr', '3': '3hr', '3h': '3hr', '3hr': '3hr', '3_hourly': '3hr', '3hourly': '3hr', '3 hourly': '3hr', '1': '1hr', 'hr': '1hr', '1h': '1hr', 'hourly': '1hr', '1hr': '1hr', '1 hourly': '1hr', diff --git a/earthdiagnostics/general/__init__.py b/earthdiagnostics/general/__init__.py index 1c321cb46d27751ec57e05fd65a68a0128890e2e..2696ef623ab57bca72ccd6ebbf86243973988d1d 100644 --- a/earthdiagnostics/general/__init__.py +++ b/earthdiagnostics/general/__init__.py @@ -1,5 +1,7 @@ # coding=utf-8 from earthdiagnostics.general.monthlymean import MonthlyMean +from earthdiagnostics.general.dailymean import DailyMean +from earthdiagnostics.general.yearlymean import YearlyMean from earthdiagnostics.general.rewrite import Rewrite from earthdiagnostics.general.relink import Relink from earthdiagnostics.general.scale import Scale diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py new file mode 100644 index 0000000000000000000000000000000000000000..983609e146912820ce07b1178a2ff8c05e3d6e8b --- /dev/null +++ b/earthdiagnostics/general/dailymean.py @@ -0,0 +1,105 @@ +# coding=utf-8 + +import os +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticFrequencyOption, DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealm + + +class DailyMean(Diagnostic): + """ + Calculates daily mean for a given variable + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + alias = 'daymean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.frequency = frequency + self.grid = grid + + def __str__(self): + return 'Calculate daily mean Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Original frequency: {5} Grid: {6}'.format(self.startdate, self.member, self.chunk, + self.domain, self.variable, + self.frequency, self.grid) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.grid == other.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options_available = (DiagnosticVariableOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticFrequencyOption('frequency'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(DailyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + + def compute(self): + """ + Runs the diagnostic + """ + day_mean = TempFile.get() + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + handler = Utils.openCdf(variable_file) + if 'region' in handler.variables: + noregion = TempFile.get() + Utils.nco.ncks(input=variable_file, output=noregion, options='-O -C -x -v region') + Utils.cdo.daymean(input=noregion, output=day_mean) + monmean_handler = Utils.openCdf(day_mean) + Utils.copy_variable(handler, monmean_handler, 'region') + monmean_handler.close() + else: + Utils.cdo.daymean(input=variable_file, output=day_mean) + os.remove(variable_file) + + self.send_file(day_mean, self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.daily, grid=self.grid) + diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 9b084dd23f5839f02ecacb81e71a3df2f29388dd..3052a54a0191b56e9f586130e2da061af7eac3ab 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -83,12 +83,23 @@ class MonthlyMean(Diagnostic): """ Runs the diagnostic """ - temp = TempFile.get() + monmean = TempFile.get() variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=self.frequency, grid=self.grid) - Utils.cdo.monmean(input=variable_file, output=temp, options='-O') + handler = Utils.openCdf(variable_file) + if 'region' in handler.variables: + noregion = TempFile.get() + Utils.nco.ncks(input=variable_file, output=noregion, options='-O -C -x -v region') + Utils.cdo.monmean(input=noregion, output=monmean) + monmean_handler = Utils.openCdf(monmean) + Utils.copy_variable(handler, monmean_handler, 'region') + monmean_handler.close() + else: + Utils.cdo.monmean(input=variable_file, output=monmean) + handler.close() + os.remove(variable_file) - self.send_file(temp, self.domain, self.variable, self.startdate, self.member, self.chunk, + self.send_file(monmean, self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=Frequencies.monthly, grid=self.grid) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 1605df2ef6bd810ef3608363bf9a373b75a36a99..59fc369d3ab3fcef62e13836629d632bbfc39742 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,6 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticFloatOption, DiagnosticDomainOption, \ - DiagnosticVariableOption +from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils from earthdiagnostics.modelingrealm import ModelingRealm import numpy as np @@ -35,7 +34,7 @@ class Scale(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, value, offset, domain, variable, grid, - min_limit, max_limit): + min_limit, max_limit, frequency): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -47,18 +46,19 @@ class Scale(Diagnostic): self.offset = offset self.min_limit = min_limit self.max_limit = max_limit + self.frequency = frequency self.original_values = None def __str__(self): return 'Scale output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Scale value: {5} Offset: {6} Variable: {3}:{4}'.format(self.startdate, self.member, self.chunk, - self.domain, self.variable, - self.value, self.offset) + 'Scale value: {5} Offset: {6} Variable: {3}:{4} ' \ + 'Frequency: {7}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.value, self.offset, self.frequency) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency @classmethod def generate_jobs(cls, diags, options): @@ -77,13 +77,15 @@ class Scale(Diagnostic): DiagnosticFloatOption('offset'), DiagnosticOption('grid', ''), DiagnosticFloatOption('min_limit', float('nan')), - DiagnosticFloatOption('max_limit', float('nan'))) + DiagnosticFloatOption('max_limit', float('nan')), + DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency])) options = cls.process_options(options, options_available) job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Scale(diags.data_manager, startdate, member, chunk, - options['value'], options['offset'], options['domain'], options['variable'], - options['grid'], options['min_limit'], options['max_limit'])) + for frequency in options['frequencies']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Scale(diags.data_manager, startdate, member, chunk, + options['value'], options['offset'], options['domain'], options['variable'], + options['grid'], options['min_limit'], options['max_limit'], frequency)) return job_list def compute(self): @@ -91,7 +93,7 @@ class Scale(Diagnostic): Runs the diagnostic """ variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + grid=self.grid, frequency=self.frequency) handler = Utils.openCdf(variable_file) var_handler = handler.variables[self.variable] @@ -100,12 +102,12 @@ class Scale(Diagnostic): var_handler[:] = self.original_values * self.value + self.offset handler.close() self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + grid=self.grid, frequency=self.frequency) def _check_limits(self): - if not math.isnan(self.min_limit) and (self.original_values < self.min_limit).any(): + if not math.isnan(self.min_limit) and (self.original_values.min() < self.min_limit): return False - if not math.isnan(self.max_limit) and (self.original_values > self.max_limit).any(): + if not math.isnan(self.max_limit) and (self.original_values.max() > self.max_limit): return False return True diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py new file mode 100644 index 0000000000000000000000000000000000000000..99ee87d7c4323de39514960083cf97ea70441a81 --- /dev/null +++ b/earthdiagnostics/general/yearlymean.py @@ -0,0 +1,105 @@ +# coding=utf-8 + +import os +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ + DiagnosticFrequencyOption, DiagnosticVariableOption +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.modelingrealm import ModelingRealm + + +class YearlyMean(Diagnostic): + """ + Calculates yearly mean for a given variable + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + :param frequency: original frequency + :type frequency: str + :param grid: original data grid + :type grid: str + """ + + alias = 'yearmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.frequency = frequency + self.grid = grid + + def __str__(self): + return 'Calculate yearly mean Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Original frequency: {5} Grid: {6}'.format(self.startdate, self.member, self.chunk, + self.domain, self.variable, + self.frequency, self.grid) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.grid == other.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, frequency=day, grid='' + :type options: list[str] + :return: + """ + + options_available = (DiagnosticVariableOption('variable'), + DiagnosticDomainOption('domain'), + DiagnosticFrequencyOption('frequency'), + DiagnosticOption('grid', '')) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(YearlyMean(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + + def compute(self): + """ + Runs the diagnostic + """ + year_mean = TempFile.get() + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + handler = Utils.openCdf(variable_file) + if 'region' in handler.variables: + noregion = TempFile.get() + Utils.nco.ncks(input=variable_file, output=noregion, options='-O -C -x -v region') + Utils.cdo.yearmean(input=noregion, output=year_mean) + monmean_handler = Utils.openCdf(year_mean) + Utils.copy_variable(handler, monmean_handler, 'region') + monmean_handler.close() + else: + Utils.cdo.yearmean(input=variable_file, output=year_mean) + os.remove(variable_file) + + self.send_file(year_mean, self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.yearly, grid=self.grid) + diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 123447c4a9d03b4106959f09af614ea73033c501..35bbcb4c450535bee90ea65f3dc8589de75acf78 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -44,7 +44,7 @@ class Interpolate(Diagnostic): lock = threading.Lock() def __init__(self, data_manager, startdate, member, chunk, domain, variable, target_grid, model_version, - invert_lat): + invert_lat, original_grid): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -57,18 +57,20 @@ class Interpolate(Diagnostic): self.tempTemplate = '' self.grid = target_grid self.invert_latitude = invert_lat + self.original_grid = original_grid def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version and self.domain == other.domain and \ self.variable == other.variable and self.grid == other.grid and \ - self.invert_latitude == other.invert_latitude + self.invert_latitude == other.invert_latitude and self.original_grid == other.original_grid def __str__(self): return 'Interpolate Startdate: {0} Member: {1} Chunk: {2} ' \ 'Variable: {3}:{4} Target grid: {5} Invert lat: {6} ' \ - 'Model: {7}' .format(self.startdate, self.member, self.chunk, self.domain, self.variable, self.grid, - self.invert_latitude, self.model_version) + 'Model: {7} Original grid: {8}'.format(self.startdate, self.member, self.chunk, self.domain, + self.variable, self.grid, self.invert_latitude, + self.model_version, self.original_grid) @classmethod def generate_jobs(cls, diags, options): @@ -84,7 +86,8 @@ class Interpolate(Diagnostic): options_available = (DiagnosticOption('target_grid'), DiagnosticVariableOption('variable'), DiagnosticDomainOption('domain', ModelingRealms.ocean), - DiagnosticBoolOption('invert_lat', False)) + DiagnosticBoolOption('invert_lat', False), + DiagnosticOption('original_grid')) options = cls.process_options(options, options_available) job_list = list() @@ -92,14 +95,15 @@ class Interpolate(Diagnostic): job_list.append( Interpolate(diags.data_manager, startdate, member, chunk, options['domain'], options['variable'], options['target_grid'], - diags.config.experiment.model_version, options['invert_lat'])) + diags.config.experiment.model_version, options['invert_lat'], options['original_grid'])) return job_list def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.original_grid) Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False, rename_dimension=True) cdo = Utils.cdo nco = Utils.nco diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 404e4aa520e852808c22ee850cc597a809ea960c..bff647adaed13df8f48f917a8b4772780864418f 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -37,7 +37,7 @@ class InterpolateCDO(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, target_grid, model_version, - mask_oceans): + mask_oceans, original_grid): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -50,11 +50,12 @@ class InterpolateCDO(Diagnostic): self.tempTemplate = '' self.grid = target_grid self.mask_oceans = mask_oceans + self.original_grid = original_grid def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version and self.domain == other.domain and \ - self.variable == other.variable and self.grid == other.grid + self.variable == other.variable and self.grid == other.grid and self.original_grid == other.original_grid def __str__(self): return 'Interpolate with CDO Startdate: {0} Member: {1} Chunk: {2} ' \ @@ -76,14 +77,16 @@ class InterpolateCDO(Diagnostic): options_available = (DiagnosticVariableOption('variable'), DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), DiagnosticDomainOption('domain', ModelingRealms.ocean), - DiagnosticBoolOption('mask_oceans', True)) + DiagnosticBoolOption('mask_oceans', True), + DiagnosticOption('original_grid')) options = cls.process_options(options, options_available) target_grid = cls._translate_ifs_grids_to_cdo_names(options['target_grid']) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(InterpolateCDO(diags.data_manager, startdate, member, chunk, options['domain'], options['variable'], target_grid, - diags.config.experiment.model_version, options['mask_oceans'])) + diags.config.experiment.model_version, options['mask_oceans'], + options['original_grid'])) return job_list @classmethod @@ -100,7 +103,8 @@ class InterpolateCDO(Diagnostic): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.original_grid) if self.mask_oceans: handler = Utils.openCdf(variable_file) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 99734b8b5e7cac5872990a12c561fea7a1bd3f7f..0a0d0b98c82b3e8f7b7b66e3532072ac1d9bb666 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -1,4 +1,6 @@ # coding=utf-8 +import os + from earthdiagnostics import cdftools from earthdiagnostics.box import Box from earthdiagnostics.constants import Basins @@ -10,12 +12,13 @@ from earthdiagnostics.modelingrealm import ModelingRealms class RegionMean(Diagnostic): """ - Chooses vertical level in ocean, or vertically averages between - 2 or more ocean levels + Computes the mean value of the field (3D, weighted). For 3D fields, + a horizontal mean for each level is also given. If a spatial window + is specified, the mean value is computed only in this window. :original author: Javier Vegas-Regidor - :created: January 2017 + :created: March 2017 :param data_manager: data management object :type data_manager: DataManager @@ -34,17 +37,20 @@ class RegionMean(Diagnostic): alias = 'regmean' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, box, save3d, basin): + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid_point, box, save3d, basin, + variance, grid): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member self.chunk = chunk self.domain = domain self.variable = variable - self.grid = grid.upper() + self.grid_point = grid_point.upper() self.box = box self.save3d = save3d self.basin = basin + self.variance = variance + self.grid = grid self.required_vars = [variable] self.generated_vars = [variable + 'vmean'] @@ -69,11 +75,13 @@ class RegionMean(Diagnostic): """ options_available = (DiagnosticDomainOption('domain'), DiagnosticVariableOption('variable'), - DiagnosticOption('grid'), + DiagnosticOption('grid_point', ''), DiagnosticBasinOption('basin', Basins.Global), - DiagnosticBoolOption('save3D', False), DiagnosticIntOption('min_depth', 0), - DiagnosticIntOption('max_depth', 0)) + DiagnosticIntOption('max_depth', 0), + DiagnosticBoolOption('save3D', True), + DiagnosticBoolOption('variance', False), + DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) box = Box() @@ -83,36 +91,63 @@ class RegionMean(Diagnostic): job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(RegionMean(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['grid'], box, options['save3D'], - options['basin'])) + options['domain'], options['variable'], options['grid_point'], box, + options['save3D'], options['basin'], options['variance'], options['grid'])) return job_list def compute(self): """ Runs the diagnostic """ - temp = TempFile.get() - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) + mean_file = TempFile.get() + + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) - cdfmean_options = [self.variable, self.grid, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] + handler = Utils.openCdf(variable_file) + self.save3d &= 'lev' in handler.dimensions + handler.close() + + cdfmean_options = [self.variable, self.grid_point, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] + if self.variance: + cdfmean_options += ['-var'] if self.basin != Basins.Global: cdfmean_options.append('-M') cdfmean_options.append('mask_regions.3d.nc') - cdfmean_options.append(self.basin.shortname) + cdfmean_options.append(self.basin.fullname) - cdftools.run('cdfmean', input=variable_file, output=temp, options=cdfmean_options) - Utils.setminmax(temp, 'mean_{0}'.format(self.variable)) + cdftools.run('cdfmean', input=variable_file, output=mean_file, options=cdfmean_options) + Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) if self.box.min_depth == 0: - # For cdftools, this is all levels + # To cdftools, this means all levels box_save = None else: box_save = self.box - self.send_file(temp, ModelingRealms.ocean, self.variable + 'mean', self.startdate, self.member, self.chunk, - box=box_save, rename_var='mean_{0}'.format(self.variable), region=self.basin) - if self.save3d: - Utils.setminmax(temp, 'mean_3D{0}'.format(self.variable)) - self.send_file(temp, ModelingRealms.ocean, self.variable + '3dmean', self.startdate, self.member, - self.chunk, box=box_save, rename_var='mean_3D{0}'.format(self.variable), region=self.basin) + self.send_var('mean', False, box_save, mean_file) + self.send_var('mean', True, box_save, mean_file) + + if self.variance: + self.send_var('var', False, box_save, mean_file) + self.send_var('var', True, box_save, mean_file) + + os.remove(mean_file) + + def send_var(self, var, threed, box_save, mean_file): + if threed: + if not self.save3d: + return False + original_name = '{0}_{1}'.format(var, self.variable) + final_name = '{1}3d{0}'.format(var, self.variable) + levels = ',lev' + else: + original_name = '{0}_3D{1}'.format(var, self.variable) + final_name = '{1}{0}'.format(var, self.variable) + levels = '' + + temp2 = TempFile.get() + Utils.nco.ncks(input=mean_file, output=temp2, options='-O -v {0},lat,lon{1}'.format(original_name, levels)) + self.send_file(temp2, ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, + box=box_save, rename_var=original_name, region=self.basin, grid=self.grid) diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 70525d60c686db2c5676cc52b55329760df16c8f..3effb07c85a78f68a5cfd31abb2ef2d2f459e74e 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -88,28 +88,47 @@ class Rotation(Diagnostic): urotated = TempFile.get() vrotated = TempFile.get() - namelist_file = TempFile.get(suffix='') - rotate_namelist = open(namelist_file, 'w') - rotate_namelist.write("&nam_rotUV\n") - rotate_namelist.write(" Ufilein = {0}\n".format(ufile)) - rotate_namelist.write(" Uvarin = {0}\n".format(self.variableu)) - rotate_namelist.write(" Vfilein = {0}\n".format(vfile)) - rotate_namelist.write(" Vvarin = {0}\n".format(self.variablev)) - rotate_namelist.write(" meshmask = mask.nc\n") - rotate_namelist.write(" Ufileout = {0}\n".format(urotated)) - rotate_namelist.write(" Vfileout = {0}\n".format(vrotated)) - - rotate_namelist.writelines("/\n") - rotate_namelist.close() + namelist_file = self._create_namelist(ufile, urotated, vfile, vrotated) Utils.execute_shell_command('{0} {1}'.format(self.executable, namelist_file), Log.INFO) + ufile_handler = Utils.openCdf(ufile) + self._add_metadata_and_vars(ufile_handler, urotated, self.variableu) + self._add_metadata_and_vars(ufile_handler, vrotated, self.variablev) + ufile_handler.close() + os.remove(ufile) os.remove(vfile) self.send_file(urotated, self.domain, self.variableu, self.startdate, self.member, self.chunk, grid='rotated') self.send_file(vrotated, self.domain, self.variablev, self.startdate, self.member, self.chunk, grid='rotated') + def _create_namelist(self, ufile, urotated, vfile, vrotated): + namelist_file = TempFile.get(suffix='') + rotate_namelist = open(namelist_file, 'w') + rotate_namelist.write('&nam_rotUV\n') + rotate_namelist.write(' Ufilein = "{0}"\n'.format(ufile)) + rotate_namelist.write(' Uvarin = "{0}"\n'.format(self.variableu)) + rotate_namelist.write(' Vfilein = "{0}"\n'.format(vfile)) + rotate_namelist.write(' Vvarin = "{0}"\n'.format(self.variablev)) + rotate_namelist.write(' meshmask = "mask.nc"\n') + rotate_namelist.write(' Ufileout = "{0}"\n'.format(urotated)) + rotate_namelist.write('Vfileout = "{0}"\n'.format(vrotated)) + rotate_namelist.writelines("/\n") + rotate_namelist.close() + return namelist_file + + def _add_metadata_and_vars(self, reference_file_handler, rotaded_file, var_name): + rotated_handler = Utils.openCdf(rotaded_file) + self._copy_extra_variables(reference_file_handler, rotated_handler) + Utils.copy_attributes(rotated_handler.variables[var_name], reference_file_handler.variables[var_name]) + rotated_handler.close() + + def _copy_extra_variables(self, reference_file_handler, rotated_handler): + for var in reference_file_handler.variables.keys(): + if var not in rotated_handler.variables.keys() and var not in [self.variableu, self.variablev]: + Utils.copy_variable(reference_file_handler, rotated_handler, var, True, True) + diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 604216b4c7372bc1ce4b3df660fa3e8b53a38a51..f38b7c2007404c3e36c8f3bcd72fb13cda623b38 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -98,6 +98,7 @@ class Siasiesiv(Diagnostic): sic_file = self.data_manager.get_file(ModelingRealms.seaIce, 'sic', self.startdate, self.member, self.chunk) sic_handler = Utils.openCdf(sic_file) + Utils.convert_units(sic_handler.variables['sic'], '1.0') sic = np.asfortranarray(sic_handler.variables['sic'][:]) sic_handler.close() diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 67697b06007c8a45b328a9206cfdc70403debd18..40dd535324e8053a4e3b653362a0f056b756238e 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -7,13 +7,28 @@ import tarfile import netCDF4 import numpy as np import os +import stat import re import tempfile from bscearth.utils.log import Log from cdo import Cdo, CDOException +from cfunits import Units from nco import Nco from earthdiagnostics.constants import Basins +from contextlib import contextmanager +import sys + + +@contextmanager +def suppress_stdout(): + with open(os.devnull, "w") as devnull: + old_stdout = sys.stdout + sys.stdout = devnull + try: + yield + finally: + sys.stdout = old_stdout class Utils(object): @@ -131,19 +146,20 @@ class Utils(object): error = True if error: - Log.info('First attemp to rename failed. Using secondary rename method for netCDF') + Log.debug('First attemp to rename failed. Using secondary rename method for netCDF') Utils._rename_vars_by_creating_new_file(dic_names, filepath, temp) - Log.info('Rename done') + Log.debug('Rename done') Utils.move_file(temp, filepath) @staticmethod def check_netcdf_file(filepath): - try: - Utils.cdo.showvar(input=filepath) - except CDOException: - return False - return True + with suppress_stdout(): + try: + Utils.cdo.showvar(input=filepath) + except CDOException: + return False + return True @staticmethod def get_file_variables(filename): @@ -455,9 +471,13 @@ class Utils(object): return original_var = source.variables[variable] new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) + Utils.copy_attributes(new_var, original_var) + new_var[:] = original_var[:] + + @staticmethod + def copy_attributes(new_var, original_var): new_var.setncatts({k: Utils.convert_to_ASCII_if_possible(original_var.getncattr(k)) for k in original_var.ncattrs()}) - new_var[:] = original_var[:] @staticmethod def copy_dimension(source, destiny, dimension, must_exist=True, new_names=None): @@ -563,6 +583,26 @@ class Utils(object): if not os.path.isdir(path): raise + @staticmethod + def give_group_write_permissions(path): + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IWGRP) + + @staticmethod + def convert_units(var_handler, new_units): + if new_units == var_handler.units: + return + new_unit = Units(new_units) + old_unit = Units(var_handler.units) + var_handler[:] = Units.conform(var_handler[:], old_unit, new_unit, inplace=True) + if 'valid_min' in var_handler.ncattrs(): + var_handler.valid_min = Units.conform(float(var_handler.valid_min), old_unit, new_unit, + inplace=True) + if 'valid_max' in var_handler.ncattrs(): + var_handler.valid_max = Units.conform(float(var_handler.valid_max), old_unit, new_unit, + inplace=True) + var_handler.units = new_units + @staticmethod def untar(files, destiny_path): """ diff --git a/earthdiagnostics/variable_alias/default.csv b/earthdiagnostics/variable_alias/default.csv index 993ca7b55e0a042ffc144871681902e50f22fb72..f78af47e0ccbb8fe8b9bfd0e4a80507211af9695 100644 --- a/earthdiagnostics/variable_alias/default.csv +++ b/earthdiagnostics/variable_alias/default.csv @@ -42,6 +42,9 @@ sophteiv,hfbasinba,, qt_oce:sohefldo:qt,hfds,, slhf,hfls,, sshf,hfss,, +zqla,hflso,, +zqsb,hfsso,, +zqlw,rlntds,, sophtove,htovovrt,, q,hus,, soicealb,ialb,, @@ -291,4 +294,7 @@ wo,wo,, w2o,wosq,, difvho,difvho,, vovematr,wmo,, -qtr_ice,qtr,, \ No newline at end of file +qtr_ice,qtr,, +var78,tclw,, +var79,tciw,, +clwc,clw,, \ No newline at end of file diff --git a/model_diags.conf b/model_diags.conf index 72bf0d20ad2e367d6c472c85447578f4d75d4b0b..0ac9a54e9f8fe1b83ed0a85a7a24ca59f22a0953 100644 --- a/model_diags.conf +++ b/model_diags.conf @@ -1,10 +1,16 @@ [DIAGNOSTICS] -# The next few configurations are mandatory - # Temporary folder for the calculations. Final results will never be stored here. SCRATCH_DIR = /scratch/Earth/$USER +# Common scratch folder for the ocean masks. This is useful to avoid replicating them for each run at the fat nodes. +# By default is '/scratch/Earth/ocean_masks' +# SCRATCH_MASKS = + +# By default, Earth Diagnostics only copies the mask files if they are not present in the scratch folder. If this +# option is set to true, Earth Diagnostics will copy them regardless of existence. Default is False. +# RESTORE_MESHES = + # ':' separated list of folders to look for data in. It will look for file in the path $DATA_FOLDER/$EXPID and # $DATA_FOLDER/$DATA_TYPE/$MODEL/$EXPID DATA_DIR = /esnas:/esarchive @@ -16,57 +22,92 @@ CON_FILES = /esnas/autosubmit/con_files/ # ignore it completely. FREQUENCY = mon -# All the other configurations in this section are optional - # Type of the dataset to use. It can be exp, obs or recon. Default is exp. -DATA_TYPE = exp +# DATA_TYPE = exp # This is used to choose the mechanism for storing and retrieving data. Options are CMOR (for our own experiments) or # THREDDS (for anything else). Default value is CMOR -DATA_ADAPTOR = CMOR +# DATA_ADAPTOR = CMOR +# Convention to use for file paths and names and variable naming among other things. Can be SPECS, PRIMAVERA or CMIP6. +# Default is SPECS. +DATA_CONVENTION = CMIP6 +# Path to the folder containing CDFTOOLS executables. By default is empty, so CDFTOOLS binaries must be added to the +# system path. +# CDFTOOLS_PATH = -* DATA_CONVENTION - Convention to use for file paths and names and variable naming among other things. Can be SPECS, PRIMAVERA or CMIP6. - Default is SPECS. +# Maximum number of cores to use. By default the diagnostics will use all cores available to them. It is not +# necessary when launching through a scheduler, as Earthdiagnostics can detect how many cores the scheduler has +# allocated to it. +# MAX_CORES = 1 -* CDFTOOLS_PATH - Path to the folder containing CDFTOOLS executables. By default is empty, so CDFTOOLS binaries must be added to the - system path. +# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or +# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it +# empty +DIAGS = -* MAX_CORES - Maximum number of cores to use. By default the diagnostics will use all cores available to them. It is not - necessary when launching through a scheduler, as Earthdiagnostics can detect how many cores the scheduler has - allocated to it. +[EXPERIMENT] -# Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) +# Institute that made the experiment, observation or reconstruction +INSTITUTE = BSC +# Name of the model used for the experiment. +MODEL = EC-EARTH -# Root path for the cmorized data to use +# Model version. Used to get the correct mask and mesh files. +# Available versions: +# Ec2.3_O1L42 +# Ec3.0_O1L46 Ec3.0_O25L46 Ec3.0_O25L75 +# Ec3.1_O1L46 Ec3.1_O25L75 +# Ec3.2beta_O1L75 Ec3.2_O1L75 Ec3.2_O25L75 +# N3.2_O1L42 N3.3_O1L46 N3.6_O025L75 N3.6_O1L75 N3.6_O25L75 +# nemovar_O1L42 CNRM_O1L42.nc glorys2v1_O25L75 ucl_O2L31 +# ORCA12L75 +MODEL_VERSION =Ec3.2_O1L75 -# Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) +# Time between outputs from the atmosphere. This is not the model simulation timestep! Default is 6 +ATMOS_TIMESTEP = 6 -# CMORization type to use. Important also for THREDDS as it affects variable name conventions. -# Options: SPECS (default), PRIMAVERA, CMIP6 +# Time between outputs from the ocean. This is not the model simulation timestep! Default is 6 +OCEAN_TIMESTEP = 6 -# Path to NEMO's mask and grid files needed for CDFTools +# Unique identifier for the experiment +EXPID = -# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or -# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it -# empty -DIAGS = -# DIAGS = OHC -# Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores -# its results at monthly frequency (obvious) and has a parameter to specify input's frequency. +# Startdates to run as a space separated list +STARTDATES = + +# Members to run as a space separated list. You can just provide the number or also add the prefix +MEMBERS = + +# Number of minimum digits to compose the member name. By default it is 1. For example, for member 1 member name +# will be fc1 if MEMBER_DIGITS is 1 or fc01 if MEMBER_DIGITS is 2 +MEMBER_DIGITS = + +# Prefix to use for the member names. By default is 'fc' +MEMBER_PREFIX = + +# Number corresponding to the first member. For example, if your first member is 'fc1', it should be 1. +# If it is 'fc0', it should be 0. By default is 0 +MEMBER_COUNT_START = + +# Length of the chunks in months +CHUNK_SIZE = + +# Number of chunks to run +CHUNKS = + +# Atmospheric grid definition. Will be used as a default target for interpolation diagnostics. +# ATMOS_GRID = + +# Experiment's name. By default it is the EXPID. +# NAME = + +# Calendar to use for date calculation. All calendars supported by Autosubmit are available. Default is 'standard' +# CALENDAR = -# Path to CDFTOOLS binaries -CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin -# If true, copies the mesh files regardless of presence in scratch dir -RESTORE_MESHES = False -# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available) -MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False @@ -107,34 +148,6 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, [THREDDS] SERVER_URL = https://earth.bsc.es/thredds -[EXPERIMENT] -# Experiments parameters as defined in CMOR standard -INSTITUTE = BSC -MODEL = EC-EARTH -# Model version: Available versions -MODEL_VERSION =Ec3.2_O1L75 -# Atmospheric output timestep in hours -ATMOS_TIMESTEP = 6 -# Ocean output timestep in hours -OCEAN_TIMESTEP = 6 - -# For those who use Autosubmit, this will be easy -# EXPID is the unique identifier of the experiment. -# STARTDATES is the list of start dates -# MEMBERS is the list of members of your experiment (only the numbers, the fc will be added by the tool) -# MEMBER_DIGITS is the minimum number of digits to use for the member name: if 1 the name for member 0 will be fc0, -# if 2, fc00 -# CHUNK_SIZE is the size of each data file, given in months -# CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = a0c2 -STARTDATES = 19900101 -MEMBERS = 0 -MEMBER_DIGITS = 1 -CHUNK_SIZE = 12 -CHUNKS = 2 -# CHUNKS = 1 - - # This ALIAS section is a bit different # Inside this, you can provide alias for frequent diagnostics calls. # By default, there are some of the diagnostics available at the previous version. diff --git a/test/unit/test_maxmoc.py b/test/unit/test_maxmoc.py index 35117a673907aa68ea13978ae39111e75bdba608..f9542b2996ae9c8cf4d5670b1dcbc128827184a1 100644 --- a/test/unit/test_maxmoc.py +++ b/test/unit/test_maxmoc.py @@ -12,11 +12,11 @@ class TestMaxMoc(TestCase): def setUp(self): self.data_manager = Mock() - self.box = Box() - self.box.min_lat = 0 - self.box.max_lat = 0 - self.box.min_depth = 0 - self.box.max_depth = 0 + self.box = Box(True) + self.box.min_lat = 0.0 + self.box.max_lat = 0.0 + self.box.min_depth = 0.0 + self.box.max_depth = 0.0 self.maxmoc = MaxMoc(self.data_manager, '20000101', 1, 2000, Basins.Global, self.box) diff --git a/test/unit/test_verticalmeanmeters.py b/test/unit/test_verticalmeanmeters.py index ecfb3da3f5d9b88f2cd5c249b457d70b26563b43..39a86c3a66c21fc7de7d2d1232402e2ff8c9ca55 100644 --- a/test/unit/test_verticalmeanmeters.py +++ b/test/unit/test_verticalmeanmeters.py @@ -53,5 +53,5 @@ class TestVerticalMeanMeters(TestCase): VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' - 'Box: 0m-100m') + self.assertEquals(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Box: 0m-100m')