diff --git a/VERSION b/VERSION index 2638df168bc2aebcad08a8bc38589754868cc234..40020fcf5dc73df73f4e501cfa508d3f1744bf02 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc3 +3.0.0rc5 diff --git a/diags.conf b/diags.conf index 64dc8269cdee95d1bd35d4d52d236d018205fcb3..92216b57c2711907440747e024a553869cd22513 100644 --- a/diags.conf +++ b/diags.conf @@ -1,12 +1,12 @@ [DIAGNOSTICS] # Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) -DATA_ADAPTOR = OBSRECON +DATA_ADAPTOR = CMOR # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use DATA_DIR = /esarchive # Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) -DATA_TYPE = recon +DATA_TYPE = exp # CMORization type to use. Important also for THREDDS as it affects variable name conventions. # Options: SPECS (default), PRIMAVERA, CMIP6 DATA_CONVENTION = SPECS @@ -16,11 +16,11 @@ CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = interpcdo,atmos,prlr,r240x121,bilinear,False,,False +DIAGS = # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = weekly +FREQUENCY = mon # Path to CDFTOOLS binaries CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir @@ -30,15 +30,18 @@ MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False -FORCE = False +FORCE = True +USE_GRIB = False +FORCE_UNTAR = False +#CHUNKS = # If true, CMORizes ocean files. Default = True OCEAN_FILES = False FILTER_FILES = # If true, CMORizes atmosphere files. Default = True -ATMOSPHERE_FILES = False +ATMOSPHERE_FILES = True # You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var VARIABLE_LIST = -CHUNK_LIST = 1 +CHUNK_LIST = # Variables to be CMORized from the grib atmospheric files, separated by comma. # You can also specify the levels to extract using the following syntax @@ -64,7 +67,7 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, # PHYSICS_DESCRIPTION = # ASSOCIATED_MODEL = # SOURCE = EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 -VERSION = v20170705 +VERSION = ACTIVITY = CMIP #ACTIVITY = CMIP_ece2cmor @@ -73,15 +76,15 @@ SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = gloh2o -MODEL = mswep -NAME = wekkly_means +INSTITUTE = BSC +MODEL = EC-EARTH3 +#NAME = # Model version: Available versions MODEL_VERSION = # Atmospheric output timestep in hours -ATMOS_TIMESTEP = 3 +ATMOS_TIMESTEP = 6 # Ocean output timestep in hours -OCEAN_TIMESTEP = 3 +OCEAN_TIMESTEP = 6 # For those who use Autosubmit, this will be easy # EXPID is the unique identifier of the experiment. @@ -91,12 +94,13 @@ OCEAN_TIMESTEP = 3 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = mswep -# STARTDATES = {19970101,20161231,1D} -STARTDATES = 19970322 20010425 -MEMBERS = 0 +EXPID = m02r +#STARTDATES = {19970101,20161231,1D} +STARTDATES = {19930501,20090501,1Y} +#STARTDATES = 19970322 20010425 +MEMBERS = 0-9 MEMBER_DIGITS = 1 -CHUNK_SIZE = 1 +CHUNK_SIZE = 4 CHUNKS = 1 # CHUNKS = 1 diff --git a/earthdiagnostics/cmor_tables/cmip6 b/earthdiagnostics/cmor_tables/cmip6 index 78eb04bd32dcc398323b21b1cb0636b2f07ffc68..a6375fe65de3cbbf936303c61a819fd96b8df2ab 160000 --- a/earthdiagnostics/cmor_tables/cmip6 +++ b/earthdiagnostics/cmor_tables/cmip6 @@ -1 +1 @@ -Subproject commit 78eb04bd32dcc398323b21b1cb0636b2f07ffc68 +Subproject commit a6375fe65de3cbbf936303c61a819fd96b8df2ab diff --git a/earthdiagnostics/cmor_tables/primavera b/earthdiagnostics/cmor_tables/primavera index f25073770569ea73540d09a058637128db024c55..5236593cf11fd3175dd1e48d201cd507bd2e37ac 160000 --- a/earthdiagnostics/cmor_tables/primavera +++ b/earthdiagnostics/cmor_tables/primavera @@ -1 +1 @@ -Subproject commit f25073770569ea73540d09a058637128db024c55 +Subproject commit 5236593cf11fd3175dd1e48d201cd507bd2e37ac diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 42099f32ab84ffa4db1c9c4ceae56e473e766dea..0b1c54053951a8b2a63f525ba8315e2f1f76ed13 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -176,22 +176,21 @@ class Cmorizer(object): def _merge_mma_files(self, tarfile): temp = TempFile.get() - sh_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_SH_*.nc')) - gg_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_GG_*.nc')) - - merged_sh = TempFile.get() - merged_gg = TempFile.get() - - for filename in sh_files: - Utils.cdo.sp2gpl(options='-O', input=filename, output=temp) - shutil.move(temp, filename) - Utils.cdo.mergetime(input=sh_files, output=merged_sh) - Utils.cdo.mergetime(input=gg_files, output=merged_gg) - for filename in sh_files + gg_files: - os.remove(filename) - tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') - shutil.move(merged_gg, os.path.join(self.cmor_scratch, 'MMAGG_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) - shutil.move(merged_sh, os.path.join(self.cmor_scratch, 'MMASH_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) + for grid in ['SH', 'GG']: + files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_{}_*.nc'.format(grid))) + if not files: + continue + merged = TempFile.get() + if grid == 'SH': + for filename in files: + Utils.cdo.sp2gpl(options='-O', input=filename, output=temp) + shutil.move(temp, filename) + Utils.cdo.mergetime(input=files, output=merged) + for filename in files: + os.remove(filename) + tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') + filename = 'MMA{0}_1m_{1[0]}_{1[1]}.nc'.format(grid, tar_startdate) + shutil.move(merged, os.path.join(self.cmor_scratch, filename)) def cmorize_atmos(self): """Cmorize atmospheric data, from grib or MMA files""" @@ -371,7 +370,7 @@ class Cmorizer(object): # Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) - Utils.rename_variables(filename, self.alt_coord_names, False, True) + Utils.rename_variables(filename, self.alt_coord_names, False) handler = Utils.open_cdf(filename) Cmorizer._remove_valid_limits(handler) self._add_common_attributes(handler, frequency) @@ -527,12 +526,12 @@ class Cmorizer(object): def _rename_level_variables(temp, var_cmor): if var_cmor.domain == ModelingRealms.ocean: Utils.rename_variables(temp, {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', - 'depth': 'lev'}, False, True) + 'depth': 'lev'}, False) if var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: Utils.rename_variables(temp, {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', - 'depth_4': 'sdepth'}, False, True) + 'depth_4': 'sdepth'}, False) if var_cmor.domain == ModelingRealms.atmos: - Utils.rename_variables(temp, {'depth': 'plev'}, False, True) + Utils.rename_variables(temp, {'depth': 'plev'}, False) def _merge_grib_files(self, current_month, prev_gribfile, gribfile): Log.info('Merging data from different files...') diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 9d2fa84f22fd3f90888f4773449c7abe0958c411..7cb0628a8084dcb45a696af229908c5e9352f81c 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -484,8 +484,8 @@ class CMORManager(DataManager): """ Prepare the data to be used by the diagnostic. - If CMOR data is not created, it show a warning and closes. In the future, an automatic cmorization procedure - will be launched + If CMOR data is not created, an automatic cmorization procedure + is launched If CMOR data is available but packed, the procedure will unpack it. @@ -494,27 +494,32 @@ class CMORManager(DataManager): if self.config.data_convention == 'meteofrance': return + if self.config.cmor.skip_prepare: + return for startdate, member in self.experiment.get_member_list(): - Log.info('Checking data for startdate {0} member {1}', startdate, member) - if not self.config.cmor.force: - cmorized = False - for chunk in range(1, self.experiment.num_chunks + 1): - if not self.config.cmor.chunk_cmorization_requested(chunk): - Log.debug('Skipping chunk {0}', chunk) - continue - if not self.config.cmor.force_untar: - Log.debug('Checking chunk {0}...', chunk) - for domain in (ModelingRealms.atmos, ModelingRealms.ocean, ModelingRealms.seaIce): - if self.is_cmorized(startdate, member, chunk, domain): - Log.debug('Chunk {0} ready', chunk) - continue - if self._unpack_chunk(startdate, member, chunk): - cmorized = True - if cmorized: - Log.info('Startdate {0} member {1} ready', startdate, member) - return - self._cmorize_member(startdate, member) + self._prepare_member(startdate, member) + + def _prepare_member(self, startdate, member): + Log.info('Checking data for startdate {0} member {1}', startdate, member) + if not self.config.cmor.force: + cmorized = False + for chunk in range(1, self.experiment.num_chunks + 1): + if not self.config.cmor.chunk_cmorization_requested(chunk): + Log.debug('Skipping chunk {0}', chunk) + continue + if not self.config.cmor.force_untar: + Log.debug('Checking chunk {0}...', chunk) + for domain in (ModelingRealms.atmos, ModelingRealms.ocean, ModelingRealms.seaIce): + if self.is_cmorized(startdate, member, chunk, domain): + Log.debug('Chunk {0} ready', chunk) + return + if self._unpack_chunk(startdate, member, chunk): + cmorized = True + if cmorized: + Log.info('Startdate {0} member {1} ready', startdate, member) + return + self._cmorize_member(startdate, member) def is_cmorized(self, startdate, member, chunk, domain): """ @@ -597,7 +602,7 @@ class CMORManager(DataManager): filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar') if len(filepaths) > 0: Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) - Utils.untar(filepaths, self.cmor_path) + Utils.untar(filepaths, os.path.join(self.cmor_path, 'cmorfiles')) self._correct_paths(startdate) self.create_links(startdate, member) return True @@ -687,8 +692,10 @@ class CMORManager(DataManager): def _create_links_cmip5(self, member_str, path): for freq in os.listdir(path): + Log.debug('Creating links for frequency {0}', freq) frequency = Frequency.parse(freq) for domain in os.listdir(os.path.join(path, freq)): + Log.debug('Creating links for domain {0}', domain) for var in os.listdir(os.path.join(path, freq, domain)): for member in os.listdir(os.path.join(path, freq, domain, var)): if member_str is not None and member_str != member: @@ -708,6 +715,7 @@ class CMORManager(DataManager): for table in os.listdir(os.path.join(path, member)): frequency = self.variable_list.tables[table].frequency domain = None + Log.debug('Creating links for table {0}', table) for var in os.listdir(os.path.join(path, member, table)): for grid in os.listdir(os.path.join(path, member, table, var)): if member_str is not None and member_str != member: diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 734f94db7904708d40a44a51e492ce821b65cde5..2b2dea62b1ddc969203a81a7c47d2695a55f7fcc 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -217,6 +217,7 @@ class CMORConfig(object): def __init__(self, parser, var_manager): self.force = parser.get_bool_option('CMOR', 'FORCE', False) self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) + self.skip_prepare = parser.get_bool_option('CMOR', 'SKIP_PREPARE', False) self.filter_files = parser.get_option('CMOR', 'FILTER_FILES', '') self.ocean = parser.get_bool_option('CMOR', 'OCEAN_FILES', True) self.atmosphere = parser.get_bool_option('CMOR', 'ATMOSPHERE_FILES', True) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 086273e146a64142d8062dcd5575d73631bc0dbe..b94f7b617b40c2e73f953732d0764ac494a088be 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -123,6 +123,7 @@ class DataFile(Publisher): if self.has_modifiers(): return True + return False def add_modifier(self, diagnostic): """ @@ -308,6 +309,9 @@ class DataFile(Publisher): coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name, 'leadtime', 'region', 'time_centered'}, set(handler.variables.keys())) var_handler.coordinates = ' '.join(coords) + if 'time_centered' in handler.variables: + if hasattr(handler.variables['time_centered'], 'standard_name'): + del handler.variables['time_centered'].standard_name if not self.cmor_var: handler.close() return @@ -471,7 +475,7 @@ class DataFile(Publisher): variables['nav_lon_grid_U'] = self.lon_name variables['nav_lat_grid_T'] = self.lat_name variables['nav_lon_grid_T'] = self.lon_name - Utils.rename_variables(self.local_file, variables, False, True) + Utils.rename_variables(self.local_file, variables, False) def add_diagnostic_history(self): """Add the history line corresponding to the diagnostic to the local file""" @@ -627,7 +631,7 @@ class NetCDFFile(DataFile): alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} - Utils.rename_variables(self.local_file, alt_coord_names, must_exist=False, rename_dimension=True) + Utils.rename_variables(self.local_file, alt_coord_names, must_exist=False) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index ebf6e8a813b251a493fb54ec509d779fc68ec7ec..9bd4fb949d2e42af4ece538799621e7b9fc0f883 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -29,7 +29,7 @@ class DataManager(object): if filepath not in self.requested_files: self.requested_files[filepath] = NCfile.from_storage(filepath, self.config.data_convention) file_object = self.requested_files[filepath] - file_object.local_satatus = LocalStatus.PENDING + file_object.local_status = LocalStatus.PENDING return self.requested_files[filepath] def _declare_generated_file(self, remote_file, domain, final_var, cmor_var, data_convention, diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 099ef4f1781d81b43e343a57c1c1c6ad6c6722d3..8a4d43980d4b6d70196b8cd04a7273f12dc0bf7d 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -86,6 +86,8 @@ class Diagnostic(Publisher): bool """ + if not self._generated_files: + return False for file_generated in self._generated_files: if file_generated.storage_status != StorageStatus.READY: return False @@ -326,7 +328,7 @@ class Diagnostic(Publisher): chunk: int or None grid: str or None box: Box or None - frequency: Frequency or None + frequency: Frequency or str or None to_modify: bool Flag that must be active if the diagnostic is going to generate a modified version of this data. In this case this data must not be declared as an output of the diagnostic @@ -425,8 +427,9 @@ class Diagnostic(Publisher): int """ - return len([request.storage_status != StorageStatus.READY or request.local_status != LocalStatus.READY - for request in self._requests]) + return len([request for request in self._requests + if request.storage_status != StorageStatus.READY or + request.local_status != LocalStatus.READY]) def _different_type(self, other): return type(self) is not type(other) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 86a5e6bc2b24a4ef5381003421cf9db12ca5b641..e52035bceca454a7e63723509d1d798e28c05031 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -414,7 +414,7 @@ class EarthDiags(object): Log.info('Copying file {0}', destiny) shutil.copyfile(source, destiny) Log.info('File {0} ready', destiny) - Utils.rename_variables(destiny, self.dic_variables, False, True) + Utils.rename_variables(destiny, self.dic_variables, False) return True def _link_file(self, source, destiny): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index fcae8e514310fa9dc924f0fef5b1757a8c408a11..aecbfa953ec8377b18a678e73c65de2df4ddc765 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -46,6 +46,9 @@ class Relink(Diagnostic): return 'Relink output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Move old: {0.move_old} ' \ 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): if self._different_type(other): return False diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 1b59e1b24f70aa8984aba44d7c68c69cc991715f..10a6dd0d5bc88d58073c6770ee705a8f2a62f33e 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -27,6 +27,9 @@ class RelinkAll(Diagnostic): def __str__(self): return 'Relink all output Startdate: {0}'.format(self.startdate) + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): if self._different_type(other): return False diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 2addbe5e168c7345de90a535524b696b7e2876a9..74d3724c1327f3153432441cf479ac0e6df760ad 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -1,6 +1,11 @@ # coding=utf-8 """Time mean diagnostics""" -import os +import iris +import iris.coord_categorisation +import iris.analysis +import iris.exceptions + +import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticFrequencyOption, DiagnosticVariableOption @@ -40,11 +45,15 @@ class TimeMean(Diagnostic): self.frequency = frequency self.grid = grid self._target_frequency = None + self.mean_file = None def __str__(self): return 'Calculate {0._target_frequency} mean Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ 'Variable: {0.domain}:{0.variable} Original frequency: {0.frequency} Grid: {0.grid}'.format(self) + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): if self._different_type(other): return False @@ -84,38 +93,54 @@ class TimeMean(Diagnostic): self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=self.frequency, grid=self.grid) - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.daymean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.daily, grid=self.grid) - - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ raise NotImplementedError() def compute(self): """Run the diagnostic""" temp = TempFile.get() - handler = Utils.open_cdf(self.variable_file.local_file) - if 'region' in handler.variables: - noregion = TempFile.get() - Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) - self.compute_mean(noregion, temp) - os.remove(noregion) - monmean_handler = Utils.open_cdf(temp) - Utils.copy_variable(handler, monmean_handler, 'region') - monmean_handler.close() - else: - self.compute_mean(self.variable_file.local_file, temp) - self.daymean.set_local_file(temp) + cube = iris.load_cube(self.variable_file.local_file) + time_centered = [coord for coord in cube.coords() if coord.var_name == 'time_centered'] + if time_centered: + cube.remove_coord(time_centered[0]) + iris.coord_categorisation.add_day_of_month(cube, 'time') + iris.coord_categorisation.add_month_number(cube, 'time') + iris.coord_categorisation.add_year(cube, 'time') + + cube = self.compute_mean(cube) + cube.remove_coord('day_of_month') + cube.remove_coord('month_number') + cube.remove_coord('year') + try: + region_coord = cube.coord('region') + cube.remove_coord(region_coord) + except iris.exceptions.CoordinateNotFoundError: + region_coord = None + iris.FUTURE.netcdf_no_unlimited = True + iris.save(cube, temp) + if region_coord: + handler = Utils.open_cdf(temp) + region = handler.createVariable('region', str, ('dim0',)) + region.standard_name = region_coord.standard_name + region[...] = region_coord.points.astype(np.dtype(str)) + + handler.variables[self.variable].coordinates += ' region' + handler.close() + + Utils.rename_variable(temp, 'dim0', 'region', False) + + self.mean_file.set_local_file(temp) class DailyMean(TimeMean): @@ -146,17 +171,24 @@ class DailyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'daily' - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ - Utils.cdo.daymean(input=input_file, output=output_file) + return cube.aggregated_by(['day_of_month', 'month_number', 'year'], iris.analysis.MEAN) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.mean_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.daily, grid=self.grid) class MonthlyMean(TimeMean): @@ -187,17 +219,24 @@ class MonthlyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'monthly' - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ - Utils.cdo.monmean(input=input_file, output=output_file) + return cube.aggregated_by(['month_number', 'year'], iris.analysis.MEAN) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.mean_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, grid=self.grid) class YearlyMean(TimeMean): @@ -228,14 +267,21 @@ class YearlyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'yearly' - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ - Utils.cdo.monmean(input=input_file, output=output_file) + return cube.aggregated_by(['year'], iris.analysis.MEAN) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.mean_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.yearly, grid=self.grid) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 135f9f4b1bb2af78a46a0ba5e786dc76f7f9d3c7..f2e0246570aae29b4998fd16ac6fec2bdcbfaffb 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -169,7 +169,7 @@ class HeatContentLayer(Diagnostic): handler.close() nco.ncks(input=thetao_file, output=results, options=('-O -v lon,lat,time',)) - Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False, True) + Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False) handler_results = Utils.open_cdf(results) handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) handler_results.sync() diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 76e9d78a142b29de9a88c1110278e4f630730fbf..1ec9a4c847948c40c94a4be17962416876b673ac 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -120,7 +120,7 @@ class Interpolate(Diagnostic): """Run the diagnostic""" variable_file = TempFile.get() Utils.copy_file(self.original.local_file, variable_file) - Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False, rename_dimension=True) + Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False) cdo = Utils.cdo nco = Utils.nco handler = Utils.open_cdf(variable_file) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index eccdf7eaa17cc3cbc008a5d32456876a3b7bc71d..2bd5dcbd38580ab0f03070c7bfcef57613d2b795 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -259,8 +259,7 @@ class InterpolateCDO(Diagnostic): Utils.rename_variables(variable_file, {'jpib': 'i', 'jpjb': 'j', 'x': 'i', 'y': 'j', 'time_counter': 'time', 't': 'time', 'SSTK_ens0': 'tos', 'SSTK_ens1': 'tos', 'SSTK_ens2': 'tos', - 'nav_lat': 'lat', 'nav_lon': 'lon'}, - must_exist=False, rename_dimension=True) + 'nav_lat': 'lat', 'nav_lon': 'lon'}, must_exist=False) handler = Utils.open_cdf(variable_file) lat_name, lon_name = self._get_lat_lon_alias(handler) var = handler.variables[self.variable] @@ -290,7 +289,7 @@ class InterpolateCDO(Diagnostic): handler.close() if lat_name != 'lat': - Utils.rename_variables(temp, {'lat': lat_name, 'lon': lon_name}, True, True) + Utils.rename_variables(temp, {'lat': lat_name, 'lon': lon_name}, True) self.regridded.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index 529d71084a7020ac2e091bd83411608c8640a490..d35e2876f33bbbfc4baed79975054817cb6eb2ab 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -86,6 +86,6 @@ class MixedLayerHeatContent(Diagnostic): os.remove(temperature_file) - Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlheatc': 'ohcvsumlotst'}, False, True) + Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlheatc': 'ohcvsumlotst'}, False) Utils.setminmax(temp, 'ohcvsumlotst') self.ohcsum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 170815fe87c77dfc63699384b0bcfba0f5806606..4d6f20190b1750ac7c0611b4ec9c11c720e0f513 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -85,6 +85,6 @@ class MixedLayerSaltContent(Diagnostic): cdftools.run('cdfmxlsaltc', input_file=salinity_file, output_file=temp) os.remove(salinity_file) - Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False, True) + Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False) Utils.setminmax(temp, 'scvsummlotst') self.sosum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index afb452242a39c524f808eacfec05ca57ef142279..c0b3150b1d0fca2b1189689aae322163e69e5e46 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -142,7 +142,7 @@ class RegionSum(Diagnostic): cdftools.run('cdfsum', input_file=variable_file, input_option='-f', output_file=mean_file, options=cdfmean_options) - Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) + Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False) self._send_var(False, mean_file) self._send_var(True, mean_file) diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 21e2e053310ae93a9e481f49f51d88be7cc452c2..58fdd14328c15ff565858e47e4f0518187af6f6f 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -130,7 +130,7 @@ class Rotation(Diagnostic): handler.renameDimension('record', 'lev') handler.close() Utils.nco.ncpdq(input=temp, output=temp, options=('-O -h -a time,lev',)) - Utils.rename_variables(temp, {'x': 'i', 'y': 'j'}, must_exist=False, rename_dimension=True) + Utils.rename_variables(temp, {'x': 'i', 'y': 'j'}, must_exist=False) else: Utils.move_file(self._get_level_file(0, direction), temp) return temp diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index d60d2bb7fb5b7d018ea9667300f17fdc6b1824da..8124ffc2e013b4736a948e71a242327cbfadc8ef 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -183,11 +183,11 @@ class Siasiesiv(Diagnostic): region = data.coord('region').points data.remove_coord('region') iris.save(data, temp, zlib=True) - Utils.rename_variable(temp, 'dim0', 'region', False, True) + Utils.rename_variable(temp, 'dim0', 'region', False) handler = Utils.open_cdf(temp) var = handler.createVariable('region2', str, ('region',)) var[...] = region handler.close() - Utils.rename_variable(temp, 'region2', 'region', True, False) + Utils.rename_variable(temp, 'region2', 'region', True) generated_file.set_local_file(temp, region=basins.keys()) diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index ca1c77dfd0960a546b2ac694945337ccd470e83b..85ffaa559bab47549d2d9ffdeb31af6327e9224a 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -203,8 +203,7 @@ class DaysOverPercentile(Diagnostic): def _save_to_file(perc, results_over, var_daysover): temp = TempFile.get() iris.save(results_over[perc].merge_cube(), temp, zlib=True, unlimited_dimensions=['time']) - Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, - must_exist=False, rename_dimension=True) + Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, must_exist=False) handler = Utils.open_cdf(temp) if 'time' not in handler.dimensions: new_file = TempFile.get() diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index bbd33dd6f85773572b629379399bb65ac0b82f43..8fb146ac87a52d72ee95521ba1fa79c7ab4e0ab8 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -156,7 +156,7 @@ class MonthlyPercentile(Diagnostic): start_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[start_index]) end_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[end_index]) Utils.cdo.seldate('{0},{1}'.format(start_date, end_date), input=self.variable_file.local_file, output=temp) - Utils.rename_variable(temp, 'lev', 'ensemble', False, True) + Utils.rename_variable(temp, 'lev', 'ensemble', False) else: Utils.copy_file(self.variable_file.local_file, temp) @@ -171,19 +171,19 @@ class MonthlyPercentile(Diagnostic): for percentile in self.percentiles: Log.debug('Computing percentile {0}', percentile) Utils.cdo.monpctl(str(percentile), input=[temp, monmin_file, monmax_file], output=temp) - Utils.rename_variable(temp, 'lev', 'ensemble', False, True) + Utils.rename_variable(temp, 'lev', 'ensemble', False) handler = Utils.open_cdf(monmax_file) handler.variables[self.variable].long_name += ' {0} Percentile'.format(percentile) handler.close() self.percentiles[percentile].set_local_file(temp, rename_var=self.variable) - Utils.rename_variable(monmax_file, 'lev', 'ensemble', False, True) + Utils.rename_variable(monmax_file, 'lev', 'ensemble', False) handler = Utils.open_cdf(monmax_file) handler.variables[self.variable].long_name += ' Monthly Maximum' handler.close() self.max_file.set_local_file(monmax_file, rename_var=self.variable) - Utils.rename_variable(monmin_file, 'lev', 'ensemble', False, True) + Utils.rename_variable(monmin_file, 'lev', 'ensemble', False) handler = Utils.open_cdf(monmin_file) handler.variables[self.variable].long_name += ' Monthly Minimum' handler.close() diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index a570f39a7eaa897b69ca385b0772f99808e07907..632debe4bc43eff8a18e1d2f7ab7e996b9fca2b8 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -109,7 +109,7 @@ class Utils(object): handler.close() @staticmethod - def rename_variable(filepath, old_name, new_name, must_exist=True, rename_dimension=False): + def rename_variable(filepath, old_name, new_name, must_exist=True): """ Rename variable from a NetCDF file @@ -121,17 +121,16 @@ class Utils(object): old_name: str new_name: str must_exist: bool, optional - rename_dimension: bool, optional See Also -------- Utils.rename_variables """ - Utils.rename_variables(filepath, {old_name: new_name}, must_exist, rename_dimension) + Utils.rename_variables(filepath, {old_name: new_name}, must_exist) @staticmethod - def rename_variables(filepath, dic_names, must_exist=True, rename_dimension=False): + def rename_variables(filepath, dic_names, must_exist=True): """ Rename multiple variables from a NetCDF file @@ -141,7 +140,6 @@ class Utils(object): dic_names: dict of str: str Gives the renaming to do in the form old_name: new_name must_exist: bool, optional - rename_dimension: bool, optional Raises ------- @@ -153,36 +151,26 @@ class Utils(object): for old, new in six.iteritems(dic_names): if old == new: raise ValueError('{0} original name is the same as the new') - handler = Utils.open_cdf(filepath) + original_handler = Utils.open_cdf(filepath) - original_names = set(handler.variables.keys()).union(handler.dimensions.keys()) + original_names = set(original_handler.variables.keys()).union(original_handler.dimensions.keys()) if not any((True for x in dic_names.keys() if x in original_names)): - handler.close() + original_handler.close() if must_exist: raise Exception("Variables {0} does not exist in file {1}".format(','.join(dic_names.keys()), filepath)) return - handler.close() temp = TempFile.get() - shutil.copyfile(filepath, temp) - - handler = Utils.open_cdf(temp) - error = False - - try: - Utils._rename_vars_directly(dic_names, filepath, handler, must_exist, rename_dimension) - except RuntimeError as ex: - Log.debug('Renaming error: {0}', ex) - error = True - handler.close() - - if not error and not Utils.check_netcdf_file(temp): - error = True - - if error: - Log.debug('First attemp to rename failed. Using secondary rename method for netCDF') - Utils._rename_by_new_file(dic_names, filepath, temp) - Log.debug('Rename done') + new_handler = Utils.open_cdf(temp, 'w') + for attribute in original_handler.ncattrs(): + original = getattr(original_handler, attribute) + setattr(new_handler, attribute, Utils.convert_to_ascii_if_possible(original)) + for dimension in original_handler.dimensions.keys(): + Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) + for variable in original_handler.variables.keys(): + Utils.copy_variable(original_handler, new_handler, variable, new_names=dic_names) + original_handler.close() + new_handler.close() Utils.move_file(temp, filepath) @@ -239,20 +227,6 @@ class Utils(object): handler.close() return variables - @staticmethod - def _rename_by_new_file(dic_names, filepath, temp): - original_handler = Utils.open_cdf(filepath) - new_handler = Utils.open_cdf(temp, 'w') - for attribute in original_handler.ncattrs(): - original = getattr(original_handler, attribute) - setattr(new_handler, attribute, Utils.convert_to_ascii_if_possible(original)) - for dimension in original_handler.dimensions.keys(): - Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) - for variable in original_handler.variables.keys(): - Utils.copy_variable(original_handler, new_handler, variable, new_names=dic_names) - original_handler.close() - new_handler.close() - # noinspection PyPep8Naming @staticmethod def convert_to_ascii_if_possible(string, encoding='ascii'): @@ -834,6 +808,7 @@ class Utils(object): if os.path.exists(os.path.join(destiny_path, file_compressed.name)): os.remove(os.path.join(destiny_path, file_compressed.name)) tar.extract(file_compressed, destiny_path) + Log.debug('File {0} extracted', os.path.basename(file_compressed.name)) tar.close() @staticmethod diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index daec40612d2e4a11ed0870ffecf04947d39f1f02..b58f8d896910851c54ac5888fdec3f8d15c429f9 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -382,7 +382,7 @@ class Downloader(object): def _prioritize(datafile1, datafile2): waiting = Downloader._suscribers_waiting(datafile1) - Downloader._suscribers_waiting(datafile2) if waiting: - return waiting + return -waiting suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) if suscribers: diff --git a/launch_diags.sh b/launch_diags.sh index 1ff20d775f46951304c507c5c74a3ad7f8fb971d..00cd9290467cbb210215a7c5e5f1dc4236602f9d 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -7,7 +7,7 @@ -PATH_TO_CONF_FILE=~/earthdiags_confs/ruben/diags_a0dc_DEBUG.conf +PATH_TO_CONF_FILE=~jvegas/PycharmProjects/earthdiagnostics/diags.conf PATH_TO_DIAGNOSTICS=~jvegas/PycharmProjects/earthdiagnostics PATH_TO_CONDAENV=/home/Earth/jvegas/.conda/envs/earthdiagnostics3/ diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index 313456fe3b51ec3095b0284c9fee5510c0f8827a..8a5900d6681b01c23fd64decddba7a190d899dfb 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -8,6 +8,7 @@ from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.ocean.regionmean import RegionMean +from earthdiagnostics.utils import TempFile class TestRegionMean(TestCase): @@ -22,7 +23,12 @@ class TestRegionMean(TestCase): raise DiagnosticOptionError return value + @staticmethod + def fake_get(): + return 'tempfile' + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + @patch.object(TempFile, 'get', fake_get) def test_generate_jobs(self): box = Box() diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 9c32c184f7ef58a73bb9f17321091eba9d517c39..8e634029b42af500c671dd7a45fa950a207c7067 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -37,6 +37,9 @@ class TestCMORManager(TestCase): self.config.cmor.activity = 'activity' self.config.cmor.force = False self.config.cmor.force_untar = False + self.config.cmor.cmorize_atmosphere = True + self.config.cmor.cmorize_ocean = True + self.config.cmor.skip_prepare = False self.config.cmor.append_startdate = False self.tmp_dir = tempfile.mkdtemp() diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 3b274e3f1075d34f70d6d1253570d79f09268af6..7e096de5f94abe30d85a595439fa6efbd8c62689 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -54,6 +54,6 @@ class TestUtils(TestCase): def test_rename_variable(self): with mock.patch('earthdiagnostics.utils.Utils.rename_variables') as rename_mock: Utils.rename_variable('file', 'old', 'new') - Utils.rename_variable('file', 'old', 'new', False, True) - rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True, False), - mock.call('file', {'old': 'new'}, False, True))) + Utils.rename_variable('file', 'old', 'new', False) + rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True), + mock.call('file', {'old': 'new'}, False))) diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 121e56173ad00343d13137681d41e29e59fc67e4..2cf96ad96e89e0b0924772fa0e4d542121829855 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -168,8 +168,8 @@ class TestDownloader(TestCase): self.downloader.start() self.downloader.shutdown() self.assertListEqual(cmd.output, - ['INFO:bscearth.utils:Suscribers: (2,) Size: 0', - 'INFO:bscearth.utils:Suscribers: (1,) Size: 0']) + ['INFO:bscearth.utils:Suscribers: (1,) Size: 0', + 'INFO:bscearth.utils:Suscribers: (2,) Size: 0']) else: self.downloader.start() self.downloader.shutdown() diff --git a/variable_to_interpolated.py b/variable_to_interpolated.py new file mode 100644 index 0000000000000000000000000000000000000000..0c4b35422978a7bba4bd0cfdf8d88195b3907d38 --- /dev/null +++ b/variable_to_interpolated.py @@ -0,0 +1,21 @@ +"""Script to move a normal variable to make it look as it is an interpolated one""" +import shutil +import os + +EXP_PATH = '/esarchive/exp/ecearth/a0pe/cmorfiles/BSC/EC-EARTH3/a0pe/' +GRID_NAME = 'grib' + +for startdate in os.listdir(EXP_PATH): + var_path = os.path.join(EXP_PATH, startdate, 'mon', 'atmos', 'ta') + new_var_path = os.path.join(var_path, GRID_NAME) + if not os.path.isdir(var_path): + continue + if not os.path.exists(new_var_path): + os.makedirs(new_var_path) + + for member in os.listdir(var_path): + if not member.endswith('i1p1'): + continue + member_path = os.path.join(var_path, member) + destiny_member_path = os.path.join(new_var_path, member) + shutil.move(member_path, destiny_member_path)