From ade0b955312226416fe570f76636cc133171bba7 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Mar 2018 16:59:39 +0100 Subject: [PATCH 01/12] Bumped version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2638df16..7bd1094b 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc3 +3.0.0rc4 -- GitLab From cc4e2d64bdfd510607cfc443aa1e8ce61bf19499 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Mar 2018 17:00:58 +0100 Subject: [PATCH 02/12] Rename will always create a new file and removed CDO from timemeans. Improved MMA cmorization --- earthdiagnostics/cmorizer.py | 39 +++--- earthdiagnostics/cmormanager.py | 6 +- earthdiagnostics/config.py | 1 + earthdiagnostics/datafile.py | 10 +- earthdiagnostics/datamanager.py | 2 +- earthdiagnostics/diagnostic.py | 4 +- earthdiagnostics/earthdiags.py | 2 +- earthdiagnostics/general/relink.py | 3 + earthdiagnostics/general/relinkall.py | 3 + earthdiagnostics/general/timemean.py | 112 ++++++++++++------ earthdiagnostics/ocean/heatcontentlayer.py | 2 +- earthdiagnostics/ocean/interpolate.py | 2 +- earthdiagnostics/ocean/interpolatecdo.py | 5 +- .../ocean/mixedlayerheatcontent.py | 2 +- .../ocean/mixedlayersaltcontent.py | 2 +- earthdiagnostics/ocean/regionsum.py | 2 +- earthdiagnostics/ocean/rotation.py | 2 +- .../statistics/daysoverpercentile.py | 3 +- earthdiagnostics/utils.py | 57 +++------ 19 files changed, 147 insertions(+), 112 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 42099f32..0b1c5405 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -176,22 +176,21 @@ class Cmorizer(object): def _merge_mma_files(self, tarfile): temp = TempFile.get() - sh_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_SH_*.nc')) - gg_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_GG_*.nc')) - - merged_sh = TempFile.get() - merged_gg = TempFile.get() - - for filename in sh_files: - Utils.cdo.sp2gpl(options='-O', input=filename, output=temp) - shutil.move(temp, filename) - Utils.cdo.mergetime(input=sh_files, output=merged_sh) - Utils.cdo.mergetime(input=gg_files, output=merged_gg) - for filename in sh_files + gg_files: - os.remove(filename) - tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') - shutil.move(merged_gg, os.path.join(self.cmor_scratch, 'MMAGG_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) - shutil.move(merged_sh, os.path.join(self.cmor_scratch, 'MMASH_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) + for grid in ['SH', 'GG']: + files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_{}_*.nc'.format(grid))) + if not files: + continue + merged = TempFile.get() + if grid == 'SH': + for filename in files: + Utils.cdo.sp2gpl(options='-O', input=filename, output=temp) + shutil.move(temp, filename) + Utils.cdo.mergetime(input=files, output=merged) + for filename in files: + os.remove(filename) + tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') + filename = 'MMA{0}_1m_{1[0]}_{1[1]}.nc'.format(grid, tar_startdate) + shutil.move(merged, os.path.join(self.cmor_scratch, filename)) def cmorize_atmos(self): """Cmorize atmospheric data, from grib or MMA files""" @@ -371,7 +370,7 @@ class Cmorizer(object): # Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) - Utils.rename_variables(filename, self.alt_coord_names, False, True) + Utils.rename_variables(filename, self.alt_coord_names, False) handler = Utils.open_cdf(filename) Cmorizer._remove_valid_limits(handler) self._add_common_attributes(handler, frequency) @@ -527,12 +526,12 @@ class Cmorizer(object): def _rename_level_variables(temp, var_cmor): if var_cmor.domain == ModelingRealms.ocean: Utils.rename_variables(temp, {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', - 'depth': 'lev'}, False, True) + 'depth': 'lev'}, False) if var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: Utils.rename_variables(temp, {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', - 'depth_4': 'sdepth'}, False, True) + 'depth_4': 'sdepth'}, False) if var_cmor.domain == ModelingRealms.atmos: - Utils.rename_variables(temp, {'depth': 'plev'}, False, True) + Utils.rename_variables(temp, {'depth': 'plev'}, False) def _merge_grib_files(self, current_month, prev_gribfile, gribfile): Log.info('Merging data from different files...') diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 9d2fa84f..1fbccd75 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -484,8 +484,8 @@ class CMORManager(DataManager): """ Prepare the data to be used by the diagnostic. - If CMOR data is not created, it show a warning and closes. In the future, an automatic cmorization procedure - will be launched + If CMOR data is not created, an automatic cmorization procedure + is launched If CMOR data is available but packed, the procedure will unpack it. @@ -494,6 +494,8 @@ class CMORManager(DataManager): if self.config.data_convention == 'meteofrance': return + if self.config.cmor.skip_prepare: + return for startdate, member in self.experiment.get_member_list(): Log.info('Checking data for startdate {0} member {1}', startdate, member) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 734f94db..2b2dea62 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -217,6 +217,7 @@ class CMORConfig(object): def __init__(self, parser, var_manager): self.force = parser.get_bool_option('CMOR', 'FORCE', False) self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) + self.skip_prepare = parser.get_bool_option('CMOR', 'SKIP_PREPARE', False) self.filter_files = parser.get_option('CMOR', 'FILTER_FILES', '') self.ocean = parser.get_bool_option('CMOR', 'OCEAN_FILES', True) self.atmosphere = parser.get_bool_option('CMOR', 'ATMOSPHERE_FILES', True) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 086273e1..5534209b 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -123,6 +123,7 @@ class DataFile(Publisher): if self.has_modifiers(): return True + return False def add_modifier(self, diagnostic): """ @@ -308,10 +309,15 @@ class DataFile(Publisher): coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name, 'leadtime', 'region', 'time_centered'}, set(handler.variables.keys())) var_handler.coordinates = ' '.join(coords) + if 'time_centered' in handler.variables: + if hasattr(handler.variables['time_centered'], 'standard_name'): + del handler.variables['time_centered'].standard_name if not self.cmor_var: handler.close() return + + self._fix_variable_name(var_handler) handler.modeling_realm = self.cmor_var.domain.name table = self.cmor_var.get_table(self.frequency, self.data_convention) @@ -471,7 +477,7 @@ class DataFile(Publisher): variables['nav_lon_grid_U'] = self.lon_name variables['nav_lat_grid_T'] = self.lat_name variables['nav_lon_grid_T'] = self.lon_name - Utils.rename_variables(self.local_file, variables, False, True) + Utils.rename_variables(self.local_file, variables, False) def add_diagnostic_history(self): """Add the history line corresponding to the diagnostic to the local file""" @@ -627,7 +633,7 @@ class NetCDFFile(DataFile): alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} - Utils.rename_variables(self.local_file, alt_coord_names, must_exist=False, rename_dimension=True) + Utils.rename_variables(self.local_file, alt_coord_names, must_exist=False) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index ebf6e8a8..9bd4fb94 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -29,7 +29,7 @@ class DataManager(object): if filepath not in self.requested_files: self.requested_files[filepath] = NCfile.from_storage(filepath, self.config.data_convention) file_object = self.requested_files[filepath] - file_object.local_satatus = LocalStatus.PENDING + file_object.local_status = LocalStatus.PENDING return self.requested_files[filepath] def _declare_generated_file(self, remote_file, domain, final_var, cmor_var, data_convention, diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 099ef4f1..20d41ac4 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -86,6 +86,8 @@ class Diagnostic(Publisher): bool """ + if not self._generated_files: + return False for file_generated in self._generated_files: if file_generated.storage_status != StorageStatus.READY: return False @@ -326,7 +328,7 @@ class Diagnostic(Publisher): chunk: int or None grid: str or None box: Box or None - frequency: Frequency or None + frequency: Frequency or str or None to_modify: bool Flag that must be active if the diagnostic is going to generate a modified version of this data. In this case this data must not be declared as an output of the diagnostic diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 86a5e6bc..e52035bc 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -414,7 +414,7 @@ class EarthDiags(object): Log.info('Copying file {0}', destiny) shutil.copyfile(source, destiny) Log.info('File {0} ready', destiny) - Utils.rename_variables(destiny, self.dic_variables, False, True) + Utils.rename_variables(destiny, self.dic_variables, False) return True def _link_file(self, source, destiny): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index fcae8e51..aecbfa95 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -46,6 +46,9 @@ class Relink(Diagnostic): return 'Relink output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Move old: {0.move_old} ' \ 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): if self._different_type(other): return False diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 1b59e1b2..10a6dd0d 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -27,6 +27,9 @@ class RelinkAll(Diagnostic): def __str__(self): return 'Relink all output Startdate: {0}'.format(self.startdate) + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): if self._different_type(other): return False diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 2addbe5e..0aa3d014 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -1,6 +1,11 @@ # coding=utf-8 """Time mean diagnostics""" -import os +import iris +import iris.coord_categorisation +import iris.analysis +import iris.exceptions + +import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticFrequencyOption, DiagnosticVariableOption @@ -40,11 +45,15 @@ class TimeMean(Diagnostic): self.frequency = frequency self.grid = grid self._target_frequency = None + self.mean_file = None def __str__(self): return 'Calculate {0._target_frequency} mean Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ 'Variable: {0.domain}:{0.variable} Original frequency: {0.frequency} Grid: {0.grid}'.format(self) + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): if self._different_type(other): return False @@ -84,38 +93,54 @@ class TimeMean(Diagnostic): self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, frequency=self.frequency, grid=self.grid) - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.daymean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.daily, grid=self.grid) - - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ raise NotImplementedError() def compute(self): """Run the diagnostic""" temp = TempFile.get() - handler = Utils.open_cdf(self.variable_file.local_file) - if 'region' in handler.variables: - noregion = TempFile.get() - Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) - self.compute_mean(noregion, temp) - os.remove(noregion) - monmean_handler = Utils.open_cdf(temp) - Utils.copy_variable(handler, monmean_handler, 'region') - monmean_handler.close() - else: - self.compute_mean(self.variable_file.local_file, temp) - self.daymean.set_local_file(temp) + cube = iris.load_cube(self.variable_file.local_file) + time_centered = [coord for coord in cube.coords() if coord.var_name == 'time_centered'] + if time_centered: + cube.remove_coord(time_centered[0]) + iris.coord_categorisation.add_day_of_month(cube, 'time') + iris.coord_categorisation.add_month_number(cube, 'time') + iris.coord_categorisation.add_year(cube, 'time') + + cube = self.compute_mean(cube) + cube.remove_coord('day_of_month') + cube.remove_coord('month_number') + cube.remove_coord('year') + try: + region_coord = cube.coord('region') + cube.remove_coord(region_coord) + except iris.exceptions.CoordinateNotFoundError: + region_coord = None + iris.FUTURE.netcdf_no_unlimited = True + iris.save(cube, temp) + if region_coord: + handler = Utils.open_cdf(temp) + region = handler.createVariable('region', str, ('dim0',)) + region.standard_name = region_coord.standard_name + region[...] = region_coord.points.astype(np.dtype(str)) + + handler.variables[self.variable].coordinates += ' region' + handler.close() + + Utils.rename_variable(temp, 'dim0', 'region', False, True, force_copy=True) + + self.mean_file.set_local_file(temp) class DailyMean(TimeMean): @@ -146,17 +171,24 @@ class DailyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'daily' - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ - Utils.cdo.daymean(input=input_file, output=output_file) + return cube.aggregated_by(['day_of_month', 'month_number', 'year'], iris.analysis.MEAN) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.mean_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.daily, grid=self.grid) class MonthlyMean(TimeMean): @@ -187,17 +219,24 @@ class MonthlyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'monthly' - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ - Utils.cdo.monmean(input=input_file, output=output_file) + return cube.aggregated_by(['month_number', 'year'], iris.analysis.MEAN) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.mean_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, grid=self.grid) class YearlyMean(TimeMean): @@ -228,14 +267,21 @@ class YearlyMean(TimeMean): TimeMean.__init__(self, data_manager, startdate, member, chunk, domain, variable, frequency, grid) self._target_frequency = 'yearly' - def compute_mean(self, input_file, output_file): + def compute_mean(self, cube): """ Compute the time mean Parameters ---------- - input_file: str - output_file: str + cube: iris.cube.Cube + Returns + ------- + iris.cube.Cube """ - Utils.cdo.monmean(input=input_file, output=output_file) + return cube.aggregated_by(['year'], iris.analysis.MEAN) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.mean_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.yearly, grid=self.grid) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 135f9f4b..f2e02465 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -169,7 +169,7 @@ class HeatContentLayer(Diagnostic): handler.close() nco.ncks(input=thetao_file, output=results, options=('-O -v lon,lat,time',)) - Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False, True) + Utils.rename_variables(results, {'x': 'i', 'y': 'j'}, False) handler_results = Utils.open_cdf(results) handler_results.createVariable('heatc', float, ('time', 'j', 'i'), fill_value=1.e20) handler_results.sync() diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 76e9d78a..1ec9a4c8 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -120,7 +120,7 @@ class Interpolate(Diagnostic): """Run the diagnostic""" variable_file = TempFile.get() Utils.copy_file(self.original.local_file, variable_file) - Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False, rename_dimension=True) + Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False) cdo = Utils.cdo nco = Utils.nco handler = Utils.open_cdf(variable_file) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index eccdf7ea..2bd5dcbd 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -259,8 +259,7 @@ class InterpolateCDO(Diagnostic): Utils.rename_variables(variable_file, {'jpib': 'i', 'jpjb': 'j', 'x': 'i', 'y': 'j', 'time_counter': 'time', 't': 'time', 'SSTK_ens0': 'tos', 'SSTK_ens1': 'tos', 'SSTK_ens2': 'tos', - 'nav_lat': 'lat', 'nav_lon': 'lon'}, - must_exist=False, rename_dimension=True) + 'nav_lat': 'lat', 'nav_lon': 'lon'}, must_exist=False) handler = Utils.open_cdf(variable_file) lat_name, lon_name = self._get_lat_lon_alias(handler) var = handler.variables[self.variable] @@ -290,7 +289,7 @@ class InterpolateCDO(Diagnostic): handler.close() if lat_name != 'lat': - Utils.rename_variables(temp, {'lat': lat_name, 'lon': lon_name}, True, True) + Utils.rename_variables(temp, {'lat': lat_name, 'lon': lon_name}, True) self.regridded.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index 529d7108..d35e2876 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -86,6 +86,6 @@ class MixedLayerHeatContent(Diagnostic): os.remove(temperature_file) - Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlheatc': 'ohcvsumlotst'}, False, True) + Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlheatc': 'ohcvsumlotst'}, False) Utils.setminmax(temp, 'ohcvsumlotst') self.ohcsum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 170815fe..4d6f2019 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -85,6 +85,6 @@ class MixedLayerSaltContent(Diagnostic): cdftools.run('cdfmxlsaltc', input_file=salinity_file, output_file=temp) os.remove(salinity_file) - Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False, True) + Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False) Utils.setminmax(temp, 'scvsummlotst') self.sosum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index afb45224..c0b3150b 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -142,7 +142,7 @@ class RegionSum(Diagnostic): cdftools.run('cdfsum', input_file=variable_file, input_option='-f', output_file=mean_file, options=cdfmean_options) - Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) + Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False) self._send_var(False, mean_file) self._send_var(True, mean_file) diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index 21e2e053..58fdd143 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -130,7 +130,7 @@ class Rotation(Diagnostic): handler.renameDimension('record', 'lev') handler.close() Utils.nco.ncpdq(input=temp, output=temp, options=('-O -h -a time,lev',)) - Utils.rename_variables(temp, {'x': 'i', 'y': 'j'}, must_exist=False, rename_dimension=True) + Utils.rename_variables(temp, {'x': 'i', 'y': 'j'}, must_exist=False) else: Utils.move_file(self._get_level_file(0, direction), temp) return temp diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index ca1c77df..85ffaa55 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -203,8 +203,7 @@ class DaysOverPercentile(Diagnostic): def _save_to_file(perc, results_over, var_daysover): temp = TempFile.get() iris.save(results_over[perc].merge_cube(), temp, zlib=True, unlimited_dimensions=['time']) - Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, - must_exist=False, rename_dimension=True) + Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, must_exist=False) handler = Utils.open_cdf(temp) if 'time' not in handler.dimensions: new_file = TempFile.get() diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index a570f39a..a5c00b94 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -109,7 +109,7 @@ class Utils(object): handler.close() @staticmethod - def rename_variable(filepath, old_name, new_name, must_exist=True, rename_dimension=False): + def rename_variable(filepath, old_name, new_name, must_exist=True, rename_dimension=False, force_copy=False): """ Rename variable from a NetCDF file @@ -128,10 +128,10 @@ class Utils(object): Utils.rename_variables """ - Utils.rename_variables(filepath, {old_name: new_name}, must_exist, rename_dimension) + Utils.rename_variables(filepath, {old_name: new_name}, must_exist) @staticmethod - def rename_variables(filepath, dic_names, must_exist=True, rename_dimension=False): + def rename_variables(filepath, dic_names, must_exist=True): """ Rename multiple variables from a NetCDF file @@ -141,7 +141,6 @@ class Utils(object): dic_names: dict of str: str Gives the renaming to do in the form old_name: new_name must_exist: bool, optional - rename_dimension: bool, optional Raises ------- @@ -153,36 +152,26 @@ class Utils(object): for old, new in six.iteritems(dic_names): if old == new: raise ValueError('{0} original name is the same as the new') - handler = Utils.open_cdf(filepath) + original_handler = Utils.open_cdf(filepath) - original_names = set(handler.variables.keys()).union(handler.dimensions.keys()) + original_names = set(original_handler.variables.keys()).union(original_handler.dimensions.keys()) if not any((True for x in dic_names.keys() if x in original_names)): - handler.close() + original_handler.close() if must_exist: raise Exception("Variables {0} does not exist in file {1}".format(','.join(dic_names.keys()), filepath)) return - handler.close() temp = TempFile.get() - shutil.copyfile(filepath, temp) - - handler = Utils.open_cdf(temp) - error = False - - try: - Utils._rename_vars_directly(dic_names, filepath, handler, must_exist, rename_dimension) - except RuntimeError as ex: - Log.debug('Renaming error: {0}', ex) - error = True - handler.close() - - if not error and not Utils.check_netcdf_file(temp): - error = True - - if error: - Log.debug('First attemp to rename failed. Using secondary rename method for netCDF') - Utils._rename_by_new_file(dic_names, filepath, temp) - Log.debug('Rename done') + new_handler = Utils.open_cdf(temp, 'w') + for attribute in original_handler.ncattrs(): + original = getattr(original_handler, attribute) + setattr(new_handler, attribute, Utils.convert_to_ascii_if_possible(original)) + for dimension in original_handler.dimensions.keys(): + Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) + for variable in original_handler.variables.keys(): + Utils.copy_variable(original_handler, new_handler, variable, new_names=dic_names) + original_handler.close() + new_handler.close() Utils.move_file(temp, filepath) @@ -239,20 +228,6 @@ class Utils(object): handler.close() return variables - @staticmethod - def _rename_by_new_file(dic_names, filepath, temp): - original_handler = Utils.open_cdf(filepath) - new_handler = Utils.open_cdf(temp, 'w') - for attribute in original_handler.ncattrs(): - original = getattr(original_handler, attribute) - setattr(new_handler, attribute, Utils.convert_to_ascii_if_possible(original)) - for dimension in original_handler.dimensions.keys(): - Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) - for variable in original_handler.variables.keys(): - Utils.copy_variable(original_handler, new_handler, variable, new_names=dic_names) - original_handler.close() - new_handler.close() - # noinspection PyPep8Naming @staticmethod def convert_to_ascii_if_possible(string, encoding='ascii'): -- GitLab From 9331243e19b983bc49fa9bb43acf2ce521966f02 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Mar 2018 11:11:45 +0100 Subject: [PATCH 03/12] Fixed tests --- earthdiagnostics/datafile.py | 2 -- earthdiagnostics/general/timemean.py | 2 +- earthdiagnostics/ocean/siasiesiv.py | 4 ++-- .../statistics/monthlypercentile.py | 8 ++++---- earthdiagnostics/utils.py | 3 +-- test/unit/test_cmormanager.py | 3 +++ test/unit/test_utils.py | 6 +++--- variable_to_interpolated.py | 19 +++++++++++++++++++ 8 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 variable_to_interpolated.py diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 5534209b..b94f7b61 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -316,8 +316,6 @@ class DataFile(Publisher): handler.close() return - - self._fix_variable_name(var_handler) handler.modeling_realm = self.cmor_var.domain.name table = self.cmor_var.get_table(self.frequency, self.data_convention) diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 0aa3d014..74d3724c 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -138,7 +138,7 @@ class TimeMean(Diagnostic): handler.variables[self.variable].coordinates += ' region' handler.close() - Utils.rename_variable(temp, 'dim0', 'region', False, True, force_copy=True) + Utils.rename_variable(temp, 'dim0', 'region', False) self.mean_file.set_local_file(temp) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index d60d2bb7..8124ffc2 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -183,11 +183,11 @@ class Siasiesiv(Diagnostic): region = data.coord('region').points data.remove_coord('region') iris.save(data, temp, zlib=True) - Utils.rename_variable(temp, 'dim0', 'region', False, True) + Utils.rename_variable(temp, 'dim0', 'region', False) handler = Utils.open_cdf(temp) var = handler.createVariable('region2', str, ('region',)) var[...] = region handler.close() - Utils.rename_variable(temp, 'region2', 'region', True, False) + Utils.rename_variable(temp, 'region2', 'region', True) generated_file.set_local_file(temp, region=basins.keys()) diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index bbd33dd6..8fb146ac 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -156,7 +156,7 @@ class MonthlyPercentile(Diagnostic): start_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[start_index]) end_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[end_index]) Utils.cdo.seldate('{0},{1}'.format(start_date, end_date), input=self.variable_file.local_file, output=temp) - Utils.rename_variable(temp, 'lev', 'ensemble', False, True) + Utils.rename_variable(temp, 'lev', 'ensemble', False) else: Utils.copy_file(self.variable_file.local_file, temp) @@ -171,19 +171,19 @@ class MonthlyPercentile(Diagnostic): for percentile in self.percentiles: Log.debug('Computing percentile {0}', percentile) Utils.cdo.monpctl(str(percentile), input=[temp, monmin_file, monmax_file], output=temp) - Utils.rename_variable(temp, 'lev', 'ensemble', False, True) + Utils.rename_variable(temp, 'lev', 'ensemble', False) handler = Utils.open_cdf(monmax_file) handler.variables[self.variable].long_name += ' {0} Percentile'.format(percentile) handler.close() self.percentiles[percentile].set_local_file(temp, rename_var=self.variable) - Utils.rename_variable(monmax_file, 'lev', 'ensemble', False, True) + Utils.rename_variable(monmax_file, 'lev', 'ensemble', False) handler = Utils.open_cdf(monmax_file) handler.variables[self.variable].long_name += ' Monthly Maximum' handler.close() self.max_file.set_local_file(monmax_file, rename_var=self.variable) - Utils.rename_variable(monmin_file, 'lev', 'ensemble', False, True) + Utils.rename_variable(monmin_file, 'lev', 'ensemble', False) handler = Utils.open_cdf(monmin_file) handler.variables[self.variable].long_name += ' Monthly Minimum' handler.close() diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index a5c00b94..f89df7d5 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -109,7 +109,7 @@ class Utils(object): handler.close() @staticmethod - def rename_variable(filepath, old_name, new_name, must_exist=True, rename_dimension=False, force_copy=False): + def rename_variable(filepath, old_name, new_name, must_exist=True): """ Rename variable from a NetCDF file @@ -121,7 +121,6 @@ class Utils(object): old_name: str new_name: str must_exist: bool, optional - rename_dimension: bool, optional See Also -------- diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 9c32c184..8e634029 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -37,6 +37,9 @@ class TestCMORManager(TestCase): self.config.cmor.activity = 'activity' self.config.cmor.force = False self.config.cmor.force_untar = False + self.config.cmor.cmorize_atmosphere = True + self.config.cmor.cmorize_ocean = True + self.config.cmor.skip_prepare = False self.config.cmor.append_startdate = False self.tmp_dir = tempfile.mkdtemp() diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 3b274e3f..7e096de5 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -54,6 +54,6 @@ class TestUtils(TestCase): def test_rename_variable(self): with mock.patch('earthdiagnostics.utils.Utils.rename_variables') as rename_mock: Utils.rename_variable('file', 'old', 'new') - Utils.rename_variable('file', 'old', 'new', False, True) - rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True, False), - mock.call('file', {'old': 'new'}, False, True))) + Utils.rename_variable('file', 'old', 'new', False) + rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True), + mock.call('file', {'old': 'new'}, False))) diff --git a/variable_to_interpolated.py b/variable_to_interpolated.py new file mode 100644 index 00000000..61e9e4b5 --- /dev/null +++ b/variable_to_interpolated.py @@ -0,0 +1,19 @@ +import shutil +import os + +path = '/esarchive/exp/ecearth/a0pe/cmorfiles/BSC/EC-EARTH3/a0pe/' + +for startdate in os.listdir(path): + var_path = os.path.join(path, startdate, 'mon', 'atmos', 'ta') + new_var_path = os.path.join(var_path, 'grib') + if not os.path.isdir(var_path): + continue + if not os.path.exists(new_var_path): + os.makedirs(new_var_path) + + for member in os.listdir(var_path): + if member == 'grib': + continue + member_path = os.path.join(var_path, member) + new_member_path = os.path.join(new_var_path, member) + shutil.move(member_path, new_member_path) -- GitLab From e2259c6b09dc660580f4714039b48b04e8019b79 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Mar 2018 11:24:47 +0100 Subject: [PATCH 04/12] Fixed codacy warnings --- earthdiagnostics/cmormanager.py | 41 ++++++++++++++++++--------------- variable_to_interpolated.py | 6 +++-- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 1fbccd75..1445a00f 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -498,25 +498,28 @@ class CMORManager(DataManager): return for startdate, member in self.experiment.get_member_list(): - Log.info('Checking data for startdate {0} member {1}', startdate, member) - if not self.config.cmor.force: - cmorized = False - for chunk in range(1, self.experiment.num_chunks + 1): - if not self.config.cmor.chunk_cmorization_requested(chunk): - Log.debug('Skipping chunk {0}', chunk) - continue - if not self.config.cmor.force_untar: - Log.debug('Checking chunk {0}...', chunk) - for domain in (ModelingRealms.atmos, ModelingRealms.ocean, ModelingRealms.seaIce): - if self.is_cmorized(startdate, member, chunk, domain): - Log.debug('Chunk {0} ready', chunk) - continue - if self._unpack_chunk(startdate, member, chunk): - cmorized = True - if cmorized: - Log.info('Startdate {0} member {1} ready', startdate, member) - return - self._cmorize_member(startdate, member) + self._prepare_member(startdate, member) + + def _prepare_member(self, startdate, member): + Log.info('Checking data for startdate {0} member {1}', startdate, member) + if not self.config.cmor.force: + cmorized = False + for chunk in range(1, self.experiment.num_chunks + 1): + if not self.config.cmor.chunk_cmorization_requested(chunk): + Log.debug('Skipping chunk {0}', chunk) + continue + if not self.config.cmor.force_untar: + Log.debug('Checking chunk {0}...', chunk) + for domain in (ModelingRealms.atmos, ModelingRealms.ocean, ModelingRealms.seaIce): + if self.is_cmorized(startdate, member, chunk, domain): + Log.debug('Chunk {0} ready', chunk) + return + if self._unpack_chunk(startdate, member, chunk): + cmorized = True + if cmorized: + Log.info('Startdate {0} member {1} ready', startdate, member) + return + self._cmorize_member(startdate, member) def is_cmorized(self, startdate, member, chunk, domain): """ diff --git a/variable_to_interpolated.py b/variable_to_interpolated.py index 61e9e4b5..6ea61a51 100644 --- a/variable_to_interpolated.py +++ b/variable_to_interpolated.py @@ -1,18 +1,20 @@ +"""Script to """ import shutil import os path = '/esarchive/exp/ecearth/a0pe/cmorfiles/BSC/EC-EARTH3/a0pe/' +new_grid = 'grib' for startdate in os.listdir(path): var_path = os.path.join(path, startdate, 'mon', 'atmos', 'ta') - new_var_path = os.path.join(var_path, 'grib') + new_var_path = os.path.join(var_path, new_grid) if not os.path.isdir(var_path): continue if not os.path.exists(new_var_path): os.makedirs(new_var_path) for member in os.listdir(var_path): - if member == 'grib': + if not member.endswith('i1p1'): continue member_path = os.path.join(var_path, member) new_member_path = os.path.join(new_var_path, member) -- GitLab From a3c50f8342b282c393d17b0279d08adbbb440fa6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 19 Mar 2018 11:48:32 +0100 Subject: [PATCH 05/12] Fixed bug in download priorities --- diags.conf | 13 +++++++------ earthdiagnostics/work_manager.py | 2 +- test/unit/test_workmanager.py | 4 ++-- variable_to_interpolated.py | 3 ++- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/diags.conf b/diags.conf index 64dc8269..cddd6d9f 100644 --- a/diags.conf +++ b/diags.conf @@ -16,7 +16,7 @@ CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = interpcdo,atmos,prlr,r240x121,bilinear,False,,False +DIAGS = interpcdo,atmos,sfcWind,r240x121,bilinear,False,s2s,False # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. @@ -73,13 +73,13 @@ SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = gloh2o -MODEL = mswep +INSTITUTE = ecmwf +MODEL = erainterim NAME = wekkly_means # Model version: Available versions MODEL_VERSION = # Atmospheric output timestep in hours -ATMOS_TIMESTEP = 3 +ATMOS_TIMESTEP = 6 # Ocean output timestep in hours OCEAN_TIMESTEP = 3 @@ -92,8 +92,9 @@ OCEAN_TIMESTEP = 3 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = mswep -# STARTDATES = {19970101,20161231,1D} -STARTDATES = 19970322 20010425 +#STARTDATES = {19970101,20161231,1D} +STARTDATES = {19940101,19961231,1D} +#STARTDATES = 19970322 20010425 MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index daec4061..b58f8d89 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -382,7 +382,7 @@ class Downloader(object): def _prioritize(datafile1, datafile2): waiting = Downloader._suscribers_waiting(datafile1) - Downloader._suscribers_waiting(datafile2) if waiting: - return waiting + return -waiting suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) if suscribers: diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 121e5617..2cf96ad9 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -168,8 +168,8 @@ class TestDownloader(TestCase): self.downloader.start() self.downloader.shutdown() self.assertListEqual(cmd.output, - ['INFO:bscearth.utils:Suscribers: (2,) Size: 0', - 'INFO:bscearth.utils:Suscribers: (1,) Size: 0']) + ['INFO:bscearth.utils:Suscribers: (1,) Size: 0', + 'INFO:bscearth.utils:Suscribers: (2,) Size: 0']) else: self.downloader.start() self.downloader.shutdown() diff --git a/variable_to_interpolated.py b/variable_to_interpolated.py index 6ea61a51..b554755f 100644 --- a/variable_to_interpolated.py +++ b/variable_to_interpolated.py @@ -1,4 +1,4 @@ -"""Script to """ +"""Script to move a normal variable to make it look as it is an interpolated one""" import shutil import os @@ -19,3 +19,4 @@ for startdate in os.listdir(path): member_path = os.path.join(var_path, member) new_member_path = os.path.join(new_var_path, member) shutil.move(member_path, new_member_path) + -- GitLab From 0a9e2584075eb8ac819781fe588998ebb9a29f70 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 19 Mar 2018 12:10:30 +0100 Subject: [PATCH 06/12] Fixed codacy warnings --- variable_to_interpolated.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/variable_to_interpolated.py b/variable_to_interpolated.py index b554755f..0c4b3542 100644 --- a/variable_to_interpolated.py +++ b/variable_to_interpolated.py @@ -2,12 +2,12 @@ import shutil import os -path = '/esarchive/exp/ecearth/a0pe/cmorfiles/BSC/EC-EARTH3/a0pe/' -new_grid = 'grib' +EXP_PATH = '/esarchive/exp/ecearth/a0pe/cmorfiles/BSC/EC-EARTH3/a0pe/' +GRID_NAME = 'grib' -for startdate in os.listdir(path): - var_path = os.path.join(path, startdate, 'mon', 'atmos', 'ta') - new_var_path = os.path.join(var_path, new_grid) +for startdate in os.listdir(EXP_PATH): + var_path = os.path.join(EXP_PATH, startdate, 'mon', 'atmos', 'ta') + new_var_path = os.path.join(var_path, GRID_NAME) if not os.path.isdir(var_path): continue if not os.path.exists(new_var_path): @@ -17,6 +17,5 @@ for startdate in os.listdir(path): if not member.endswith('i1p1'): continue member_path = os.path.join(var_path, member) - new_member_path = os.path.join(new_var_path, member) - shutil.move(member_path, new_member_path) - + destiny_member_path = os.path.join(new_var_path, member) + shutil.move(member_path, destiny_member_path) -- GitLab From 9a310fec53ac72990fc7c58ca57c801a4b356ece Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Mar 2018 15:39:10 +0100 Subject: [PATCH 07/12] Improvements for PRIMAVERA CMOR unpacking --- diags.conf | 28 +++++++++++++++------------- earthdiagnostics/cmormanager.py | 5 ++++- earthdiagnostics/utils.py | 1 + launch_diags.sh | 2 +- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/diags.conf b/diags.conf index cddd6d9f..727a2353 100644 --- a/diags.conf +++ b/diags.conf @@ -1,26 +1,26 @@ [DIAGNOSTICS] # Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) -DATA_ADAPTOR = OBSRECON +DATA_ADAPTOR = CMOR # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use DATA_DIR = /esarchive # Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) -DATA_TYPE = recon +DATA_TYPE = exp # CMORization type to use. Important also for THREDDS as it affects variable name conventions. # Options: SPECS (default), PRIMAVERA, CMIP6 -DATA_CONVENTION = SPECS +DATA_CONVENTION = PRIMAVERA # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = interpcdo,atmos,sfcWind,r240x121,bilinear,False,s2s,False +DIAGS = # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = weekly +FREQUENCY = mon # Path to CDFTOOLS binaries CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir @@ -31,6 +31,8 @@ MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False FORCE = False +FORCE_UNTAR = True +CHUNKS = 2 3 4 5 6 # If true, CMORizes ocean files. Default = True OCEAN_FILES = False FILTER_FILES = @@ -64,7 +66,7 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, # PHYSICS_DESCRIPTION = # ASSOCIATED_MODEL = # SOURCE = EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 -VERSION = v20170705 +VERSION = v20180308 ACTIVITY = CMIP #ACTIVITY = CMIP_ece2cmor @@ -73,9 +75,9 @@ SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = ecmwf -MODEL = erainterim -NAME = wekkly_means +INSTITUTE = EC-Earth-Consortium +MODEL = EC-Earth3-HR +NAME = piControl # Model version: Available versions MODEL_VERSION = # Atmospheric output timestep in hours @@ -91,14 +93,14 @@ OCEAN_TIMESTEP = 3 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = mswep +EXPID = a0xi #STARTDATES = {19970101,20161231,1D} -STARTDATES = {19940101,19961231,1D} +STARTDATES = 19500101 #STARTDATES = 19970322 20010425 MEMBERS = 0 MEMBER_DIGITS = 1 -CHUNK_SIZE = 1 -CHUNKS = 1 +CHUNK_SIZE = 12 +CHUNKS = 7 # CHUNKS = 1 [REPORT] diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 1445a00f..7cb0628a 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -602,7 +602,7 @@ class CMORManager(DataManager): filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar') if len(filepaths) > 0: Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) - Utils.untar(filepaths, self.cmor_path) + Utils.untar(filepaths, os.path.join(self.cmor_path, 'cmorfiles')) self._correct_paths(startdate) self.create_links(startdate, member) return True @@ -692,8 +692,10 @@ class CMORManager(DataManager): def _create_links_cmip5(self, member_str, path): for freq in os.listdir(path): + Log.debug('Creating links for frequency {0}', freq) frequency = Frequency.parse(freq) for domain in os.listdir(os.path.join(path, freq)): + Log.debug('Creating links for domain {0}', domain) for var in os.listdir(os.path.join(path, freq, domain)): for member in os.listdir(os.path.join(path, freq, domain, var)): if member_str is not None and member_str != member: @@ -713,6 +715,7 @@ class CMORManager(DataManager): for table in os.listdir(os.path.join(path, member)): frequency = self.variable_list.tables[table].frequency domain = None + Log.debug('Creating links for table {0}', table) for var in os.listdir(os.path.join(path, member, table)): for grid in os.listdir(os.path.join(path, member, table, var)): if member_str is not None and member_str != member: diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index f89df7d5..08eae55f 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -808,6 +808,7 @@ class Utils(object): if os.path.exists(os.path.join(destiny_path, file_compressed.name)): os.remove(os.path.join(destiny_path, file_compressed.name)) tar.extract(file_compressed, destiny_path) + Log.debug('File {0} extracted', file_compressed) tar.close() @staticmethod diff --git a/launch_diags.sh b/launch_diags.sh index 1ff20d77..00cd9290 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -7,7 +7,7 @@ -PATH_TO_CONF_FILE=~/earthdiags_confs/ruben/diags_a0dc_DEBUG.conf +PATH_TO_CONF_FILE=~jvegas/PycharmProjects/earthdiagnostics/diags.conf PATH_TO_DIAGNOSTICS=~jvegas/PycharmProjects/earthdiagnostics PATH_TO_CONDAENV=/home/Earth/jvegas/.conda/envs/earthdiagnostics3/ -- GitLab From 5a10344cb7cdaddd91ef302f9c582227ec51afc5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Mar 2018 15:45:41 +0100 Subject: [PATCH 08/12] Cleaner tar log --- earthdiagnostics/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 08eae55f..52e9c29d 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -808,7 +808,7 @@ class Utils(object): if os.path.exists(os.path.join(destiny_path, file_compressed.name)): os.remove(os.path.join(destiny_path, file_compressed.name)) tar.extract(file_compressed, destiny_path) - Log.debug('File {0} extracted', file_compressed) + Log.debug('File {0} extracted', file_compressed.name) tar.close() @staticmethod -- GitLab From abc402c5c22928526105153f984c0465a43741ac Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 4 Apr 2018 16:46:06 +0200 Subject: [PATCH 09/12] Fixed bug in download priorities --- earthdiagnostics/diagnostic.py | 5 +++-- earthdiagnostics/utils.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 20d41ac4..8a4d4398 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -427,8 +427,9 @@ class Diagnostic(Publisher): int """ - return len([request.storage_status != StorageStatus.READY or request.local_status != LocalStatus.READY - for request in self._requests]) + return len([request for request in self._requests + if request.storage_status != StorageStatus.READY or + request.local_status != LocalStatus.READY]) def _different_type(self, other): return type(self) is not type(other) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 52e9c29d..632debe4 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -808,7 +808,7 @@ class Utils(object): if os.path.exists(os.path.join(destiny_path, file_compressed.name)): os.remove(os.path.join(destiny_path, file_compressed.name)) tar.extract(file_compressed, destiny_path) - Log.debug('File {0} extracted', file_compressed.name) + Log.debug('File {0} extracted', os.path.basename(file_compressed.name)) tar.close() @staticmethod -- GitLab From dcceecf8ea0ad9548f0197598fd01c40535f5188 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 4 Apr 2018 16:48:37 +0200 Subject: [PATCH 10/12] Updated cmor tables --- earthdiagnostics/cmor_tables/cmip6 | 2 +- earthdiagnostics/cmor_tables/primavera | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/cmor_tables/cmip6 b/earthdiagnostics/cmor_tables/cmip6 index 78eb04bd..a6375fe6 160000 --- a/earthdiagnostics/cmor_tables/cmip6 +++ b/earthdiagnostics/cmor_tables/cmip6 @@ -1 +1 @@ -Subproject commit 78eb04bd32dcc398323b21b1cb0636b2f07ffc68 +Subproject commit a6375fe65de3cbbf936303c61a819fd96b8df2ab diff --git a/earthdiagnostics/cmor_tables/primavera b/earthdiagnostics/cmor_tables/primavera index f2507377..5236593c 160000 --- a/earthdiagnostics/cmor_tables/primavera +++ b/earthdiagnostics/cmor_tables/primavera @@ -1 +1 @@ -Subproject commit f25073770569ea73540d09a058637128db024c55 +Subproject commit 5236593cf11fd3175dd1e48d201cd507bd2e37ac -- GitLab From 07cdb9e613715099c7c0d7563806a08a42a91781 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 4 Apr 2018 16:49:00 +0200 Subject: [PATCH 11/12] Bumped version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7bd1094b..40020fcf 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc4 +3.0.0rc5 -- GitLab From 1e7dd790215a434154f6ca122a9d0711040e1ee0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 4 Apr 2018 17:13:20 +0200 Subject: [PATCH 12/12] Fixed region mean test --- diags.conf | 33 +++++++++++++++-------------- test/unit/ocean/test_region_mean.py | 6 ++++++ 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/diags.conf b/diags.conf index 727a2353..92216b57 100644 --- a/diags.conf +++ b/diags.conf @@ -9,7 +9,7 @@ DATA_DIR = /esarchive DATA_TYPE = exp # CMORization type to use. Important also for THREDDS as it affects variable name conventions. # Options: SPECS (default), PRIMAVERA, CMIP6 -DATA_CONVENTION = PRIMAVERA +DATA_CONVENTION = SPECS # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ @@ -30,17 +30,18 @@ MAX_CORES = 1 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False -FORCE = False -FORCE_UNTAR = True -CHUNKS = 2 3 4 5 6 +FORCE = True +USE_GRIB = False +FORCE_UNTAR = False +#CHUNKS = # If true, CMORizes ocean files. Default = True OCEAN_FILES = False FILTER_FILES = # If true, CMORizes atmosphere files. Default = True -ATMOSPHERE_FILES = False +ATMOSPHERE_FILES = True # You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var VARIABLE_LIST = -CHUNK_LIST = 1 +CHUNK_LIST = # Variables to be CMORized from the grib atmospheric files, separated by comma. # You can also specify the levels to extract using the following syntax @@ -66,7 +67,7 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, # PHYSICS_DESCRIPTION = # ASSOCIATED_MODEL = # SOURCE = EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 -VERSION = v20180308 +VERSION = ACTIVITY = CMIP #ACTIVITY = CMIP_ece2cmor @@ -75,15 +76,15 @@ SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = EC-Earth-Consortium -MODEL = EC-Earth3-HR -NAME = piControl +INSTITUTE = BSC +MODEL = EC-EARTH3 +#NAME = # Model version: Available versions MODEL_VERSION = # Atmospheric output timestep in hours ATMOS_TIMESTEP = 6 # Ocean output timestep in hours -OCEAN_TIMESTEP = 3 +OCEAN_TIMESTEP = 6 # For those who use Autosubmit, this will be easy # EXPID is the unique identifier of the experiment. @@ -93,14 +94,14 @@ OCEAN_TIMESTEP = 3 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = a0xi +EXPID = m02r #STARTDATES = {19970101,20161231,1D} -STARTDATES = 19500101 +STARTDATES = {19930501,20090501,1Y} #STARTDATES = 19970322 20010425 -MEMBERS = 0 +MEMBERS = 0-9 MEMBER_DIGITS = 1 -CHUNK_SIZE = 12 -CHUNKS = 7 +CHUNK_SIZE = 4 +CHUNKS = 1 # CHUNKS = 1 [REPORT] diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index 313456fe..8a5900d6 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -8,6 +8,7 @@ from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.ocean.regionmean import RegionMean +from earthdiagnostics.utils import TempFile class TestRegionMean(TestCase): @@ -22,7 +23,12 @@ class TestRegionMean(TestCase): raise DiagnosticOptionError return value + @staticmethod + def fake_get(): + return 'tempfile' + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + @patch.object(TempFile, 'get', fake_get) def test_generate_jobs(self): box = Box() -- GitLab