diff --git a/VERSION b/VERSION index ea15350b27ab8756f6f8668b3d64d9ff70489af7..cdea6da7350a6da094f36ba34c61058b3e7cba84 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0b53 +3.0.0b54 diff --git a/diags.conf b/diags.conf index 7b966d685ad9ec84b1fd7fa9293279d228b862c4..2bdd1cc518d1e523069d8c04a64c495e7fd8df8c 100644 --- a/diags.conf +++ b/diags.conf @@ -106,11 +106,11 @@ STC = mocarea,0,25,0,200,Pac mocarea,-25,0,0,200,Pac mocarea,0,25,0,200,Atl moca HEAT_SAL_MXL = mlotstsc mlotsthc LMSALC = vertmeanmeters,so,300,5400 USALC = vertmeanmeters,so,0,300 -OHC = ohc,glob,0,1,10 +OHC = ohc,glob,0,0,2000 XOHC = ohc,glob,1,0,0 -LOHC = ohc,glob,0,23,46 -MOHC = ohc,glob,0,18,22 -UOHC = ohc,glob,0,1,17 +LOHC = ohc,glob,0,700,2000 +MOHC = ohc,glob,0,300,700 +UOHC = ohc,glob,0,0,300 OHC_SPECIFIED_LAYER = ohclayer,0,300 ohclayer,300,800 3DTEMP = interp,thetao 3DSAL = interp,so diff --git a/doc/source/conf.py b/doc/source/conf.py index eef27fe9c310774501b63ebc62b3fb52e33ca03f..0a95293ed37b4cc2ac37c7dd34c37e5c18fd2656 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b53' +release = '3.0.0b54' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst index 6f42fbb5bcc184c673178657b08ee6853f982735..b26b6a26cf6fd53f77763ceed62ed1df4705fbd5 100644 --- a/doc/source/diagnostic_list.rst +++ b/doc/source/diagnostic_list.rst @@ -392,10 +392,6 @@ Options: 5. Basin = 'Global': Basin to calculate the heat content on. - options_available = (DiagnosticIntOption('min_depth'), - DiagnosticIntOption('max_depth'), - DiagnosticBasinOption('basin', Basins.Global)) - interpolate ~~~~~~~~~~~ diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index 02f26274a6e001dc73d8740bf4025b9b1053b30e..81b334839c751dec6b6610280287ea7a6d91df25 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 8307cb14838cb8eabc48c50988827867da082816..46e9500dd1ca310d4986995a4c19a94c6a8c994d 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -80,7 +80,9 @@ class Cmorizer(object): count = 1 for tarfile in tar_files: - if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), ModelingRealms.ocean): + if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), (ModelingRealms.ocean, + ModelingRealms.seaIce, + ModelingRealms.ocnBgchem)): Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) count += 1 continue @@ -195,7 +197,7 @@ class Cmorizer(object): if len(tar_files) == 0: Log.error('MMA files not found in {0}'.format(self.original_files_path)) for tarfile in tar_files: - if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), ModelingRealms.atmos): + if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), (ModelingRealms.atmos,)): Log.info('No need to unpack file {0}/{1}'.format(count, len(tar_files))) count += 1 continue @@ -218,7 +220,7 @@ class Cmorizer(object): while os.path.exists(self.get_original_grib_path(chunk_start, 'GG')) or \ os.path.exists(self.get_original_grib_path(chunk_start, 'SH')): - if self.cmorization_required(chunk, ModelingRealms.atmos): + if self.cmorization_required(chunk, (ModelingRealms.atmos,)): chunk_end = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', self.experiment.calendar) chunk_end = previous_day(chunk_end, self.experiment.calendar) Log.info('CMORizing chunk {0}-{1}', date2str(chunk_start), date2str(chunk_end)) @@ -410,7 +412,7 @@ class Cmorizer(object): if alias.basin is None: region = None else: - region = alias.basin.fullname + region = alias.basin.name date_str = self.get_date_str(file_path) if date_str is None: diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index c34ffba8444642edfb89542e833489eec1044a32..f426e0f25550339b0c98b7128cb1c63d8a7bb4a6 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -158,7 +158,7 @@ class CMORManager(DataManager): else: grid = '' - file_name = '{0}_{1}_{2}_{3}_S{4}-{5}{6}{7}'.format(var, cmor_table.name, self.experiment.model, + file_name = '{0}_{1}_{2}_{3}_S{4}_{5}{6}{7}'.format(var, cmor_table.name, self.experiment.model, self.experiment.experiment_name, startdate, self._get_member_str(member), grid, time_bound) else: @@ -375,11 +375,14 @@ class CMORManager(DataManager): if not self._unpack_cmor_files(startdate, member): self._cmorize_member(startdate, member) - def is_cmorized(self, startdate, member, chunk, domain): - identifier = (startdate, member, chunk, domain.name) - if identifier not in self._dic_cmorized: - self._dic_cmorized[identifier] = self._is_cmorized(startdate, member, chunk, domain) - return self._dic_cmorized[identifier] + def is_cmorized(self, startdate, member, chunk, domains): + for domain in domains: + identifier = (startdate, member, chunk, domain.name) + if identifier not in self._dic_cmorized: + self._dic_cmorized[identifier] = self._is_cmorized(startdate, member, chunk, domain) + if self._dic_cmorized[identifier]: + return True + return False def _is_cmorized(self, startdate, member, chunk, domain): startdate_path = self._get_startdate_path(startdate) @@ -413,14 +416,19 @@ class CMORManager(DataManager): cmorized = False if not self.config.cmor.force_untar: - while self.is_cmorized(startdate, member, chunk, ModelingRealms.ocean) or\ - self.is_cmorized(startdate, member, chunk, ModelingRealms.atmos): + while self.is_cmorized(startdate, member, chunk, + (ModelingRealms.ocean, ModelingRealms.seaIce, + ModelingRealms.atmos, ModelingRealms.ocnBgchem)): chunk += 1 while self._unpack_chunk(startdate, member, chunk): chunk += 1 cmorized = True + if self.experiment.num_chunks <= chunk: + cmorized = True + if cmorized: + Log.info('Startdate {0} member {1} ready', startdate, member) return cmorized def _unpack_chunk(self, startdate, member, chunk): diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 3119171d48a09cb2e8a75c0eb637858d744fd165..2ad86e0f51c5174caa7b64e07b88621e753d2dfa 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -28,6 +28,14 @@ class Config(object): "Scratch folder path" self.scratch_dir = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_DIR') "Scratch folder path" + self.use_ramdisk = parser.get_bool_option('DIAGNOSTICS', 'USE_RAMDISK', False) + "If True, the scratch dir is created as a ram disk" + self.auto_clean = parser.get_bool_option('DIAGNOSTICS', 'AUTO_CLEAN', True) + "If True, the scratch dir is removed after finishing" + if not self.auto_clean and self.use_ramdisk: + Log.warning('RAM disk scratch dir is always automatically cleaned.') + self.auto_clean = True + self.scratch_masks = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_MASKS', '/scratch/Earth/ocean_masks') "Common scratch folder for masks" self.data_dir = parser.get_path_option('DIAGNOSTICS', 'DATA_DIR') @@ -36,6 +44,16 @@ class Config(object): "Data type (experiment, observation or reconstruction)" self.con_files = parser.get_path_option('DIAGNOSTICS', 'CON_FILES') "Mask and meshes folder path" + self.mesh_mask = parser.get_path_option('DIAGNOSTICS', 'MESH_MASK', '') + "Custom mesh mask file to use" + self.new_mask_glo = parser.get_path_option('DIAGNOSTICS', 'NEW_MASK_GLO', '') + "Custom new mask glo file to use" + self.mask_regions = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS', '') + "Custom mask regions file to use" + self.mask_regions_3d = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS_3D', '') + "Custom mask regions 3D file to use" + + self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', ('specs', 'primavera', 'cmip6'), 'specs', ignore_case=True) VariableManager().load_variables(self.data_convention) diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 971e36a4d26ec6531517c4d94025ec99fe6b8ce7..d0ba76adb46cd364e792793eef02b88d47193e2a 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -2,204 +2,153 @@ """ Contains the enumeration-like classes used by the diagnostics """ -from earthdiagnostics.box import Box +import netCDF4 +from singleton import SingletonType class Basin(object): """ Class representing a given basin - :param shortname: sfull basin's name - :type shortname: str - :param fullname: full basin's name - :type fullname: str - :param box: box defining the basin - :type box: Box + :param name: full basin's name + :type name: str """ - def __init__(self, shortname, fullname, box=None): - self._shortname = shortname - self._fullname = fullname - if box is None: - box = Box() - - self.box = box - """ - Box representing the basin - """ + def __init__(self, name): + self._name = name def __eq__(self, other): - if self.shortname != other.shortname or self.fullname != other.fullname: + if self.name != other.name: return False return True - def __str__(self): - return self._fullname + def __ne__(self, other): + return not self == other - @property - def shortname(self): - """ - Basin's short name - :rtype: str - """ - return self._shortname + def __str__(self): + return self._name @property - def fullname(self): + def name(self): """ Basin's full name :rtype: str """ - return self._fullname + return self._name class Basins(object): """ Predefined basins """ - Global = Basin('glob', 'Global_Ocean') - """ Global ocean """ - - Atlantic = Basin('Atl', 'Atlantic_Ocean') - """ Atlantic ocean """ - NorthAtlantic = Basin('NAtl', 'North_Atlantic_Ocean') - """ North Atlantic Ocean """ - TropicalAtlantic = Basin('TAtl', 'Tropical_Atlantic_Ocean') - """ Tropical Atlantic Ocean """ - - Pacific = Basin('Pac', 'Pacific_Ocean') - """ Pacific Ocean """ - NorthPacific = Basin('NPac', 'North_Pacific_Ocean') - """ North Pacific Ocean """ - TropicalPacific = Basin('TPac', 'Tropical_Pacific_Ocean') - """ Tropical Pacific Ocean """ - IndoPacific = Basin('IndPac', 'Indo_Pacific_Ocean') - """ Indo Pacific Ocean """ - - Indian = Basin('Ind', 'Indian_Ocean') - """ Indian Ocean """ - TropicalIndian = Basin('TInd', 'Tropical_Indian_Ocean') - """ Tropical Indian Ocean """ - - Antarctic = Basin('Anta', 'Antarctic_Ocean') - """ Antarctic Ocean """ - AntarcticAtlantic = Basin('AntaAtl', 'Antarctic_Atlantic_Sector') - """ Antarctic Ocean Atlantic Sector """ - AntarcticIndian = Basin('AntaInd', 'Antarctic_Indian_Sector') - """ Antarctic Ocean Indian Sector""" - - Arctic = Basin('Arct', 'Arctic_Ocean') - """ Arctic Ocean """ - ArcticNorthAtlantic = Basin('ArctNAtl', 'Arctic_Ocean_North_Atlantic') - """ Arctic Ocean North Atlantic""" - ArcticMarginalSeas = Basin('ArctMarg', 'Arctic_Marginal_Seas') - """ Arctic Ocean """ - - # ArctOcn - - Baffin = Basin('Baffin', 'Baffin') - " Baffin " - Baffin_Bay = Basin('BaffBay', 'Baffin_Bay') - " Baffin_Bay " - Baltic_Sea = Basin('Baltic', 'Baltic_Sea') - " Baltic_Sea " - BarKara = Basin('BarKara', 'BarKara') - " BarKara " - Barents_Sea = Basin('Barents', 'Barents_Sea') - " Barents_Sea " - Beaufort_Chukchi_Sea = Basin('BeaufortChukchi', 'Beaufort_Chukchi_Sea') - " Beaufort_Chukchi_Sea " - Beaufort_Sea = Basin('Beaufort', 'Beaufort_Sea') - " Beaufort_Sea " - Bellingshausen_Sea = Basin('Bellingshausen_', 'Bellingshausen_Sea') - " Bellingshausen_Sea " - Bering = Basin('Bering', 'Bering') - " Bering " - Bering_Strait = Basin('BeringStr', 'Bering_Strait') - " Bering_Strait " - CanArch = Basin('CanArch', 'CanArch') - " CanArch " - Canadian_Waters = Basin('Canadian', 'Canadian_Waters') - " Canadian_Waters " - Caspian_Sea = Basin('Caspian', 'Caspian_Sea') - " Caspian_Sea " - Central_Arctic = Basin('CArct', 'Central_Arctic') - " Central_Arctic " - Chukchi_Sea = Basin('Chukchi', 'Chukchi_Sea') - " Chukchi_Sea " - East_Siberian_Sea = Basin('ESiberian', 'East_Siberian_Sea') - " East_Siberian_Sea " - Eastern_Central_Arctic = Basin('ECArct', 'Eastern_Central_Arctic') - " Eastern_Central_Arctic " - Fram_Strait = Basin('Fram', 'Fram_Strait') - " Fram_Strait " - Global_Ocean = Basin('Global', 'Global_Ocean') - " Global_Ocean " - Greenland_Sea = Basin('Greenland', 'Greenland_Sea') - " Greenland_Sea " - Grnland = Basin('Grnland', 'Grnland') - " Grnland " - Hudson = Basin('Hudson', 'Hudson') - " Hudson " - Icelandic_Sea = Basin('Iceland', 'Icelandic_Sea') - " Icelandic_Sea " - Irminger_Sea = Basin('Irminger', 'Irminger_Sea') - " Irminger_Sea " - Kara_Gate_Strait = Basin('KaraGate', 'Kara_Gate_Strait') - " Kara_Gate_Strait " - Kara_Sea = Basin('Kara', 'Kara_Sea') - " Kara_Sea " - Labrador_Sea = Basin('Labrador', 'Labrador_Sea') - " Labrador_Sea " - Laptev_East_Siberian_Chukchi_Seas = Basin('LaptevESiberianChukchi', 'Laptev_East_Siberian_Chukchi_Seas') - " Laptev_East_Siberian_Chukchi_Seas " - Laptev_East_Siberian_Seas = Basin('LaptevESiberian', 'Laptev_East_Siberian_Seas') - " Laptev_East_Siberian_Seas " - Laptev_Sea = Basin('Laptev', 'Laptev_Sea') - " Laptev_Sea " - Lincoln_Sea = Basin('Lincoln', 'Lincoln_Sea') - " Lincoln_Sea " - Mediterranean_Sea = Basin('Medit', 'Mediterranean_Sea') - " Mediterranean_Sea " - Nares_Strait = Basin('Nares', 'Nares_Strait') - " Nares_Strait " - Nordic_Barents_Seas = Basin('NordicBarents', 'Nordic_Barents_Seas') - " Nordic_Barents_Seas " - Nordic_Seas = Basin('Nordic', 'Nordic_Seas') - " Nordic_Seas " - NorthWest_Passage = Basin('NWPass', 'NorthWest_Passage') - " NorthWest_Passage " - North_Atlantic_Arctic = Basin('North_Atlantic_Arctic', 'North_Atlantic_Arctic') - " North_Atlantic_Arctic " - North_Hemisphere_Ocean = Basin('NHem', 'North_Hemisphere_Ocean') - " North_Hemisphere_Ocean " - Norwegian_Sea = Basin('Norwe', 'Norwegian_Sea') - " Norwegian_Sea " - Okhotsk = Basin('Okhotsk', 'Okhotsk') - " Okhotsk " - OpenOcean = Basin('OpenOcean', 'OpenOcean') - " OpenOcean " - Ross_Sea = Basin('Ross', 'Ross_Sea') - " Ross_Sea " - Serreze_Arctic = Basin('SerArc', 'Serreze_Arctic') - " Serreze_Arctic " - Southern_Hemisphere = Basin('SHem', 'Southern_Hemisphere') - " Southern_Hemisphere " - StLawr = Basin('StLawr', 'StLawr') - " StLawr " - Subpolar_Gyre = Basin('Subpolar', 'Subpolar_Gyre') - " Subpolar_Gyre " - TotalArc = Basin('TotalArc', 'TotalArc') - " TotalArc " - Vilkitsky_Strait = Basin('Vilkitsky', 'Vilkitsky_Strait') - " Vilkitsky_Strait " - Weddell_Sea = Basin('Weddell', 'Weddell_Sea') - " Weddell_Sea " - Western_Central_Arctic = Basin('Western_Central_Arctic', 'Western_Central_Arctic') - " Western_Central_Arctic " - - @classmethod - def parse(cls, basin): + + __metaclass__ = SingletonType + + def __init__(self): + + self.aliases = { + 'Atlantic_Ocean': ('atl', 'atlantic'), + 'North_Atlantic_Ocean': ('natl', 'north_atlantic'), + 'Tropical_Atlantic_Ocean': ('tatl', 'tropical_atlantic'), + + 'Pacific_Ocean': ('pac', 'pacific'), + 'North_Pacific_Ocean': ('npac', 'north_pacific'), + 'Tropical_Pacific_Ocean': ('tpac', 'tropical_pacific'), + 'Indo_Pacific_Ocean': ('indpac', 'indo_pacific'), + + 'Indian_Ocean': ('ind', 'indian'), + 'Tropical_Indian_Ocean': ('tind', 'tropical_indian'), + + 'Antarctic_Ocean': ('anta', 'antarctiv'), + 'Antarctic_Atlantic_Sector': ('antaatl', 'antarctic_atlantic'), + 'Antarctic_Indian_Sector': ('antaind', 'antarctic_indian'), + + 'Arctic_Ocean': ('arct', 'arctic'), + 'Arctic_Ocean_North_Atlantic': ('arctnatl', 'artic_north_atlantic'), + 'Arctic_Marginal_Seas': ('arctmarg', 'arctic_marginal'), + + 'Baffin': ('Baffin',), + 'Baffin_Bay': ('BaffBay',), + 'Baltic_Sea': ('Baltic',), + 'BarKara': ('BarKara',), + 'Barents_Sea': ('Barents',), + 'Beaufort_Chukchi_Sea': ('BeaufortChukchi',), + 'Beaufort_Sea': ('Beaufort',), + 'Bellingshausen_Sea': ('Bellingshausen',), + 'Bering': ('Bering',), + 'Bering_Strait': ('BeringStr',), + 'CanArch': ('CanArch',), + 'Canadian_Waters': ('Canadian',), + 'Caspian_Sea': ('Caspian',), + 'Central_Arctic': ('CArct',), + 'Chukchi_Sea': ('Chukchi',), + 'East_Siberian_Sea': ('ESiberian',), + 'Eastern_Central_Arctic': ('ECArct',), + 'Fram_Strait': ('Fram',), + 'Greenland_Sea': ('Greenland',), + 'Grnland': ('Grnland',), + 'Hudson': ('Hudson',), + 'Icelandic_Sea': ('Iceland',), + 'Irminger_Sea': ('Irminger',), + 'Kara_Gate_Strait': ('KaraGate',), + 'Kara_Sea': ('Kara',), + 'Labrador_Sea': ('Labrador',), + 'Laptev_East_Siberian_Chukchi_Seas': ('LaptevESiberianChukchi',), + 'Laptev_East_Siberian_Seas': ('LaptevESiberian',), + 'Laptev_Sea': ('Laptev',), + 'Lincoln_Sea': ('Lincoln',), + 'Mediterranean_Sea': ('Medit',), + 'Nares_Strait': ('Nares',), + 'Nordic_Barents_Seas': ('NordicBarents',), + 'Nordic_Seas': ('Nordic',), + 'NorthWest_Passage': ('NWPass',), + 'North_Atlantic-Arctic': ('North_Atlantic-Arctic',), + 'North_Hemisphere_Ocean': ('NHem',), + 'Norwegian_Sea': ('Norwegian',), + 'Okhotsk': ('Okhotsk',), + 'OpenOcean': ('OpenOcean',), + 'Ross_Sea': ('Ross',), + 'Serreze_Arctic': ('SerArc',), + 'Southern_Hemisphere': ('SHem',), + 'StLawr': ('StLawr',), + 'Subpolar_Gyre': ('Subpolar_Gyre',), + 'TotalArc': ('TotalArc',), + 'Vilkitsky_Strait': ('Vilkitsky_Strait',), + 'Weddell_Sea': ('Weddell_Sea',), + 'Western_Central_Arctic': ('Western_Central_Arctic',), + } + + self.Global = Basin('Global') + self._known_aliases = {} + + def get_available_basins(self, handler): + """ + + :param handler: + :type handler: netCDF4.Dataset + """ + basin_names = handler.variables.keys() + + ignored_names = ('lat', 'lon', 'i', 'j', 'time', 'lev') + + for basin in basin_names: + if basin in ignored_names: + continue + basin_object = Basin(basin) + setattr(self, basin, basin_object) + self._add_alias(basin, basin_object) + try: + for alias in self.aliases[basin]: + self._add_alias(alias, basin_object) + except KeyError: + pass + + self._add_alias('glob', self.Global) + + def _add_alias(self, basin, basin_object): + self._known_aliases[basin.lower()] = basin_object + + def parse(self, basin): """ Return the basin matching the given name. If the parameter basin is a Basin instance, directly returns the same instance. This bahaviour is intended to facilitate the development of methods that can either accept a name @@ -212,15 +161,11 @@ class Basins(object): """ if isinstance(basin, Basin): return basin - for name in cls.__dict__.keys(): - if name.startswith('_'): - continue - # noinspection PyCallByClass - value = cls.__getattribute__(cls, name) - if isinstance(value, Basin): - if basin.lower() in [value.shortname.lower(), value.fullname.lower()]: - return value - return None + basin = basin.lower() + try: + return self._known_aliases[basin] + except KeyError: + return None class Models(object): diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index f572592bca90ed35c9e72887c0f9f99f47ed7d2b..f51ddfd84b10a03a1c295b299902332fbecda90f 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -295,6 +295,7 @@ class NetCDFFile(object): self._rename_coordinate_variables() Utils.move_file(self.local_file, self.remote_file) + Utils.give_group_write_permissions(self.remote_file) def _prepare_region(self): if not self.region: @@ -304,12 +305,12 @@ class NetCDFFile(object): else: self._update_var_with_region_data() self._correct_metadata() - Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('-O --fix_rec_dmn region',)) + Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('--fix_rec_dmn region',)) def _update_var_with_region_data(self): temp = TempFile.get() shutil.copyfile(self.remote_file, temp) - Utils.nco.ncks(input=temp, output=temp, options=('-O --mk_rec_dmn region',)) + Utils.nco.ncks(input=temp, output=temp, options=('--mk_rec_dmn region',)) handler = Utils.openCdf(temp) handler_send = Utils.openCdf(self.local_file) value = handler_send.variables[self.var][:] @@ -338,7 +339,7 @@ class NetCDFFile(object): value = original_var[:] new_var[..., 0] = value handler.close() - Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('-O -x -v {0}'.format(self.var),)) + Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('-x -v {0}'.format(self.var),)) Utils.rename_variable(self.local_file, 'new_var', self.var) def _correct_metadata(self): @@ -364,7 +365,7 @@ class NetCDFFile(object): var_handler.short_name = self.cmor_var.short_name def _fix_values_metadata(self, var_type): - options = ['-O -a _FillValue,{0},o,{1},"1.e20"'.format(self.var, var_type.char), + options = ['-a _FillValue,{0},o,{1},"1.e20"'.format(self.var, var_type.char), '-a missingValue,{0},o,{1},"1.e20"'.format(self.var, var_type.char)] if self.cmor_var.valid_min != '': options.append('-a valid_min,{0},o,{1},"{2}" '.format(self.var, var_type.char, self.cmor_var.valid_min)) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 6d8036956f5e18758622b92102539c750054fb8e..5e4756941040792ac6bbd561ecb8ad8438d82f7d 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -84,7 +84,7 @@ class Diagnostic(object): :return: """ if isinstance(region, Basin): - region = region.fullname + region = region.name self.data_manager.send_file(filetosend, domain, var, startdate, member, chunk, grid, region, box, rename_var, frequency, year, date_str, move_old, diagnostic=self, vartype=vartype) @@ -244,7 +244,7 @@ class DiagnosticFrequencyOption(DiagnosticOption): class DiagnosticBasinOption(DiagnosticOption): def parse(self, option_value): - return Basins.parse(self.check_default(option_value)) + return Basins().parse(self.check_default(option_value)) class DiagnosticComplexStrOption(DiagnosticOption): diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 1742e90d9d0393fc617169550bd4db4edbc3f78b..4c166fd02d1a8587ff6a3825d8a17bb81f530b5e 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -12,6 +12,7 @@ import os from bscearth.utils.date import * import bscearth.utils.path +from earthdiagnostics.constants import Basins from earthdiagnostics.config import Config from earthdiagnostics.cmormanager import CMORManager from earthdiagnostics.threddsmanager import THREDDSManager @@ -23,6 +24,7 @@ from earthdiagnostics.general import * from earthdiagnostics.statistics import * from earthdiagnostics.variable import VariableManager from earthdiagnostics.diagnostic import DiagnosticOptionError +import tempfile class EarthDiags(object): @@ -143,11 +145,11 @@ class EarthDiags(object): """ self.had_errors = False Log.debug('Using netCDF version {0}', netCDF4.getlibversion()) - if not os.path.exists(self.config.scratch_dir): - os.makedirs(self.config.scratch_dir) - os.chdir(self.config.scratch_dir) + + self._prepare_scratch_dir() self._prepare_mesh_files() + self._initialize_basins() self._register_diagnostics() @@ -180,8 +182,32 @@ class EarthDiags(object): Log.result("Elapsed time: {0}\n", finish_time - time) self.print_errors() self.print_stats() + if self.config.auto_clean: + self._remove_scratch_dir() return not self.had_errors + def _initialize_basins(self): + self._read_basins_from_file('mask_regions.nc') + self._read_basins_from_file('mask_regions.3d.nc') + + def _read_basins_from_file(self, filename): + if not os.path.isfile(filename): + return + handler = Utils.openCdf(filename) + Basins().get_available_basins(handler) + handler.close() + + + def _prepare_scratch_dir(self): + if self.config.use_ramdisk: + self._remove_scratch_dir() + tempfile.mkdtemp(dir='/dev/shm') + os.symlink(tempfile.mkdtemp(dir='/dev/shm'), self.config.scratch_dir) + else: + if not os.path.exists(self.config.scratch_dir): + os.makedirs(self.config.scratch_dir) + os.chdir(self.config.scratch_dir) + def _prepare_data_manager(self): if self.config.data_adaptor == 'CMOR': self.data_manager = CMORManager(self.config) @@ -283,11 +309,17 @@ class EarthDiags(object): def clean(self): Log.info('Removing scratch folder...') - if os.path.exists(self.config.scratch_dir): - shutil.rmtree(self.config.scratch_dir) + self._remove_scratch_dir() Log.result('Scratch folder removed') return True + def _remove_scratch_dir(self): + if os.path.islink(self.config.scratch_dir): + shutil.rmtree(os.path.realpath(self.config.scratch_dir)) + os.remove(self.config.scratch_dir) + elif os.path.isdir(self.config.scratch_dir): + shutil.rmtree(self.config.scratch_dir) + def report(self): Log.info('Looking for existing vars...') self._prepare_data_manager() @@ -388,48 +420,70 @@ class EarthDiags(object): con_files = self.config.con_files model_version = self.config.experiment.model_version restore_meshes = self.config.restore_meshes + mesh_mask = 'mesh_mask_nemo.{0}.nc'.format(model_version) new_mask_glo = 'new_maskglo.{0}.nc'.format(model_version) mask_regions = 'mask.regions.{0}.nc'.format(model_version) mask_regions_3d = 'mask.regions.3d.{0}.nc'.format(model_version) + + if self.config.mesh_mask: + mesh_mask_path = self.config.mesh_mask + else: + mesh_mask_path = os.path.join(con_files, mesh_mask) + + if self.config.new_mask_glo: + new_mask_glo_path = self.config.new_mask_glo + else: + new_mask_glo_path = os.path.join(con_files, new_mask_glo) + + if self.config.mask_regions: + mask_regions_path = self.config.mask_regions + else: + mask_regions_path = os.path.join(con_files, mask_regions) + + if self.config.mask_regions_3d: + mask_regions_3d_path = self.config.mask_regions_3d + else: + mask_regions_3d_path = os.path.join(con_files, mask_regions_3d) + if self.config.scratch_masks: Utils.create_folder_tree(self.config.scratch_masks) Utils.give_group_write_permissions(self.config.scratch_masks) - mesh_mask_path = os.path.join(self.config.scratch_masks, mesh_mask) - if self._copy_file(os.path.join(con_files, mesh_mask), mesh_mask_path, + mesh_mask_scratch_path = os.path.join(self.config.scratch_masks, mesh_mask) + if self._copy_file(mesh_mask_path, mesh_mask_scratch_path, restore_meshes): - Utils.give_group_write_permissions(mesh_mask_path) - self._link_file(mesh_mask_path, 'mesh_hgr.nc') - self._link_file(mesh_mask_path, 'mesh_zgr.nc') - self._link_file(mesh_mask_path, 'mask.nc') + Utils.give_group_write_permissions(mesh_mask_scratch_path) + self._link_file(mesh_mask_scratch_path, 'mesh_hgr.nc') + self._link_file(mesh_mask_scratch_path, 'mesh_zgr.nc') + self._link_file(mesh_mask_scratch_path, 'mask.nc') new_maskglo_scratch_path = os.path.join(self.config.scratch_masks, new_mask_glo) - if self._copy_file(os.path.join(con_files, new_mask_glo), + if self._copy_file(new_mask_glo_path, new_maskglo_scratch_path, restore_meshes): Utils.give_group_write_permissions(new_maskglo_scratch_path) self._link_file(new_maskglo_scratch_path, 'new_maskglo.nc') mask_regions_scratch_path = os.path.join(self.config.scratch_masks, mask_regions) - if self._copy_file(os.path.join(con_files, mask_regions), + if self._copy_file(mask_regions_path, mask_regions_scratch_path, restore_meshes): Utils.give_group_write_permissions(mask_regions_scratch_path) self._link_file(mask_regions_scratch_path, 'mask_regions.nc') mask_regions3d_scratch_path = os.path.join(self.config.scratch_masks, mask_regions_3d) - if self._copy_file(os.path.join(con_files, mask_regions_3d), + if self._copy_file(mask_regions_3d_path, mask_regions3d_scratch_path, restore_meshes): Utils.give_group_write_permissions(mask_regions3d_scratch_path) self._link_file(mask_regions3d_scratch_path, 'mask_regions.3d.nc') else: - self._copy_file(os.path.join(con_files, mesh_mask), 'mesh_hgr.nc', restore_meshes) + self._copy_file(mesh_mask_path, 'mesh_hgr.nc', restore_meshes) self._link_file('mesh_hgr.nc', 'mesh_zgr.nc') self._link_file('mesh_hgr.nc', 'mask.nc') - self._copy_file(os.path.join(con_files, new_mask_glo), 'new_maskglo.nc', + self._copy_file(new_mask_glo_path, 'new_maskglo.nc', restore_meshes) - self._copy_file(os.path.join(con_files, mask_regions), + self._copy_file(mask_regions_path, 'mask_regions.nc', restore_meshes) - self._copy_file(os.path.join(con_files, mask_regions_3d), + self._copy_file(mask_regions_3d_path, 'mask_regions.3d.nc', restore_meshes) Log.result('Mesh files ready!') @@ -465,6 +519,8 @@ class EarthDiags(object): Log.info('File {0} ready', destiny) + + def main(): if not EarthDiags.parse_args(): exit(1) diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index a12fd7c6d83ceab594fbfa6ce227de2da7c381f1..156be229f1a15625dd22dfe00ec3a0d82de3d328 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -74,7 +74,7 @@ class AreaMoc(Diagnostic): DiagnosticIntOption('max_lat'), DiagnosticIntOption('min_depth'), DiagnosticIntOption('max_depth'), - DiagnosticBasinOption('basin', Basins.Global)) + DiagnosticBasinOption('basin', Basins().Global)) options = cls.process_options(options, options_available) box = Box() box.min_lat = options['min_lat'] @@ -102,7 +102,7 @@ class AreaMoc(Diagnostic): nco.ncwa(input=temp, output=temp, options=('-O -a i',)) handler = Utils.openCdf(temp) - basin_index = np.where(handler.variables['basin'][:] == self.basin.fullname) + basin_index = np.where(handler.variables['basin'][:] == self.basin.name) lat_values = handler.variables['lat'][:] lat_type = handler.variables['lat'].dtype lat_units = handler.variables['lat'].units diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index c94a3a5d8067d4e3e1fe31526646374087d1f8cb..a9432167da130e3148b1e8d3f683b2b51967a603 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -85,28 +85,7 @@ class HeatContent(Diagnostic): if box.min_depth > 0 or box.max_depth > 0: - handler = Utils.openCdf('mesh_zgr.nc') - - if 'gdepw_1d' in handler.variables: - depth = handler.variables['gdepw_1d'][0, :] - else: - raise Exception('gdepw 1D variable can not be found') - handler.close() - - def find_nearest(array, value): - idx = (np.abs(array - value)).argmin() - return idx, round(array[idx]) - - if box.min_depth > 0: - min_level, box.min_depth = find_nearest(depth, box.min_depth) - else: - min_level = 1 - - if box.max_depth > 0: - max_level, box.max_depth = find_nearest(depth, box.max_depth) - else: - max_level = len(depth) - box.max_depth = round(depth[-1]) + max_level, min_level = cls._get_levels_from_meters(box) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): @@ -114,6 +93,62 @@ class HeatContent(Diagnostic): options['basin'], options['mixed_layer'], box, min_level, max_level)) return job_list + @classmethod + def _get_levels_from_meters(cls, box): + depth_t, depth_w = cls._read_level_values() + + def find_nearest(array, value): + idx = (np.abs(array - value)).argmin() + return idx + + if box.min_depth > 0: + min_level = find_nearest(depth_t, box.min_depth) + if depth_t[min_level] < box.min_depth: + min_level += 1 + else: + min_level = 0 + numlevels = len(depth_t) + + if box.max_depth > 0: + max_level = find_nearest(depth_t, box.max_depth) + if depth_t[max_level] >= box.max_depth: + max_level -= 1 + else: + max_level = numlevels - 1 + + if min_level < 0: + min_level = 0 + if max_level >= numlevels: + max_level = numlevels - 1 + + box.min_depth = round(depth_w[min_level]) + + if max_level < numlevels - 1: + box.max_depth = round(depth_w[max_level + 1]) + else: + box.max_depth = round(depth_w[-1]) + min_level += 1 + max_level += 1 + return max_level, min_level + + @classmethod + def _read_level_values(cls): + handler = Utils.openCdf('mesh_zgr.nc') + if 'gdepw_1d' in handler.variables: + depth_w = handler.variables['gdepw_1d'][0, :] + elif 'gdepw_0' in handler.variables: + depth_w = handler.variables['gdepw_0'][0, :] + else: + raise Exception('gdepw 1D variable can not be found') + if 'gdept_1d' in handler.variables: + depth_t = handler.variables['gdept_1d'][0, :] + elif 'gdept_0' in handler.variables: + depth_t = handler.variables['gdept_0'][0, :] + else: + raise Exception('gdept 1D variable can not be found') + handler.close() + return depth_t, depth_w + def compute(self): """ Runs the diagnostic @@ -127,7 +162,7 @@ class HeatContent(Diagnostic): nco.ncks(input=mlotst_file, output=temperature_file, options=('-A -v mlotst',)) para = list() - if self.box.min_depth != 0: + if self.min_level != 0: para.append('-zoom') para.append(0) para.append(0) @@ -138,15 +173,15 @@ class HeatContent(Diagnostic): if self.mxloption != 0: para.append('-mxloption') para.append(str(self.mxloption)) - if self.basin != Basins.Global: + if self.basin != Basins().Global: handler = Utils.openCdf('mask_regions.3d.nc') - if self.basin.fullname not in handler.variables: - raise Exception('Basin {0} is not defined on mask_regions.nc'.format(self.basin.fullname)) + if self.basin.name not in handler.variables: + raise Exception('Basin {0} is not defined on mask_regions.nc'.format(self.basin.name)) handler.close() para.append('-M') para.append('mask_regions.3d.nc') - para.append(self.basin.fullname) + para.append(self.basin.name) temp2 = TempFile.get() @@ -184,7 +219,7 @@ class HeatContent(Diagnostic): Utils.setminmax(heatcsum_temp, 'heatcsum') self.send_file(heatcsum_temp, ModelingRealms.ocean, 'heatcsum', self.startdate, self.member, self.chunk, - box=box_save, region=self.basin.fullname) + box=box_save, region=self.basin.name) Utils.setminmax(heatcvmean_temp, 'heatcvmean') self.send_file(heatcvmean_temp, ModelingRealms.ocean, 'heatcvmean', self.startdate, self.member, self.chunk, - box=box_save, region=self.basin.fullname) + box=box_save, region=self.basin.name) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 003cee2df309093ded35e4990b81819e33d94360..d855f04ba68100cbc7f239eb1f7fd6d1aec596b1 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -63,7 +63,7 @@ class HeatContentLayer(Diagnostic): """ options_available = (DiagnosticIntOption('min_depth'), DiagnosticIntOption('max_depth'), - DiagnosticBasinOption('basin', Basins.Global)) + DiagnosticBasinOption('basin', Basins().Global)) options = cls.process_options(options, options_available) box = Box(True) diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 10be14c5f11818cfc4436876615e01847922e85b..0f60dc788df4f926135dca0957dad3d3938cf84d 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -5,7 +5,7 @@ import threading import os from bscearth.utils.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption, \ - DiagnosticVariableOption + DiagnosticVariableListOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms @@ -84,7 +84,7 @@ class Interpolate(Diagnostic): :return: """ options_available = (DiagnosticOption('target_grid'), - DiagnosticVariableOption('variable'), + DiagnosticVariableListOption('variable'), DiagnosticDomainOption('domain', ModelingRealms.ocean), DiagnosticBoolOption('invert_lat', False), DiagnosticOption('original_grid', '')) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 8bd9c40dfe85996330d655b30da7d622570a1595..aecfd2cab3b553cf94660f3667b5ddce42280f64 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -98,7 +98,7 @@ class InterpolateCDO(Diagnostic): elif method == InterpolateCDO.BICUBIC: Utils.cdo.genbic(target_grid, input=temp, output=weights) elif method == InterpolateCDO.CONSERVATIVE: - Utils.cdo.gencon(target_grid, input=temp, output=weights) + Utils.cdo.genycon(target_grid, input=temp, output=weights) elif method == InterpolateCDO.CONSERVATIVE2: Utils.cdo.gencon2(target_grid, input=temp, output=weights) @@ -187,7 +187,7 @@ class InterpolateCDO(Diagnostic): if self.mask_oceans: - mask = Utils.get_mask(Basins.Global).astype(float) + mask = Utils.get_mask(Basins().Global).astype(float) mask[mask == 0] = np.nan var[:] = mask * var[:] handler.close() diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index 6d649edc72a12b9e4bac12cd6eabe44eb7c0f17d..291a28b8fed4a39de6426995b5e3483c7646e1cf 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -51,7 +51,7 @@ class MaxMoc(Diagnostic): def __str__(self): return 'Max moc Startdate: {0} Member: {1} Year: {2} Box: {3} ' \ - 'Basin: {4}'.format(self.startdate, self.member, self.year, self.box, self.basin.fullname) + 'Basin: {4}'.format(self.startdate, self.member, self.year, self.box, self.basin.name) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.year == other.year and \ @@ -72,7 +72,7 @@ class MaxMoc(Diagnostic): DiagnosticFloatOption('max_lat'), DiagnosticFloatOption('min_depth'), DiagnosticFloatOption('max_depth'), - DiagnosticBasinOption('basin', Basins.Global)) + DiagnosticBasinOption('basin', Basins().Global)) options = cls.process_options(options, options_available) box = Box(True) box.min_lat = options['min_lat'] @@ -106,9 +106,9 @@ class MaxMoc(Diagnostic): else: handler.close() handler = Utils.openCdf(temp) - basin_index = np.where(handler.variables['basin'][:] == self.basin.fullname) + basin_index = np.where(handler.variables['basin'][:] == self.basin.name) if len(basin_index) == 0: - raise Exception("Basin {1} is not defined in {0}", temp, self.basin.fullname) + raise Exception("Basin {1} is not defined in {0}", temp, self.basin.name) basin_index = basin_index[0][0] lev = handler.variables['lev'][:] diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 1b0a463a351d1a1107727aa741a9d49cbe5fb123..93d9c44705d3f53c58a0c4759a6d68cfefcbff15 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -80,10 +80,10 @@ class Moc(Diagnostic): Log.debug('Reformatting variables') handler = Utils.openCdf(temp) - basins_list = [Basins.Global.fullname] + basins_list = [Basins().Global.name] if 'zomsfatl' in handler.variables: - basins_list += [Basins.Atlantic.fullname, Basins.Pacific.fullname, Basins.IndoPacific.fullname, - Basins.Indian.fullname] + basins_list += [Basins().Atlantic.name, Basins().Pacific.name, Basins().IndoPacific.name, + Basins().Indian.name] handler.createDimension('basin', len(basins_list)) handler.createVariable('basin', str, 'basin') diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 630c2a1bd3e63a85fe239e863c78b4240a6a4cbb..4a2209f5cb95d900c8ab0b9bc535bacb376e4b04 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -76,7 +76,7 @@ class RegionMean(Diagnostic): options_available = (DiagnosticDomainOption('domain'), DiagnosticVariableOption('variable'), DiagnosticOption('grid_point', ''), - DiagnosticBasinOption('basin', Basins.Global), + DiagnosticBasinOption('basin', Basins().Global), DiagnosticIntOption('min_depth', 0), DiagnosticIntOption('max_depth', 0), DiagnosticBoolOption('save3D', True), @@ -111,10 +111,10 @@ class RegionMean(Diagnostic): cdfmean_options = [self.variable, self.grid_point, 0, 0, 0, 0, self.box.min_depth, self.box.max_depth] if self.variance: cdfmean_options += ['-var'] - if self.basin != Basins.Global: + if self.basin != Basins().Global: cdfmean_options.append('-M') cdfmean_options.append('mask_regions.3d.nc') - cdfmean_options.append(self.basin.fullname) + cdfmean_options.append(self.basin.name) cdftools.run('cdfmean', input=variable_file, output=mean_file, options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index f38b7c2007404c3e36c8f3bcd72fb13cda623b38..35ac972c7115ecb9bcd12af52a6ac6d3772b6247 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -2,6 +2,8 @@ import netCDF4 import os +from bscearth.utils.log import Log + from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption from earthdiagnostics.utils import Utils, TempFile # noinspection PyUnresolvedReferences @@ -10,6 +12,7 @@ import numpy as np from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.constants import Basins +from earthdiagnostics.variable import VariableManager class Siasiesiv(Diagnostic): @@ -57,7 +60,7 @@ class Siasiesiv(Diagnostic): def __str__(self): return 'Siasiesiv Startdate: {0} Member: {1} Chunk: {2} Basin: {3}'.format(self.startdate, self.member, - self.chunk, self.basin.fullname) + self.chunk, self.basin) @classmethod def generate_jobs(cls, diags, options): @@ -70,9 +73,13 @@ class Siasiesiv(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticBasinOption('basin', Basins.Global), ) + options_available = (DiagnosticBasinOption('basin', Basins().Global), ) options = cls.process_options(options, options_available) + if options['basin'] is None: + Log.error('Basin not recognized') + return () + mask = Utils.get_mask(options['basin']) job_list = list() @@ -90,16 +97,20 @@ class Siasiesiv(Diagnostic): """ Runs the diagnostic """ - sit_file = self.data_manager.get_file(ModelingRealms.seaIce, 'sit', self.startdate, self.member, self.chunk) + var_manager = VariableManager() + sit_var = var_manager.get_variable('sit').short_name + sic_var = var_manager.get_variable('sic').short_name + + sit_file = self.data_manager.get_file(ModelingRealms.seaIce, sit_var, self.startdate, self.member, self.chunk) sit_handler = Utils.openCdf(sit_file) - sit = np.asfortranarray(sit_handler.variables['sit'][:]) + sit = np.asfortranarray(sit_handler.variables[sit_var][:]) timesteps = sit_handler.dimensions['time'].size sit_handler.close() - sic_file = self.data_manager.get_file(ModelingRealms.seaIce, 'sic', self.startdate, self.member, self.chunk) + sic_file = self.data_manager.get_file(ModelingRealms.seaIce, sic_var, self.startdate, self.member, self.chunk) sic_handler = Utils.openCdf(sic_file) - Utils.convert_units(sic_handler.variables['sic'], '1.0') - sic = np.asfortranarray(sic_handler.variables['sic'][:]) + Utils.convert_units(sic_handler.variables[sic_var], '1.0') + sic = np.asfortranarray(sic_handler.variables[sic_var][:]) sic_handler.close() result = np.empty((8, timesteps)) @@ -113,23 +124,23 @@ class Siasiesiv(Diagnostic): self.send_file(self._extract_variable_and_rename(sit_file, result[4, :], 'sivols', '10^9 m3'), ModelingRealms.seaIce, 'sivols', self.startdate, self.member, self.chunk, - region=self.basin.fullname) + region=self.basin.name) self.send_file(self._extract_variable_and_rename(sit_file, result[5, :], 'siareas', '10^9 m2'), ModelingRealms.seaIce, 'siareas', self.startdate, self.member, self.chunk, - region=self.basin.fullname) + region=self.basin.name) self.send_file(self._extract_variable_and_rename(sit_file, result[7, :], 'siextents', '10^9 m2'), ModelingRealms.seaIce, 'siextents', self.startdate, self.member, self.chunk, - region=self.basin.fullname) + region=self.basin.name) self.send_file(self._extract_variable_and_rename(sit_file, result[0, :], 'sivoln', '10^9 m3'), ModelingRealms.seaIce, 'sivoln', self.startdate, self.member, self.chunk, - region=self.basin.fullname) + region=self.basin.name) self.send_file(self._extract_variable_and_rename(sit_file, result[1, :], 'siarean', '10^9 m2'), ModelingRealms.seaIce, 'siarean', self.startdate, self.member, self.chunk, - region=self.basin.fullname) + region=self.basin.name) self.send_file(self._extract_variable_and_rename(sit_file, result[3, :], 'siextentn', '10^9 m2'), ModelingRealms.seaIce, 'siextentn', self.startdate, self.member, self.chunk, - region=self.basin.fullname) + region=self.basin.name) @staticmethod def _extract_variable_and_rename(reference_file, values, cmor_name, units): diff --git a/earthdiagnostics/singleton.py b/earthdiagnostics/singleton.py new file mode 100644 index 0000000000000000000000000000000000000000..8de5f34aea0add0c801c16089e0ca1cd16f29234 --- /dev/null +++ b/earthdiagnostics/singleton.py @@ -0,0 +1,7 @@ +class SingletonType(type): + def __call__(cls, *args): + try: + return cls.__instance + except AttributeError: + cls.__instance = super(SingletonType, cls).__call__(*args) + return cls.__instance \ No newline at end of file diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 2e32e689e77743881aaef346c92f86c5ba821232..6580d78b4ed2d32a6b76319273356d8dfb532598 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -51,11 +51,15 @@ class Utils(object): :return: mask :rtype: numpy.array """ - basin = Basins.parse(basin) - if basin != Basins.Global: - mask_handler = Utils.openCdf('mask_regions.nc') - mask = mask_handler.variables[basin.fullname][:, 0, :] - mask_handler.close() + + basin = Basins().parse(basin) + if basin != Basins().Global: + try: + mask_handler = Utils.openCdf('mask_regions.nc') + mask = mask_handler.variables[basin.name][:, 0, :] + mask_handler.close() + except IOError: + raise Exception('File mask.regions.nc is required for basin {0}'.format(basin)) else: mask_handler = Utils.openCdf('mask.nc') mask = np.asfortranarray(mask_handler.variables['tmask'][0, 0, :]) @@ -224,6 +228,7 @@ class Utils(object): if dirname_path and not os.path.exists(dirname_path): try: os.makedirs(dirname_path) + Utils.give_group_write_permissions(dirname_path) except OSError as ex: # This can be due to a race condition. If directory already exists, we don have to do nothing if not os.path.exists(dirname_path): diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 270878980351093c8e99b665049b01b2501aa78f..f8b218912f09ec2a4c5cf9dd148c50649e14b7e7 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -2,29 +2,21 @@ import csv import glob import json +import os import openpyxl -import os from bscearth.utils.log import Log from earthdiagnostics.constants import Basins from earthdiagnostics.frequency import Frequency from earthdiagnostics.modelingrealm import ModelingRealms +from singleton import SingletonType class VariableJsonException(Exception): pass -class SingletonType(type): - def __call__(cls, *args): - try: - return cls.__instance - except AttributeError: - cls.__instance = super(SingletonType, cls).__call__(*args) - return cls.__instance - - class VariableManager(object): __metaclass__ = SingletonType @@ -207,7 +199,7 @@ class VariableManager(object): continue alias_object = VariableAlias(alias) if line[2]: - alias_object.basin = Basins.parse(line[2]) + alias_object.basin = Basins().parse(line[2]) if line[3]: alias_object.grid = line[3] cmor_var.known_aliases.append(alias_object) @@ -359,7 +351,7 @@ class Variable(object): self.standard_name = var_line[2].strip() self.long_name = var_line[3].strip() self.domain = ModelingRealms.parse(var_line[4].strip()) - self.basin = Basins.parse(var_line[5]) + self.basin = Basins().parse(var_line[5]) self.units = var_line[6].strip() self.valid_min = var_line[7].strip() self.valid_max = var_line[8].strip() diff --git a/earthdiagnostics/variable_alias/cmip6.csv b/earthdiagnostics/variable_alias/cmip6.csv index d34dc918a1b934336c6c72762364f47aab6c8ac7..63708fcb0e9725d1a29c4e9aaeef2495ee6cbed7 100644 --- a/earthdiagnostics/variable_alias/cmip6.csv +++ b/earthdiagnostics/variable_alias/cmip6.csv @@ -1,5 +1,5 @@ Aliases,Shortname,Basin,Grid -iiceconc:soicecov:ileadfra,siconc,, +iiceconc:soicecov:ileadfra:sic,siconc,, alk,talk,, oxygen,o2,, calcite,calc,, diff --git a/earthdiagnostics/variable_alias/default.csv b/earthdiagnostics/variable_alias/default.csv index 8b967a4a575f19f1bde49a83f803a411ea4c8dc3..f6c32f07823b7ff90ae01ff99b140bef21ea69de 100644 --- a/earthdiagnostics/variable_alias/default.csv +++ b/earthdiagnostics/variable_alias/default.csv @@ -143,7 +143,7 @@ SExnsidc,siextents,, iiceprod,sigr,, iiceheco,siheco,, ibgsaltco,sisaltcga,, -iicethic:sithic,sit,, +iicethic:sithic::sithick,sit,, iice_hid:sithic_cat:sithicat,sitcat,, iicetemp,sitemp,, ibgtemper,sitempga,, diff --git a/earthdiagnostics/variable_alias/primavera.csv b/earthdiagnostics/variable_alias/primavera.csv index ff7b4128f33dffa4e17f822e7aac909be0b07b0c..cc704fb78757ecf18a133b5fe4a4655ab0185b63 100644 --- a/earthdiagnostics/variable_alias/primavera.csv +++ b/earthdiagnostics/variable_alias/primavera.csv @@ -1,2 +1,2 @@ Aliases,Shortname,Basin,Grid -iiceconc:siconc:soicecov:ileadfra,siconc,, +iiceconc:siconc:soicecov:ileadfra:sic,siconc,, diff --git a/model_diags.conf b/model_diags.conf index d479e349cf0f790bc4db223e985106b5aecdad6b..9edfcef0ed30ea723d52e93455c4f4c5965c189c 100644 --- a/model_diags.conf +++ b/model_diags.conf @@ -45,7 +45,7 @@ DATA_CONVENTION = CMIP6 # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = +DIAGS = siasie [EXPERIMENT] @@ -164,7 +164,7 @@ STC = mocarea,0,25,0,200,Pac mocarea,-25,0,0,200,Pac mocarea,0,25,0,200,Atl moca HEAT_SAL_MXL = mlotstsc mlotsthc LMSALC = vertmeanmeters,so,300,5400 USALC = vertmeanmeters,so,0,300 -OHC = ohc,glob,0,1,10 +OHC = ohc,glob,0,0,2000 XOHC = ohc,glob,1,0,0 LOHC = ohc,glob,0,700,2000 MOHC = ohc,glob,0,300,700 diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index 720478187ac7ca5c8d7cb60b918f6e13498db779..7801899fcc1b50961e14da3efa4bed9ef3009bf6 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -4,31 +4,17 @@ from earthdiagnostics.constants import Basins, Basin from earthdiagnostics.box import Box -class TestBasins(TestCase): - - def test_parse(self): - self.assertEquals(Basins.Arctic, Basins.parse(Basins.Arctic)) - self.assertEquals(Basins.Arctic, Basins.parse('Arct')) - self.assertEquals(Basins.Arctic, Basins.parse('Arctic_Ocean')) - self.assertIsNone(Basins.parse('Basin not found')) - - class TestBasin(TestCase): def setUp(self): - self.basin = Basin('bas', 'Basin', Box()) - - def test_shortname(self): - self.assertEquals('bas', self.basin.shortname) + self.basin = Basin('Basin') - def test_fullname(self): - self.assertEquals('Basin', self.basin.fullname) + def test_name(self): + self.assertEquals('Basin', self.basin.name) def test__eq__(self): - self.assertTrue(Basin('bas', 'Basin') == self.basin) - self.assertFalse(Basin('bas', 'OtherBasin') == self.basin) - self.assertFalse(Basin('otbas', 'Basin') == self.basin) - self.assertFalse(Basin('otbas', 'OtherBasin') == self.basin) + self.assertTrue(Basin('Basin') == self.basin) + self.assertFalse(Basin('bas') == self.basin) def test__str__(self): self.assertEquals(str(self.basin), 'Basin')