diff --git a/earthdiagnostics/CDFTOOLS_meteofrance.namlist b/earthdiagnostics/CDFTOOLS_meteofrance.namlist new file mode 100644 index 0000000000000000000000000000000000000000..8f747650cd51ec119ffe48e072dad4ab4ef9bc38 --- /dev/null +++ b/earthdiagnostics/CDFTOOLS_meteofrance.namlist @@ -0,0 +1,211 @@ + ! Thu Jun 30 16:19:27 2016 + ! Namelist automatically generated by PrintCdfNames + ! Do not edit without changing its name ... + ! ------------------------------------------ + &NAMDIM + CN_X = "i" + , + CN_Y = "j" + , + CN_Z = "lev" + , + CN_T = "time" + + / + &NAMDIMVAR + CN_VLON2D = "lon" + , + CN_VLAT2D = "lat" + , + CN_VDEPTHT = "lev" + , + CN_VDEPTHU = "lev" + , + CN_VDEPTHV = "lev" + , + CN_VDEPTHW = "lev" + , + CN_VTIMEC = "time" + , + CN_MISSING_VALUE = "_FillValue" + + / + &NAMMETRICS + CN_VE1T = "e1t" + , + CN_VE1U = "e1u" + , + CN_VE1V = "e1v" + , + CN_VE1F = "e1f" + , + CN_VE2T = "e2t" + , + CN_VE2U = "e2u" + , + CN_VE2V = "e2v" + , + CN_VE2F = "e2f" + , + CN_VE3T = "e3t" + , + CN_VE3W = "e3w" + , + CN_VFF = "ff" + , + CN_GLAMT = "glamt" + , + CN_GLAMU = "glamu" + , + CN_GLAMV = "glamv" + , + CN_GLAMF = "glamf" + , + CN_GPHIT = "gphit" + , + CN_GPHIU = "gphiu" + , + CN_GPHIV = "gphiv" + , + CN_GPHIF = "gphif" + , + CN_GDEPT = "gdept" + , + CN_GDEPW = "gdepw" + , + CN_HDEPT = "hdept" + , + CN_HDEPW = "hdepw" + + / + &NAMVARS + CN_VOTEMPER = "thetao" + , + CN_VOSALINE = "so" + , + CN_VOZOCRTX = "uo" + , + CN_VOMECRTY = "vo" + , + CN_VOMEEIVV = "vomeeivv" + , + CN_VOVECRTZ = "vovecrtz" + , + CN_SOSSHEIG = "sossheig" + , + CN_SOMXL010 = "mlotst" + , + CN_SOMXLT02 = "somxlt02" + , + CN_SOHEFLDO = "sohefldo" + , + CN_SOLHFLUP = "solhflup" + , + CN_SOSBHFUP = "sosbhfup" + , + CN_SOLWFLDO = "solwfldo" + , + CN_SOSHFLDO = "soshfldo" + , + CN_SOWAFLUP = "sowaflup" + , + CN_SOWAFLCD = "sowaflcd" + , + CN_SOWAFLDP = "sowafldp" + , + CN_IOWAFLUP = "iowaflup" + , + CN_ZOMSFATL = "zomsfatl" + , + CN_ZOMSFGLO = "zomsfglo" + , + CN_ZOMSFPAC = "zomsfpac" + , + CN_ZOMSFINP = "zomsfinp" + , + CN_ZOMSFIND = "zomsfind" + , + CN_ZOISOATL = "zoisoatl" + , + CN_ZOISOGLO = "zoisoglo" + , + CN_ZOISOPAC = "zoisopac" + , + CN_ZOISOINP = "zoisoinp" + , + CN_ZOISOIND = "zoisoind" + , + CN_VOZOUT = "vozout" + , + CN_VOMEVT = "vomevt" + , + CN_VOZOUS = "vozous" + , + CN_VOMEVS = "vomevs" + , + CN_SOZOUT = "sozout" + , + CN_SOMEVT = "somevt" + , + CN_SOZOUS = "sozous" + , + CN_SOMEVS = "somevs" + , + CN_SOZOUTRP = "sozoutrp" + , + CN_SOMEVTRP = "somevtrp" + , + CN_SOICECOV = "soicecov" + , + CN_VOSIGMA0 = "vosigma0" + , + CN_VOSIGMAI = "vosigmai" + , + CN_VOSIGNTR = "vosigntr" + , + CN_VODEPISO = "vodepiso" + , + CN_ISOTHICK = "isothick" + , + CN_IICETHIC = "iicethic" + , + CN_ILEADFRA = "ileadfra" + , + CN_INVCFC = "INVCFC" + , + CN_CFC11 = "CFC11" + , + CN_PENDEP = "pendep" + + / + &NAMBATHY + CN_FBATHYMET = "bathy_meter.nc" + , + CN_FBATHYLEV = "bathy_level.nc" + , + CN_BATHYMET = "Bathymetry" + , + CN_BATHYLEV = "bathy_level" + , + CN_MBATHY = "mbathy" + + / + ! Namelist entry namsqdvar needs manual formating before + ! it can be used as input : put variables names in between ' + ! and separate variables by , + &NAMSQDVAR + NN_SQDVAR = 4, + CN_SQDVAR = "vozocrtx vomecrty vovecrtz sossheig" , + / + &NAMMESHMASK + CN_FZGR = "mesh_zgr.nc" + , + CN_FHGR = "mesh_hgr.nc" + , + CN_FMSK = "mask.nc" + , + CN_FCOO = "coordinates.nc" + , + CN_FBASINS = "new_maskglo.nc", + + / diff --git a/earthdiagnostics/cmor_tables/meteofrance.csv b/earthdiagnostics/cmor_tables/meteofrance.csv new file mode 100644 index 0000000000000000000000000000000000000000..086e2ec0d7f084954d539a8e6bb63a3439f6c58e --- /dev/null +++ b/earthdiagnostics/cmor_tables/meteofrance.csv @@ -0,0 +1,3 @@ +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables +iiceconc:siconc:soicecov:ileadfra:ci,soicecov,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,, +iicethic:sithic,sogsit__,sea_ice_thickness,Sea Ice Thickness,seaIce,,m,,,, diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 4bfde9ded0e205fec05e44eca3eca3cab2231ee1..baf5f3de4d6fed167e143197c44edea5659577c2 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -1,7 +1,6 @@ # coding=utf-8 import glob import os -import pygrib import shutil import uuid from datetime import datetime @@ -338,6 +337,7 @@ class Cmorizer(object): def _get_atmos_timestep(self, gribfile): Log.info('Getting timestep...') + import pygrib grib_handler = pygrib.open(gribfile) dates = set() try: @@ -363,7 +363,7 @@ class Cmorizer(object): Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) - Utils.rename_variables(filename, Cmorizer.ALT_COORD_NAMES, False, True) + Utils.rename_variables(filename, self.alt_coord_names, False, True) self._remove_valid_limits(filename) self._add_common_attributes(filename, frequency) self._update_time_variables(filename) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index cfa67caefe57fce5578f933a1057dcd8b0dddcd3..a0237d94f853fee0e4b5ae731f2c490d70cbd809 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -23,11 +23,14 @@ class CMORManager(DataManager): def __init__(self, config): super(CMORManager, self).__init__(config) self._dic_cmorized = dict() + self.find_model_data() + self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid) + + def find_model_data(self): data_folders = self.config.data_dir.split(':') experiment_folder = self.experiment.model.lower() if experiment_folder.startswith('ec-earth'): experiment_folder = 'ecearth' - self.config.data_dir = None for data_folder in data_folders: if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): @@ -38,14 +41,12 @@ class CMORManager(DataManager): self.config.data_dir = test_folder break - test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) + test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) if os.path.isdir(os.path.join(test_folder, self.experiment.expid)): self.config.data_dir = test_folder break - if not self.config.data_dir: raise Exception('Can not find model data') - self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, @@ -53,29 +54,20 @@ class CMORManager(DataManager): cmor_var = self.variable_list.get_variable(var) filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) - # noinspection PyBroadException if possible_versions is None: - # noinspection PyBroadException - try: - return os.path.isfile(filepath) - except Exception: - return False + return os.path.isfile(filepath) else: for version in possible_versions: - # noinspection PyBroadException - try: - if os.path.isfile(filepath.replace(self.config.cmor.version, version)): - return True - except Exception: - pass + if os.path.isfile(filepath.replace(self.config.cmor.version, version)): + return True return False def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param vartype: - :param domain: CMOR domain + :param vartype: + :param domain: CMOR domain :type domain: Domain :param var: variable name :type var: str @@ -104,8 +96,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param year: - :param diagnostic: + :param year: + :param diagnostic: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -137,8 +129,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param region: + :param diagnostic: + :param region: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -179,8 +171,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param year: + :param diagnostic: + :param year: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -215,7 +207,7 @@ class CMORManager(DataManager): grid=None, year=None, date_str=None): """ Returns the path to a concrete file - :param cmor_var: + :param cmor_var: :param startdate: file's startdate :type startdate: str :param member: file's member @@ -239,6 +231,12 @@ class CMORManager(DataManager): :param cmor_var: variable instance describing the selected variable :type cmor_var: Variable """ + + options = sum(x is not None for x in (chunk, year, date_str)) + if options == 0: + raise ValueError('You must provide chunk, year or date_str') + elif options > 1: + raise ValueError('You must provide only one parameter in chunk, year or date_str') if not frequency: frequency = self.config.frequency @@ -286,6 +284,9 @@ class CMORManager(DataManager): self.experiment.experiment_name, self._get_member_str(member), grid, time_bound) + elif self.config.data_convention in ('meteofrance',): + time_bound = self._get_chunk_time_bounds(startdate, chunk) + file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self._get_member_str(member)) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name @@ -298,7 +299,10 @@ class CMORManager(DataManager): folder_path = os.path.join(folder_path, self._get_member_str(member)) if self.config.cmor.version: folder_path = os.path.join(folder_path, self.config.cmor.version) - else: + + elif self.config.data_convention in ('primavera', 'cmip6'): + if not self.config.cmor.version: + raise ValueError('CMOR version is mandatory for PRIMAVERA and CMIP6') if not grid: if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: grid = self.config.cmor.default_ocean_grid @@ -311,6 +315,12 @@ class CMORManager(DataManager): folder_path = os.path.join(self._get_startdate_path(startdate), self._get_member_str(member), table_name, var, grid, self.config.cmor.version) + elif self.config.data_convention == 'meteofrance': + folder_path = os.path.join(self.config.data_dir, self.experiment.experiment_name, + 'H{0}'.format(chr(64 + int(startdate[4:6]))), + startdate[0:4]) + else: + raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return folder_path def _get_chunk_time_bounds(self, startdate, chunk): @@ -322,7 +332,10 @@ class CMORManager(DataManager): separator = '_' else: separator = '-' - time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, + if self.config.data_convention == 'meteofrance': + time_bound = "{0:04}{1:02}".format(chunk_start.year, chunk_start.month) + else: + time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, chunk_end.month, separator) return time_bound @@ -332,7 +345,7 @@ class CMORManager(DataManager): """ Creates the link of a given file from the CMOR repository. - :param cmor_var: + :param cmor_var: :param move_old: :param date_str: :param year: if frequency is yearly, this parameter is used to give the corresponding year @@ -379,6 +392,9 @@ class CMORManager(DataManager): """ # Check if cmorized and convert if not + if self.config.data_convention == 'meteofrance': + return + for startdate, member in self.experiment.get_member_list(): if not self._unpack_cmor_files(startdate, member): self._cmorize_member(startdate, member) @@ -591,41 +607,9 @@ class CMORManager(DataManager): if not filename.endswith('.nc') or filename.startswith('.'): return cmorfile = os.path.join(filepath, filename) - self._fix_ij_swap(cmorfile) self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) - def _fix_ij_swap(self, cmorfile): - return - original_handler = Utils.openCdf(cmorfile) - if original_handler.dimensions['i'].size < original_handler.dimensions['j'].size: - temp = TempFile.get() - new_handler = Utils.openCdf(temp, 'w') - for attribute in original_handler.ncattrs(): - original = getattr(original_handler, attribute) - setattr(new_handler, attribute, - Utils.convert_to_ASCII_if_possible(original)) - for dimension in original_handler.dimensions.keys(): - if dimension == 'i': - new_name = 'j' - elif dimension == 'j': - new_name = 'i' - else: - new_name = dimension - new_handler.createDimension(new_name, original_handler.dimensions[dimension].size) - for variable in original_handler.variables.keys(): - original_var = original_handler.variables[variable] - translated_dimensions = Utils._translate(original_var.dimensions, - {'i': 'j', 'j': 'i'}) - new_var = new_handler.createVariable(variable, original_var.datatype, - translated_dimensions) - Utils.copy_attributes(new_var, original_var) - new_var[:] = original_var[:] - original_handler.close() - new_handler.close() - Utils.move_file(temp, cmorfile, save_hash=True) - Log.debug('File {0} translated', cmorfile) - def _get_startdate_path(self, startdate): """ Returns the path to the startdate's CMOR folder @@ -649,6 +633,8 @@ class CMORManager(DataManager): template = 'r{0}i{1}p1' elif self.config.data_convention in ('primavera', 'cmip6'): template = 'r{0}i{1}p1f1' + elif self.config.data_convention == 'meteofrance': + return '{0:02d}'.format(member) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index f3c8d63876bb289f68ccfb385af2d52c7ba01e30..e0dfa50b751723767cb67f95b191c9925e602737 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -3,7 +3,7 @@ import os import six from bscearth.utils.config_parser import ConfigParser -from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str, add_years, add_months, add_days from bscearth.utils.log import Log from earthdiagnostics import cdftools @@ -23,7 +23,7 @@ class Config(object): :param path: path to the conf file :type path: str """ - + def __init__(self, path): parser = ConfigParser() @@ -62,7 +62,8 @@ class Config(object): "Custom mask regions 3D file to use" self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', - ('specs', 'primavera', 'cmip6', 'preface'), 'specs', + ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), + 'specs', ignore_case=True) if self.data_convention in ('primavera', 'cmip6'): @@ -130,10 +131,10 @@ class Config(object): :rtype: list(str) """ return self._real_commands - + class CMORConfig(object): - + def __init__(self, parser, var_manager): self.force = parser.get_bool_option('CMOR', 'FORCE', False) self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) @@ -289,25 +290,52 @@ class ExperimentConfig(object): mem = mem[len(self.member_prefix):] members.append(int(mem)) self.members = members - + self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') startdates = parser.get_list_option('EXPERIMENT', 'STARTDATES') import exrex self.startdates = [] for startdate_pattern in startdates: - for startdate in exrex.generate(startdate_pattern): - self.startdates.append(startdate) + if startdate_pattern[0] == '{' and startdate_pattern[-1] == '}': + self._read_startdates(startdate_pattern[1:-1]) + else: + for startdate in exrex.generate(startdate_pattern): + self.startdates.append(startdate) self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') self.chunk_list = parser.get_int_list_option('EXPERIMENT', 'CHUNK_LIST', []) - self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') + self.model = parser.get_option('EXPERIMENT', 'MODEL') self.model_version = parser.get_option('EXPERIMENT', 'MODEL_VERSION', '') self.atmos_grid = parser.get_option('EXPERIMENT', 'ATMOS_GRID', '') self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) + def _read_startdates(self, pattern): + pattern = pattern.split(',') + start = parse_date(pattern[0].strip()) + end = parse_date(pattern[1].strip()) + interval = pattern[2].strip() + if len(interval) == 1: + factor = 1 + else: + factor = int(interval[0:-1]) + interval = interval[-1].upper() + while start <= end: + self.startdates.append(date2str(start)) + if interval == 'Y': + start = add_years(start, factor) + elif interval == 'M': + start = add_months(start, factor, cal=self.calendar) + elif interval == 'W': + start = add_days(start, factor * 7, cal=self.calendar) + elif interval == 'D': + start = add_days(start, factor, cal=self.calendar) + else: + raise ConfigException('Interval {0} not supported in STARTDATES definition: {1}', interval, pattern) + + def get_chunk_list(self): """ Return a list with all the chunks diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 229a61b0f0bbc0f92cca9895c34646f51e92b145..d79358380a31bd804d5655cfa61c97b57434d884 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -2,8 +2,6 @@ """ Contains the enumeration-like classes used by the diagnostics """ -import netCDF4 - from singleton import SingletonType @@ -133,7 +131,7 @@ class Basins(object): def get_available_basins(self, handler): """ - + :param handler: :type handler: netCDF4.Dataset """ diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 3b38e0771a390de9a8f7cecf65debbe672f5ea2b..b28f319a1991d3faf17dcb753a832ec57367d759 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -139,17 +139,19 @@ class DataFile(Publisher): self.dispatch(self) @classmethod - def from_storage(cls, filepath): + def from_storage(cls, filepath, data_convention): file_object = cls() file_object.remote_file = filepath file_object.local_status = LocalStatus.PENDING + file_object.data_convention = data_convention return file_object @classmethod - def to_storage(cls, remote_file): + def to_storage(cls, remote_file, data_convention): new_object = cls() new_object.remote_file = remote_file new_object.storage_status = StorageStatus.PENDING + new_object.data_convention = data_convention return new_object def download(self): @@ -511,6 +513,12 @@ class NetCDFFile(DataFile): self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) Utils.copy_file(self.remote_file, self.local_file) + if self.data_convention == 'meteofrance': + Log.debug('Converting variable names from meteofrance convention') + self.alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', + 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', + 'y': 'j'} + Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index b4ab012486095e8058845a99b50abada456fdb11..81fd37405afd4e8f2af9df52120ea7708ef5a2f1 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -29,7 +29,7 @@ class DataManager(object): def _get_file_from_storage(self, filepath): if filepath not in self.requested_files: - self.requested_files[filepath] = NCfile.from_storage(filepath) + self.requested_files[filepath] = NCfile.from_storage(filepath, self.config.data_convention) file_object = self.requested_files[filepath] file_object.local_satatus = LocalStatus.PENDING return self.requested_files[filepath] @@ -37,7 +37,7 @@ class DataManager(object): def _declare_generated_file(self, remote_file, domain, final_var, cmor_var, data_convention, region, diagnostic, grid, var_type, original_var): if remote_file not in self.requested_files: - self.requested_files[remote_file] = NCfile.to_storage(remote_file) + self.requested_files[remote_file] = NCfile.to_storage(remote_file, data_convention) file_object = self.requested_files[remote_file] file_object.diagnostic = diagnostic file_object.var_type = var_type @@ -48,7 +48,6 @@ class DataManager(object): file_object.final_name = final_var file_object.cmor_var = cmor_var file_object.region = region - file_object.data_convention = data_convention file_object.storage_status = StorageStatus.PENDING return file_object diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 0b1dbf57a346ca408883b904281ed6deff6c8502..c49dea841b6025e615959f7263fe8b4cfc3983f9 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -245,7 +245,6 @@ class Diagnostic(Publisher): def _updated_request(self, request): if self.status != DiagnosticStatus.WAITING: return - from datafile import LocalStatus if request.local_status == LocalStatus.FAILED: self.message = 'Required file {0} is not available'.format(request.remote_file) self.status = DiagnosticStatus.FAILED diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 44c7fba56a2bf986fb916bc7d7928f966e9bbca4..c8a489b90c3b098956217c8cafc6d8ff77859f61 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -46,6 +46,7 @@ class EarthDiags(object): def __init__(self, config_file): Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) self.config = Config(config_file) + os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' TempFile.scratch_folder = self.config.scratch_dir cdftools.path = self.config.cdftools_path diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index 7fb4736eba58980d40dce6f247b85f22b91850f3..1411b0ed8f883017fbc21c8241c26e001fce8fb3 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -5,7 +5,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class DailyMean(Diagnostic): diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index f72aa5f568d43f6e70fe277c528ea065d74db443..e8cf6d8504a419860b25d0edaefee4533df38ceb 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm import numpy as np diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index dca5e730af29859750df5055dc331976dfcf9df8..6062d075b6dce8c625e16e3724817606543dc9b1 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -5,7 +5,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class MonthlyMean(Diagnostic): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 60c69f4cf0894b38ec2df16ee2a379f37603587c..b2545e1b8ae11e346cfd4e57ed0bba7021ea3542 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -1,8 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption, \ DiagnosticVariableOption -from earthdiagnostics.modelingrealm import ModelingRealm -from earthdiagnostics.variable import VariableManager class Relink(Diagnostic): diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 116a978ddc632c5d7e64745f4425ef46c85da68f..099d6774c84ac1415bfea117744de2c9fddc0ae2 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,8 +1,10 @@ # coding=utf-8 +import math + +import numpy as np + from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils -from earthdiagnostics.modelingrealm import ModelingRealm -import math class Scale(Diagnostic): @@ -33,7 +35,7 @@ class Scale(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, value, offset, domain, variable, grid, - min_limit, max_limit, frequency): + min_limit, max_limit, frequency, apply_mask): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -46,18 +48,19 @@ class Scale(Diagnostic): self.min_limit = min_limit self.max_limit = max_limit self.frequency = frequency + self.apply_mask = apply_mask self.original_values = None def __str__(self): - return 'Scale output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Scale value: {5} Offset: {6} Variable: {3}:{4} ' \ - 'Frequency: {7}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, - self.value, self.offset, self.frequency) + return 'Scale output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Scale value: {0.value} Offset: {0.offset} Variable: {0.domain}:{0.variable} ' \ + 'Frequency: {0.frequency} Apply mask: {0.apply_mask}'.format(self) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.apply_mask == other.apply_mask and self.value == other.value and self.offset == other.offset @classmethod def generate_jobs(cls, diags, options): @@ -77,14 +80,16 @@ class Scale(Diagnostic): DiagnosticOption('grid', ''), DiagnosticFloatOption('min_limit', float('nan')), DiagnosticFloatOption('max_limit', float('nan')), - DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency])) + DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency]), + DiagnosticBoolOption('apply_mask', False)) options = cls.process_options(options, options_available) job_list = list() for frequency in options['frequencies']: for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Scale(diags.data_manager, startdate, member, chunk, options['value'], options['offset'], options['domain'], options['variable'], - options['grid'], options['min_limit'], options['max_limit'], frequency)) + options['grid'], options['min_limit'], options['max_limit'], frequency, + options['apply_mask'])) return job_list def request_data(self): @@ -102,10 +107,17 @@ class Scale(Diagnostic): variable_file = self.variable_file.local_file handler = Utils.openCdf(variable_file) - var_handler = handler.variables[self.variable] - self.original_values = var_handler[:] + var = handler.variables[self.variable] + self.original_values = var[:] + if self.apply_mask: + mask = Utils.get_mask(Basins().Global).astype(float) + mask[mask == 0] = np.nan + var[:] = mask * var[:] if self._check_limits(): - var_handler[:] = self.original_values * self.value + self.offset + values = self.original_values * self.value + self.offset + if self.apply_mask: + values[np.isnan(values)] = 0 + var[:] = values handler.close() self.corrected.set_local_file(self.variable_file.local_file, self) diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 1d2fb9cac33b0ebed3d5c3daf8fa3ab5b287b7bf..69931b87ddb36b54f2c68ed58fd59b1f4fd5403c 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption, DiagnosticIntOption -from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.box import Box diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index c903af0a7258259efdadf8ef6f415f9400eb8f34..c9c39e2194e0c9d983ebdd7d07c98a58449cb5de 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -3,7 +3,6 @@ import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption -from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.utils import Utils, TempFile @@ -127,7 +126,7 @@ class SimplifyDimensions(Diagnostic): '{0}_vertices'.format(self.lon_name), '{0}_vertices'.format(self.lat_name)): continue Utils.copy_variable(handler, new_file, var, new_names={'i': self.lon_name, 'j': self.lat_name}) - + self._create_var(self.lon_name, lon_values, handler, new_file) self._create_var(self.lat_name, lat_values, handler, new_file) handler.close() diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index a7af9aaa04719fc12837178e4a8af396dc3121f5..243703789a3dc9d7cb383bab44fbbbe5ffd7776f 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -25,6 +25,7 @@ class MaskLand(Diagnostic): """ alias = 'maskland' + "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, mask, grid): Diagnostic.__init__(self, data_manager) @@ -83,7 +84,7 @@ class MaskLand(Diagnostic): mask_file.close() return mask - "Diagnostic alias for the configuration file" + def request_data(self): self.var_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7c70b56e4d0e69a69d63a0a185e33d399db572be..b91fda94ecfcc125a66d6d331f56ce956e29f34e 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -10,6 +10,7 @@ import tarfile import tempfile from contextlib import contextmanager +import cf_units import iris import iris.exceptions import netCDF4 @@ -18,7 +19,6 @@ import six import xxhash from bscearth.utils.log import Log from cdo import Cdo -from cfunits import Units from nco import Nco from earthdiagnostics.constants import Basins @@ -390,7 +390,8 @@ class Utils(object): Log.log.log(log_level, line) output.append(line) if process.returncode != 0: - raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), process.returncode)) + raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), + str(process.returncode))) return output _cpu_count = None @@ -440,7 +441,9 @@ class Utils(object): handler = Utils.openCdf(filetoconvert) if not handler.file_format == 'NETCDF4': is_compressed = False + handler.close() else: + handler.close() ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) ncdump_result = ncdump_result[0].replace('\t', '').split('\n') for var in handler.variables: @@ -450,8 +453,6 @@ class Utils(object): if not '{0}:_Shuffle = "true" ;'.format(var) in ncdump_result: is_compressed = False break - - handler.close() return is_compressed @@ -666,15 +667,15 @@ class Utils(object): if hasattr(var_handler, 'calendar'): old_calendar = var_handler.calendar - new_unit = Units(new_units, calendar=calendar) - old_unit = Units(var_handler.units, calendar=old_calendar) - var_handler[:] = Units.conform(var_handler[:], old_unit, new_unit, inplace=True) + new_unit = cf_units.Unit(new_units, calendar=calendar) + old_unit = cf_units.Unit(var_handler.units, calendar=old_calendar) + var_handler[:] = old_unit.convert(var_handler[:], new_unit, inplace=True) if 'valid_min' in var_handler.ncattrs(): - var_handler.valid_min = Units.conform(float(var_handler.valid_min), old_unit, new_unit, - inplace=True) + var_handler.valid_min = old_unit.convert(float(var_handler.valid_min), new_unit, + inplace=True) if 'valid_max' in var_handler.ncattrs(): - var_handler.valid_max = Units.conform(float(var_handler.valid_max), old_unit, new_unit, - inplace=True) + var_handler.valid_max = old_unit.convert(float(var_handler.valid_max), new_unit, + inplace=True) var_handler.units = new_units @staticmethod diff --git a/earthdiagnostics/variable_alias/meteofrance.csv b/earthdiagnostics/variable_alias/meteofrance.csv new file mode 100644 index 0000000000000000000000000000000000000000..abb99d5efa983b044037db11656d46a16a2934b3 --- /dev/null +++ b/earthdiagnostics/variable_alias/meteofrance.csv @@ -0,0 +1,4 @@ +Aliases,Shortname,Basin,Grid +iiceconc:siconc:soicecov:ileadfra,soicecov,, +ci,sic,,ifs +es,sbl,, diff --git a/environment.yml b/environment.yml index 4fa44d1373b1f8f66db6d573a40c57125b2909b1..14d620055e705e0214f91ecc4d3b850a6bf96835 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,6 @@ dependencies: - openpyxl - mock - cmake -- cfunits - coverage - pip: diff --git a/launch_diags.sh b/launch_diags.sh index 477a1ca4069f72efad559e9052a14227487b5318..bcfa6b072f332d9e7700dcd518a13b7cdfca7f75 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -21,6 +21,6 @@ set -xv source activate diags -export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} -cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ -./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} + export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} + cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ + ./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} diff --git a/test/unit/general/test_relinkall.py b/test/unit/general/test_relinkall.py index cf8c9a163437d5d868401a54b6df62195d0e1f7a..cda393c8b635c8e95e1117a762bfac674d1bc769 100644 --- a/test/unit/general/test_relinkall.py +++ b/test/unit/general/test_relinkall.py @@ -6,8 +6,6 @@ from earthdiagnostics.box import Box from earthdiagnostics.general.relinkall import RelinkAll from mock import Mock, patch -from earthdiagnostics.modelingrealm import ModelingRealms - class TestRelinkAll(TestCase): diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py index e7697cc26076c55e7a8d50ea65031a74305d3ac0..b2fcd0d0e341b0a891680d29569ab8dec4125cf0 100644 --- a/test/unit/general/test_scale.py +++ b/test/unit/general/test_scale.py @@ -1,12 +1,12 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError -from earthdiagnostics.box import Box -from earthdiagnostics.general.scale import Scale -from earthdiagnostics.frequency import Frequencies from mock import Mock, patch +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.scale import Scale from earthdiagnostics.modelingrealm import ModelingRealms @@ -32,40 +32,48 @@ class TestScale(TestCase): jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', '', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', '', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.monthly)) + 0, 100, Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.monthly)) + 0, 100, Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.three_hourly)) + 0, 100, Frequencies.three_hourly, False)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly, False)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', + True]) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly, True)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.three_hourly)) + 0, 100, Frequencies.three_hourly, True)) with self.assertRaises(DiagnosticOptionError): Scale.generate_jobs(self.diags, ['diagnostic']) - with self.assertRaises(DiagnosticOptionError): - Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', - 'extra']) + with self.assertRaises(DiagnosticOptionError): + Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', 'True', + 'extra']) def test_str(self): mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, - Frequencies.three_hourly) + Frequencies.three_hourly, False) self.assertEquals(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' - 'Variable: atmos:var Frequency: 3hr') + 'Variable: atmos:var Frequency: 3hr Apply mask: False') diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index ba238f28edfa7feabcaf8f887c6e881b2f0d86a7..b7b87cf13041d717885756a7a91e15bec99c20f9 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -28,8 +28,10 @@ class TestInterpolate(TestCase): @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO.get_sample_grid_file') @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) @patch('os.remove') - def test_generate_jobs(self, mock_weights, mock_grid_file, mock_remove): + @patch('earthdiagnostics.utils.TempFile.get') + def test_generate_jobs(self, mock_weights, mock_grid_file, mock_remove, mock_get): mock_weights.return_value = None + mock_get.return_value = 'path_to_weights' jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var']) self.assertEqual(len(jobs), 2) diff --git a/test/unit/ocean/test_vertical_gradient.py b/test/unit/ocean/test_vertical_gradient.py index e2d9d22d32358059c73899c1c7aee834400dddab..892160a200efb02c1cd36b40c3c1c838954fdfeb 100644 --- a/test/unit/ocean/test_vertical_gradient.py +++ b/test/unit/ocean/test_vertical_gradient.py @@ -1,8 +1,6 @@ # coding=utf-8 from unittest import TestCase from earthdiagnostics.ocean.verticalgradient import VerticalGradient -from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption from mock import Mock, patch diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py new file mode 100644 index 0000000000000000000000000000000000000000..f536dd648fd8747321abf36d683d90a2f10c0491 --- /dev/null +++ b/test/unit/test_cmormanager.py @@ -0,0 +1,319 @@ +# coding=utf-8 +import os +import shutil +import tempfile +from unittest import TestCase + +import mock +from mock import Mock + +from earthdiagnostics.cmormanager import CMORManager +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestCMORManager(TestCase): + + def setUp(self): + self.config = Mock() + self.config.data_convention = 'specs' + self.config.data_type = 'exp' + self.config.experiment.expid = 'expid' + self.config.experiment.model = 'model' + self.config.experiment.experiment_name = 'expname' + self.config.experiment.institute = 'institute' + self.config.experiment.member_count_start = 0 + self.config.experiment.atmos_timestep = 6 + self.config.experiment.ocean_timestep = 6 + self.config.experiment.chunk_size = 12 + self.config.experiment.calendar = 'standard' + + self.config.cmor.initialization_number = 1 + self.config.cmor.version = '' + self.config.cmor.default_ocean_grid = 'ocean_grid' + self.config.cmor.default_atmos_grid = 'atmos_grid' + self.config.cmor.activity = 'activity' + + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + self.config.data_dir = self.tmp_dir + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_find_data(self): + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'expid')) + + def test_find_data_fail(self): + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + with self.assertRaises(Exception): + CMORManager(self.config) + + def test_find_data_with_model(self): + os.makedirs(os.path.join(self.tmp_dir, 'model', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'model', 'expid')) + + def test_find_data_with_ecearth_fix(self): + self.config.experiment.model = 'EC-Earth' + os.makedirs(os.path.join(self.tmp_dir, 'ecearth', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'ecearth', 'expid')) + + def test_find_data_with_type_and_model(self): + os.makedirs(os.path.join(self.tmp_dir, 'exp', 'model', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'exp', 'model', 'expid')) + + def test_get_varfolder(self): + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.get_varfolder(ModelingRealms.ocean, 'var'), + 'var_f6h') + + def test_get_file_path_specs(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_version(self): + self.config.cmor.version = 'version' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/version/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_grid(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency, 'grid') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/grid/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_year(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'year' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'year' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, year='1998') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/year/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_1998.nc')) + + frequency.frequency = 'other' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'other' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + + def test_get_file_path_raise_incomaptible_date_info(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'monthly' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, date_str='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + None, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998', date_str='1998') + + def test_get_file_path_specs_date_str(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + frequency.frequency = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, date_str='date_str') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_date_str.nc')) + + def test_get_file_path_primavera(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' + 'r2i1p1f1/Omon/var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) + + def test_get_file_path_no_version_primavera(self): + self._configure_primavera() + self.config.cmor.version = '' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 0, frequency) + + def _configure_primavera(self): + self.config.data_convention = 'primavera' + self.config.cmor.version = 'version' + + def _configure_meteofrance(self): + self.config.data_convention = 'meteofrance' + + def test_get_file_path_primavera_grid(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency, 'grid') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' + 'Omon/var/grid/version/' + 'var_Omon_model_expname_r2i1p1f1_grid_198901-198912.nc')) + + def test_get_file_path_primavera_year(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'year' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'year' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, year='1998') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/' + 'var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_1998.nc')) + + frequency.frequency = 'other' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'other' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + + def test_get_file_path_primavera_date_str(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + frequency.frequency = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, date_str='date_str') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' + 'Omon/var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_date_str.nc')) + + def test_file_exists(self): + with mock.patch('os.path.isfile') as isfile: + cmor_manager = CMORManager(self.config) + isfile.return_value = True + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) + isfile.return_value = False + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) + + + def test_file_exists_multiple_versions(self): + with mock.patch('os.path.isfile') as isfile: + cmor_manager = CMORManager(self.config) + isfile.return_value = True + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + possible_versions=('version1', 'version2'))) + isfile.return_value = False + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + possible_versions=('version1', 'version2'))) + + def test_get_file_path_meteofrance(self): + self._configure_meteofrance() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'day' + file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_01.nc')) + + file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 2, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_01.nc')) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index a1c8d25b8dfd0e6769e60f5c624cf75983efadfb..b113d8bdcd9d8d2906392bfd1620a45c3a77ea74 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -306,9 +306,28 @@ class TestExperimentConfig(TestCase): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200[0-2](02|05|08|11)01') config = ExperimentConfig(self.mock_parser) - print(config.startdates) self.assertEquals(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', u'20020801', u'20021101']) + def test_auto_startdates(self): + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,6M}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20010501', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,1W}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7D}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7F}') + with self.assertRaises(ConfigException): + ExperimentConfig(self.mock_parser)