diff --git a/VERSION b/VERSION index 9b08b73b1821da3009e264407690374daf0f3ead..989de3b2fcadf8b5f6a672023913c7a0834e5e22 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0b20 +3.0.0b22 diff --git a/diags.conf b/diags.conf index 149ce4212a9954e0afd5f17f1811e58afe802be8..0c17c47d198c03bedc86f45cca3386b6cc4c4e22 100644 --- a/diags.conf +++ b/diags.conf @@ -4,14 +4,16 @@ DATA_ADAPTOR = THREDDS # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use -DATA_DIR = /esnas/exp/:/esarchive/exp/ +DATA_DIR = /esnas:/esarchive +# Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) +DATA_TYPE = exp # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnpostics just to CMORize, leave it # empty -DIAGS = monpercent,atmos,tas,90 monpercent,atmos,tas,10 monpercent,atmos,sfcWind,90 monpercent,atmos,sfcWind,10 +DIAGS = climpercent,atmos,sfcWind,1 # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. @@ -59,7 +61,7 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, # SOURCE = 'EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 [THREDDS] -SERVER_URL = http://earth.bsc.es/thredds +SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard @@ -81,7 +83,7 @@ OCEAN_TIMESTEP = 6 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = resilience -STARTDATES = 19811101 +STARTDATES = 19810101 MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 7 diff --git a/doc/source/codedoc/general.rst b/doc/source/codedoc/general.rst index ffa9ea9c7d69dbb0c6c64a183e3224abc02fc4fa..5dc0a023ef3af8bc8399795ab95ff68ee29632bd 100644 --- a/doc/source/codedoc/general.rst +++ b/doc/source/codedoc/general.rst @@ -1,26 +1,32 @@ earthdiagnostics.general ======================== +earthdiagnostics.general.attribute +---------------------------------- +.. automodule:: earthdiagnostics.general.attribute + :show-inheritance: + :members: + earthdiagnostics.general.monthlymean ------------------------------------ .. automodule:: earthdiagnostics.general.monthlymean :show-inheritance: :members: -earthdiagnostics.ocean.relink ------------------------------ +earthdiagnostics.general.relink +------------------------------- .. automodule:: earthdiagnostics.general.relink :show-inheritance: :members: -earthdiagnostics.ocean.rewrite ------------------------------- +earthdiagnostics.general.rewrite +-------------------------------- .. automodule:: earthdiagnostics.general.rewrite :show-inheritance: :members: -earthdiagnostics.ocean.scale ----------------------------- +earthdiagnostics.general.scale +------------------------------ .. automodule:: earthdiagnostics.general.scale :show-inheritance: :members: diff --git a/doc/source/codedoc/statistics.rst b/doc/source/codedoc/statistics.rst index bc8dcc6094cc56c46bf09c92b7d91d44045913ed..e78a54c917af432d1c52d625c7963a8704e3989b 100644 --- a/doc/source/codedoc/statistics.rst +++ b/doc/source/codedoc/statistics.rst @@ -8,7 +8,7 @@ earthdiagnostics.statistics.climatologicalpercentile :members: earthdiagnostics.statistics.monthlypercentile ---------------------------------------------_ +--------------------------------------------- .. automodule:: earthdiagnostics.statistics.monthlypercentile :show-inheritance: :members: diff --git a/doc/source/conf.py b/doc/source/conf.py index 344d7a72439d8d29e2c12f0ddce15e5b52ddfaa3..ec240e9bd22f5a115b142a51e38bba79fc6b2fb4 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b20' +release = '3.0.0b22' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst index f1ea82d3b085308b923c86ff1c1eadc461e3c09b..2087ccd24b66cbc5e4342fac29baa889ac960fcb 100644 --- a/doc/source/diagnostic_list.rst +++ b/doc/source/diagnostic_list.rst @@ -13,7 +13,9 @@ Remember that diagnostics are specified separated by spaces while options are gi General ------- - +- att: + Writes a global attributte to all the netCDF files. + See :class:`~earthdiagnostics.general.attribute.Attribute` - monmean: Calculates the monthly mean of the given variable. See :class:`~earthdiagnostics.general.monthlymean.MonthlyMean` @@ -28,7 +30,7 @@ General - scale: Scales a given variable using a given scale factor and offset. Useful to correct erros on the data. - See :class:`~earthdiagnostics.general.rewrite.Scale` + See :class:`~earthdiagnostics.general.scale.Scale` Ocean ----- diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index 89ef4b703e07684779fdb2f1da6ce9ff71d8512c..b674e5c9495795b63c67b9eed0843513c33303ea 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index baa5623a24b0a95d201470a86956f7e105288729..06331b1d4d9e390f6cf7143fd5c356fb506ebb37 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -33,19 +33,9 @@ class CDFTools(object): """ line = [os.path.join(self.path, command)] - if self.path and not os.path.exists(line[0]): - raise ValueError('Error executing {0}\n Command does not exist in {1}', command, self.path) - if input: - if isinstance(input, basestring): - line.append(input) - if not os.path.exists(input): - raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, input) - else: - for element in input: - line.append(element) - if not os.path.exists(element): - raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) + self._check_command_existence(line[0]) + self._check_input(command, input, line) if options: if isinstance(options, basestring): options = options.split() @@ -59,8 +49,40 @@ class CDFTools(object): Log.debug('Executing {0}', ' '.join(line)) shell_output = Utils.execute_shell_command(line, log_level) + self._check_output_was_created(line, output) + return shell_output + + @staticmethod + def _check_output_was_created(line, output): if output: if not os.path.exists(output): raise Exception('Error executing {0}\n Output file not created', ' '.join(line)) - return shell_output + @staticmethod + def _check_input(command, input, line): + if input: + if isinstance(input, basestring): + line.append(input) + if not os.path.exists(input): + raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, input) + else: + for element in input: + line.append(element) + if not os.path.exists(element): + raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) + + @staticmethod + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + def _check_command_existence(self, command): + if self.path: + if CDFTools.is_exe(os.path.join(self.path, command)): + return + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, command) + if CDFTools.is_exe(exe_file): + return + raise ValueError('Error executing {0}\n Command does not exist in {1}', command, self.path) diff --git a/earthdiagnostics/cmor_table.csv b/earthdiagnostics/cmor_table.csv index 4e8090b10eb0ed92449b49081a8e23238cf3afd8..5c403f0d47f23dc83ae18a26a5e300f8b17797e3 100644 --- a/earthdiagnostics/cmor_table.csv +++ b/earthdiagnostics/cmor_table.csv @@ -231,9 +231,11 @@ tsn,tsn,temperature_in_surface_snow,Snow internal temperature,landIce,,,,, u,ua,eastward_wind,U velocity,atmos,,,,, u10m,uas,eastward_wind,Eastward near-surface wind,atmos,,,,, vozocrtx,uo,sea_water_x_velocity,Sea water x velocity,ocean,,,,, +uos,uos,sea_surface_x_velocity,Sea surface x velocity,ocean,,,,, v,va,northward_wind,V velocity,atmos,,,,, v10m,vas,northward_wind,Northward near-surface wind,atmos,,,,, vomecrty,vo,sea_water_y_velocity,Sea water y velocity,ocean,,,,, +vos,vos,sea_surface_y_velocity,Sea surface y velocity,ocean,,,,, voddmavs,voddmavs,salt_vertical_eddy_diffusivity,Salt vertical eddy diffusivity,ocean,,,,, vozoeivu,voeivu,sea_water_x_EIV_current,Zonal eiv current,ocean,,,,, vomeeivv,voeivv,sea_water_y_EIV_current,Meridional eiv current,ocean,,,,, diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 41ae6dfe302499e0be5e0f5f152f7e4d831d80eb..fa90b12dd3a2b548e24ba3497e248e5a8b056fa4 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -79,6 +79,17 @@ class Cmorizer(object): for filename in glob.glob(os.path.join(self.cmor_scratch, '*.nc')): self._cmorize_nc_file(filename) + def _correct_fluxes(self): + fluxes_vars = ("prsn", "rss", "rls", "rsscs", "rsds", "rlds") + for filename in glob.glob(os.path.join(self.cmor_scratch, '*.nc')): + handler = Utils.openCdf(filename) + for varname in handler.variables.keys(): + cmor_var = Variable.get_variable(varname, True) + if cmor_var.short_name not in fluxes_vars: + continue + handler.variables[varname][:] = handler.variables[varname][:] / self.experiment.atmos_timestep * 3600 + handler.close() + def _unpack_tar_file(self, tarfile): if os.path.exists(self.cmor_scratch): shutil.rmtree(self.cmor_scratch) @@ -125,6 +136,7 @@ class Cmorizer(object): Log.info('Unpacking atmospheric file {0}/{1}'.format(count, len(tar_files))) self._unpack_tar_file(tarfile) self._merge_mma_files(tarfile) + self._correct_fluxes() self._cmorize_nc_files() Log.result('Atmospheric file {0}/{1} finished'.format(count, len(tar_files))) count += 1 @@ -236,6 +248,7 @@ class Cmorizer(object): Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) Utils.rename_variables(filename, Cmorizer.ALT_COORD_NAMES, False, True) + self._remove_valid_limits(filename) self._add_common_attributes(filename, frequency) self._update_time_variables(filename) @@ -249,6 +262,16 @@ class Cmorizer(object): handler.close() os.remove(filename) + def _remove_valid_limits(self, filename): + handler = Utils.openCdf(filename) + for variable in handler.variables.keys(): + var = handler.variables[variable] + if 'valid_min' in var.ncattrs(): + del var.valid_min + if 'valid_max' in var.ncattrs(): + del var.valid_max + handler.close() + def _get_nc_file_frequency(self, filename): file_parts = os.path.basename(filename).split('_') if self.experiment.expid in [file_parts[1], file_parts[2]]: @@ -281,7 +304,6 @@ class Cmorizer(object): :type variable: str """ temp = TempFile.get() - file_parts = os.path.basename(file_path).split('_') var_cmor = Variable.get_variable(variable) if var_cmor is None: return @@ -298,14 +320,8 @@ class Cmorizer(object): else: region = var_cmor.basin.fullname - if file_parts[0] == self.experiment.expid or file_parts[0].startswith('ORCA') or \ - file_parts[0] in ('MMA', 'MMO'): - # Model output - date_str = '{0}-{1}'.format(file_parts[2][0:6], file_parts[3][0:6]) - elif file_parts[1] == self.experiment.expid: - # Files generated by the old version of the diagnostics - date_str = '{0}-{1}'.format(file_parts[4][0:6], file_parts[5][0:6]) - else: + date_str = self.get_date_str(file_path) + if date_str is None: Log.error('Variable {0} can not be cmorized. Original filename does not match a recognized pattern', var_cmor.short_name) raise CMORException('Variable {0}:{1} can not be cmorized. Original filename does not match a recognized ' @@ -315,6 +331,17 @@ class Cmorizer(object): frequency=frequency, rename_var=variable, date_str=date_str, region=region, move_old=True, grid=var_cmor.grid, cmorized=True) + def get_date_str(self, file_path): + file_parts = os.path.basename(file_path).split('_') + if file_parts[0] in (self.experiment.expid, 'MMA', 'MMO') or file_parts[0].startswith('ORCA'): + # Model output + return '{0}-{1}'.format(file_parts[2][0:6], file_parts[3][0:6]) + elif file_parts[1] == self.experiment.expid: + # Files generated by the old version of the diagnostics + return '{0}-{1}'.format(file_parts[4][0:6], file_parts[5][0:6]) + else: + return None + @staticmethod def _add_coordinate_variables(handler, temp): handler_cmor = Utils.openCdf(temp) @@ -484,7 +511,7 @@ class Cmorizer(object): handler = Utils.openCdf(filename) handler.associated_experiment = cmor.associated_experiment handler.batch = '{0}{1}'.format(experiment.institute, datetime.now().strftime('%Y-%m-%d(T%H:%M:%SZ)')) - handler.contact = 'Pierre-Antoine Bretonnière, pierre-antoine.bretonniere@bsc.es , ' \ + handler.contact = 'Pierre-Antoine Bretonniere, pierre-antoine.bretonniere@bsc.es , ' \ 'Javier Vegas-Regidor, javier.vegas@bsc.es ' handler.Conventions = 'CF-1.6' handler.creation_date = datetime.now().strftime('%Y-%m-%d(T%H:%M:%SZ)') diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 2382c62a4978b71c93490d707e0306e5105b5702..785b515ecc886bb226a07b5e4247674b6102ae0e 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -27,6 +27,7 @@ class CMORManager(DataManager): if not self.config.data_dir: raise Exception('Can not find model data') + self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VarType.MEAN): @@ -77,7 +78,7 @@ class CMORManager(DataManager): :param box: file's box :type box: Box :param grid: file's grid - :type grid: str + :type grid: str|NoneType :param year: file's year :type year: int|str :param date_str: date string to add directly. Overrides year or chunk configurations @@ -130,7 +131,7 @@ class CMORManager(DataManager): return time_bound def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, - frequency=None, year=None, date_str=None, move_old=False): + frequency=None, year=None, date_str=None, move_old=False, vartype=VarType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -161,7 +162,8 @@ class CMORManager(DataManager): if not frequency: frequency = self.config.frequency - filepath = self.get_file_path(startdate, member, domain, var, chunk, frequency, grid, str(year), date_str) + filepath = self.get_file_path(startdate, member, domain, var, chunk, frequency, grid=grid, year=str(year), + date_str=date_str) self._create_link(domain, filepath, frequency, var, grid, move_old, vartype) def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, @@ -217,7 +219,7 @@ class CMORManager(DataManager): if not frequency: frequency = self.config.frequency - filepath = self.get_file_path(startdate, member, domain, var, chunk, frequency, box, + filepath = self.get_file_path(startdate, member, domain, var, chunk, frequency, None, grid, year, date_str) netcdf_file = NetCDFFile(filepath, filetosend, domain, var, cmor_var) if diagnostic: diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 206e12a6fc1cc8e1d9e6ad528fadb21ae9dcab7a..ab206c12ab8d8953939ff066950c3d3ccf4c0d31 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -29,11 +29,17 @@ class Config(object): "Scratch folder path" self.data_dir = Utils.expand_path(parser.get_option('DIAGNOSTICS', 'DATA_DIR')) "Root data folder path" + self.data_type = Utils.expand_path(parser.get_option('DIAGNOSTICS', 'DATA_TYPE', 'exp')).lower() + "Data type (experiment, observation or reconstruction)" + if self.data_type not in ('exp', 'obs', 'recon'): + raise Exception('Data type must be exp, obs or recon') self.con_files = Utils.expand_path(parser.get_option('DIAGNOSTICS', 'CON_FILES')) "Mask and meshes folder path" self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') - self.frequency = parser.get_option('DIAGNOSTICS', 'FREQUENCY') + self.frequency = parser.get_option('DIAGNOSTICS', 'FREQUENCY').lower() "Default data frequency to be used by the diagnostics" + if self.frequency == 'month': + self.frequency = 'mon' self.cdftools_path = Utils.expand_path(parser.get_option('DIAGNOSTICS', 'CDFTOOLS_PATH')) "Path to CDFTOOLS executables" self.max_cores = parser.get_int_option('DIAGNOSTICS', 'MAX_CORES', 100000) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 2c0f8553ab739f5f4540728b1177093b491d7ff3..312300d2ecf48429981f8d1a1940d99910e0a713 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -27,7 +27,7 @@ class DataManager(object): Variable.load_variables() UnitConversion.load_conversions() self.lock = threading.Lock() - self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') + def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VarType.MEAN): @@ -197,7 +197,7 @@ class DataManager(object): # Overridable methods (not mandatory) def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, - frequency=None, year=None, date_str=None, move_old=False): + frequency=None, year=None, date_str=None, move_old=False, vartype=VarType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -423,15 +423,10 @@ class NetCDFFile(object): history_line = handler.history + history_line except AttributeError: history_line = history_line - handler.history = self.maybe_encode(history_line) + handler.history = Utils.convert_to_ASCII_if_possible(history_line) handler.close() - @staticmethod - def maybe_encode(string, encoding='ascii'): - try: - return string.encode(encoding) - except UnicodeEncodeError: - return string + class UnitConversion(object): diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index dac21637ebca786bb0fdbc2118e3f2effd2f9158..74f718a266632a6328156d7c298a07c13b36a6d3 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # coding=utf-8 - import Queue import argparse import shutil @@ -12,18 +11,15 @@ import operator import os from autosubmit.date.chunk_date_lib import * -from config import Config +from earthdiagnostics.config import Config from earthdiagnostics.cmormanager import CMORManager from earthdiagnostics.threddsmanager import THREDDSManager from earthdiagnostics import cdftools -from earthdiagnostics.utils import TempFile +from earthdiagnostics.utils import TempFile, Utils from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.ocean import * from earthdiagnostics.general import * from earthdiagnostics.statistics import * -from ocean import ConvectionSites, Gyres, Psi, MaxMoc, AreaMoc, Moc, VerticalMean, VerticalMeanMeters, Interpolate, \ - AverageSection, CutSection, MixedLayerSaltContent, Siasiesiv -from utils import Utils class EarthDiags(object): @@ -219,6 +215,7 @@ class EarthDiags(object): Diagnostic.register(Rewrite) Diagnostic.register(Relink) Diagnostic.register(Scale) + Diagnostic.register(Attribute) @staticmethod def _register_ocean_diagnostics(): diff --git a/earthdiagnostics/general/__init__.py b/earthdiagnostics/general/__init__.py index 40240e79fb4659030cd62757bd3b931194358922..1a3cf92a4f96319537cb0b137f3e494e76081b01 100644 --- a/earthdiagnostics/general/__init__.py +++ b/earthdiagnostics/general/__init__.py @@ -3,3 +3,4 @@ from earthdiagnostics.general.monthlymean import MonthlyMean from earthdiagnostics.general.rewrite import Rewrite from earthdiagnostics.general.relink import Relink from earthdiagnostics.general.scale import Scale +from earthdiagnostics.general.attribute import Attribute diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py new file mode 100644 index 0000000000000000000000000000000000000000..59edf3c65059657bb560882fc2cdd5f063edd2bd --- /dev/null +++ b/earthdiagnostics/general/attribute.py @@ -0,0 +1,99 @@ +# coding=utf-8 +from earthdiagnostics.diagnostic import Diagnostic +from earthdiagnostics.utils import Utils +from earthdiagnostics.variable import Domain + + +class Attribute(Diagnostic): + """ + Rewrites files without doing any calculations. + Can be useful to convert units or to correct wrong metadata + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: Domain + """ + + alias = 'att' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, + attributte_name, attributte_value): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + self.attributte_name = attributte_name + self.attributte_value = attributte_value + + def __str__(self): + return 'Write attributte output Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Attributte:{5}:{6}'.format(self.startdate, self.member, self.chunk, self.domain, + self.variable, self.attributte_name, self.attributte_value) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and \ + self.attributte_name == other.attributte_name and self.attributte_value == other.attributte_value + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ + num_options = len(options) - 1 + if num_options < 4: + raise Exception('You must specify the variable, domain, attributte name and value to write') + if num_options > 5: + raise Exception('You must between 4 and 5 parameters for the rewrite diagnostic') + variable = options[1] + domain = Domain(options[2]) + name = options[3] + value = options[4] + value = value.replace('&;', ',').replace('&.', ' ') + if num_options >= 5: + grid = options[5] + else: + grid = None + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(Attribute(diags.data_manager, startdate, member, chunk, domain, variable, grid, name, value)) + return job_list + + def compute(self): + """ + Runs the diagnostic + """ + variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + handler = Utils.openCdf(variable_file) + handler.setncattr(self.attributte_name, self.attributte_value) + handler.close() + if not Utils.check_netcdf_file(variable_file): + raise Exception('Attribute {0} can not be set correctly to {1}'.format(self.attributte_name, + self.attributte_value)) + self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index 38224e857dca5b2da7f5293f014d9f2d39be6c63..69983f37176c2b6774e302ee2681bc69131c8a08 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -79,7 +79,7 @@ class MonthlyMean(Diagnostic): if num_options >= 4: grid = options[4] else: - grid = '' + grid = None job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index ba89b5e8454a45fb733fd477126e61d6229da8dd..c7acd74401dd0fd5f9ce625d306849b7f62fbaae 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -40,11 +40,12 @@ class Rewrite(Diagnostic): def __str__(self): return 'Rewrite output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable) + 'Variable: {3}:{4} Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.grid) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable + self.domain == other.domain and self.variable == other.variable and self.grid == self.grid @classmethod def generate_jobs(cls, diags, options): diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 618d54c73dbcd3eca39d8343e86c6a07529adefa..5d639e3b9cc2b90daee4a385e27f5fa0d2a6bf26 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -78,7 +78,7 @@ class HeatContent(Diagnostic): raise Exception('You must specify 4 parameters for the heat content diagnostic') basin = Basins.parse(options[1]) mixed_layer = int(options[2]) - box = Box() + box = Box(True) box.min_depth = int(options[3]) box.max_depth = int(options[4]) job_list = list() diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 353a14be53258cc985eb995f170ff71ab46d57dc..5401dffa4aeb18afe59869eefc1466a1a6895fe9 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -132,7 +132,8 @@ class Interpolate(Diagnostic): Utils.move_file(self._get_level_file(0), temp) handler = Utils.openCdf(temp) - handler.renameDimension('record', 'lev') + if 'record' in handler.dimensions: + handler.renameDimension('record', 'lev') handler.close() nco.ncpdq(input=temp, output=temp, options='-O -h -a time,lev') diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 1e52049cb47c2a7e897ddeb6caa4299e6e68b2eb..5df474194dbcfc818ce629d0930a25fe6f5b5bfc 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -22,7 +22,7 @@ class ClimatologicalPercentile(Diagnostic): alias = 'climpercent' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, domain, variable, leadtimes, experiment_config): + def __init__(self, data_manager, domain, variable, leadtimes, num_bins, experiment_config): Diagnostic.__init__(self, data_manager) self.variable = variable self.domain = domain @@ -32,7 +32,7 @@ class ClimatologicalPercentile(Diagnostic): self.realizations = None self.lat_len = None self.lon_len = None - self.num_bins = 2000 + self.num_bins = num_bins self._bins = None self.percentiles = np.array([0.1, 0.25, 0.33, 0.5, 0.66, 0.75, 0.9]) self.cmor_var = Variable.get_variable(variable, silent=True) @@ -67,14 +67,18 @@ class ClimatologicalPercentile(Diagnostic): if num_options < 3: raise Exception('You must specify the variable (and its domain) and the leadtimes you want to compute ' 'the percentiles on') - if num_options > 3: - raise Exception('You must specify three parameters for the climatological percentiles') + if num_options > 4: + raise Exception('You must specify between three and 4 parameters for the climatological percentiles') domain = Domain(options[1]) variable = options[2] leadtimes = [int(i) for i in options[3].split('-')] + if num_options > 3: + num_bins = int(options[4]) + else: + num_bins = 2000 job_list = list() - job_list.append(ClimatologicalPercentile(diags.data_manager, domain, variable, leadtimes, + job_list.append(ClimatologicalPercentile(diags.data_manager, domain, variable, leadtimes, num_bins, diags.config.experiment)) return job_list @@ -98,16 +102,16 @@ class ClimatologicalPercentile(Diagnostic): percentile_var = handler.createVariable('percentile', float, ('percentile',)) percentile_var[:] = self.percentiles - handler.createDimension('lat', self.lat_len) + handler.createDimension('lat', self.lat.size) lat_var = handler.createVariable('lat', float, ('lat',)) lat_var[:] = self.lat - handler.createDimension('lon', self.lon_len) + handler.createDimension('lon', self.lon.size) lon_var = handler.createVariable('lon', float, ('lon',)) lon_var[:] = self.lon - p75_var = handler.createVariable('percent', float, ('percentile', 'lat', 'lon')) - p75_var[...] = percentile_values + percentile_var = handler.createVariable('percent', float, ('percentile', 'lat', 'lon')) + percentile_var[...] = percentile_values handler.close() @@ -117,14 +121,14 @@ class ClimatologicalPercentile(Diagnostic): def _calculate_percentiles(self, distribution): Log.debug('Calculating percentiles') - def calculate_percentiles(point_distribution): + def calculate(point_distribution): cs = np.cumsum(point_distribution) total = cs[-1] percentile_values = self.percentiles * total index = np.searchsorted(cs, percentile_values) return [(self._bins[i + 1] + self._bins[i]) / 2 for i in index] - distribution = np.apply_along_axis(calculate_percentiles, 0, distribution) + distribution = np.apply_along_axis(calculate, 0, distribution) return distribution def _get_distribution(self, member_files): @@ -189,18 +193,16 @@ class ClimatologicalPercentile(Diagnostic): return histogram var = handler.variables[self.variable] - return np.apply_along_axis(calculate_histogram, 0, var[:, realization, ...]) + if 'realization' in var.dimensions or 'ensemble' in var.dimensions: + return np.apply_along_axis(calculate_histogram, 0, var[:, realization, ...]) + else: + return np.apply_along_axis(calculate_histogram, 0, var[:]) def _get_var_size(self, handler): if self.lat_len is not None: return - self.lat = handler.variables['latitude'][:] self.lon = handler.variables['longitude'][:] - self.lat = handler.dimensions['latitude'].size - self.lon = handler.dimensions['longitude'].size - - diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 68a3fcd4094b17e8f71802382d660767997978b5..d746911e9879978f3471b3b1751e6f3b493741ed 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -1,10 +1,13 @@ # coding=utf-8 import shutil +from autosubmit.config.log import Log + from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable import Domain, VarType from calendar import monthrange +import numpy as np class MonthlyPercentile(Diagnostic): @@ -41,7 +44,8 @@ class MonthlyPercentile(Diagnostic): def __str__(self): return 'Monthly percentile {0} Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable) + 'Variable: {3}:{4} Percentile: {5}'.format(self.startdate, self.member, self.chunk, + self.domain, self.variable, self.percentile) @classmethod def generate_jobs(cls, diags, options): @@ -104,8 +108,16 @@ class MonthlyPercentile(Diagnostic): Utils.rename_variable(temp, 'lev', 'ensemble', False, True) shutil.move(temp, variable_file) - Utils.cdo.monpctl(str(self.percentile), input=[variable_file, '-monmin ' + variable_file, - '-monmax ' + variable_file], output=temp) + Log.debug('Computing minimum') + monmin_file = TempFile.get() + Utils.cdo.monmin(input=variable_file, output=monmin_file) + + Log.debug('Computing maximum') + monmax_file = TempFile.get() + Utils.cdo.monmax(input=variable_file, output=monmax_file) + + Log.debug('Computing percentile') + Utils.cdo.monpctl(str(self.percentile), input=[variable_file, monmin_file, monmax_file], output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, self.member, self.chunk, frequency='mon', rename_var=self.variable, vartype=VarType.STATISTIC) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index ded8b5571b4a2ef202fd6c977646111597410040..ae8acb004b6ecbad6c41e67e0b00d3e3a4629d59 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,10 +1,10 @@ # coding=utf-8 import os -from autosubmit.date.chunk_date_lib import parse_date, add_months +from autosubmit.date.chunk_date_lib import parse_date, add_months, chunk_start_date, chunk_end_date, date2str from earthdiagnostics.datamanager import DataManager, NetCDFFile from earthdiagnostics.utils import TempFile, Utils -import urllib +from datetime import datetime from earthdiagnostics.variable import Variable, VarType @@ -19,7 +19,7 @@ class THREDDSManager(DataManager): data_folders = self.config.data_dir.split(':') self.config.data_dir = None for data_folder in data_folders: - if os.path.isdir(os.path.join(data_folder, self.experiment.institute.lower(), + if os.path.isdir(os.path.join(data_folder, self.config.data_type, self.experiment.institute.lower(), self.experiment.model.lower())): self.config.data_dir = data_folder break @@ -27,16 +27,26 @@ class THREDDSManager(DataManager): if not self.config.data_dir: raise Exception('Can not find model data') + if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size !=1 : + raise Exception('For obs and recon data chunk_size must be always 1') + def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VarType.MEAN): - if not frequency: - frequency = self.config.frequency - aggregation_path = self.get_var_url(variable, startdate, frequency, None, False, vartype) - temp = TempFile.get() + + aggregation_path = self.get_var_url(variable, startdate, frequency, None, vartype) startdate = parse_date(startdate) + start_chunk = chunk_start_date(startdate, self.experiment.num_chunks, self.experiment.chunk_size, + 'month', 'standard') + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', 'standard') + + thredds_subset = THREDDSSubset(aggregation_path, variable, startdate, end_chunk).get_url() selected_months = ','.join([str(add_months(startdate, i, 'standard').month) for i in leadtimes]) - select_months = '-selmonth,{0} {1}'.format(selected_months, aggregation_path) - selected_years = ','.join([str(add_months(startdate, i, 'standard').year) for i in leadtimes]) - Utils.cdo.selyear(selected_years, input=select_months, output=temp) + temp = TempFile.get() + if self.config.data_type == 'exp': + select_months = '-selmonth,{0} {1}'.format(selected_months, thredds_subset) + selected_years = ','.join([str(add_months(startdate, i, 'standard').year) for i in leadtimes]) + Utils.cdo.selyear(selected_years, input=select_months, output=temp) + else: + Utils.cdo.selmonth(selected_months, input=thredds_subset, output=temp) return temp def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, @@ -63,14 +73,14 @@ class THREDDSManager(DataManager): :return: path to the copy created on the scratch folder :rtype: str """ - if not frequency: - frequency = self.config.frequency - aggregation_path = self.get_var_url(var, startdate, frequency, box, True, vartype) - temp = TempFile.get() - urllib.urlretrieve(aggregation_path, temp) - if not Utils.check_netcdf_file(temp): - raise THREDDSError('Can not retrieve {0} from server'.format(aggregation_path)) - return temp + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) + + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', 'standard') + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', 'standard') + + thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) + return thredds_subset.download() + def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, @@ -160,23 +170,26 @@ class THREDDSManager(DataManager): var = self._get_final_var_name(box, var) folder_path = self._get_folder_path(frequency, domain, var, grid, vartype) - if startdate: - file_name = '{0}_{1}.nc'.format(var, startdate) - else: - file_name = '{0}.nc'.format(var) + file_name = self._get_file_name(startdate, var) filepath = os.path.join(folder_path, file_name) return filepath def _get_folder_path(self, frequency, domain, variable, grid, vartype): - folder_path = os.path.join(self.config.data_dir, + + if self.config.data_type == 'exp': + var_folder = self.get_varfolder(domain, variable, grid) + else: + var_folder = variable + + folder_path = os.path.join(self.config.data_dir, self.config.data_type, self.experiment.institute.lower(), self.experiment.model.lower(), self.frequency_folder_name(frequency, vartype), - self.get_varfolder(domain, variable, grid)) + var_folder) return folder_path - def get_year(self, domain, var, startdate, member, year, grid=None, box=None): + def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VarType.MEAN): """ Ge a file containing all the data for one year for one variable :param domain: variable's domain @@ -195,19 +208,33 @@ class THREDDSManager(DataManager): :type box: Box :return: """ + aggregation_path = self.get_var_url(var, startdate, None, box, vartype) + thredds_subset = THREDDSSubset(aggregation_path, var, datetime(year, 1, 1), datetime(year+1, 1, 1)) + return thredds_subset.download() - def get_var_url(self, var, startdate, frequency, box, fileserver, vartype): + + def get_var_url(self, var, startdate, frequency, box, vartype): + if not frequency: + frequency = self.config.frequency var = self._get_final_var_name(box, var) - if fileserver: - protocol = 'fileServer' + full_path = os.path.join(self.server_url, 'dodsC', self.config.data_type, self.experiment.institute, + self.experiment.model, self.frequency_folder_name(frequency, vartype)) + if self.config.data_type == 'exp': + full_path = os.path.join(full_path, var, self._get_file_name(startdate, var)) else: - protocol = 'dodsC' - return os.path.join(self.server_url, protocol, 'exp', self.experiment.institute, - self.experiment.model, self.frequency_folder_name(frequency, vartype), - var, '{0}_{1}.nc'.format(var, startdate)) + full_path = os.path.join(full_path, self._get_file_name(None, var)) + return full_path + + def _get_file_name(self, startdate, var): + if startdate: + if self.config.data_type != 'exp': + startdate = startdate[0:6] + return '{0}_{1}.nc'.format(var, startdate) + else: + return '{0}.nc'.format(var) def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, - frequency=None, year=None, date_str=None, move_old=False): + frequency=None, year=None, date_str=None, move_old=False, vartype=VarType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -241,3 +268,77 @@ class THREDDSManager(DataManager): class THREDDSError(Exception): pass + +class THREDDSSubset: + def __init__(self, thredds_path, var, start_time, end_time): + self.thredds_path = thredds_path + self.var = var + self.dimension_indexes = {} + self.handler = None + self.start_time = start_time + self.end_time = end_time + + def get_url(self): + self.handler = Utils.openCdf(self.thredds_path) + self._read_metadata() + self.handler.close() + + self._get_time_indexes() + + return self._get_subset_url() + + def download(self): + url = self.get_url() + return self._download_url(url) + + def _read_metadata(self): + self.var_dimensions = self.handler.variables[self.var].dimensions + for dimension in self.var_dimensions: + if dimension == 'time': + continue + self.dimension_indexes[dimension] = (0, self.handler.dimensions[dimension].size - 1) + + if 'time' in self.var_dimensions: + self.times = Utils.get_datetime_from_netcdf(self.handler) + + def _get_time_indexes(self): + if 'time' not in self.var_dimensions: + return + + time_start = 0 + while time_start < self.times.size and self.times[time_start] < self.start_time: + time_start += 1 + if time_start == self.times.size: + raise Exception('Timesteps not available for interval {0}-{1}'.format(self.start_time, self.end_time)) + time_end = time_start + if self.times[time_end] >= self.end_time: + raise Exception('Timesteps not available for interval {0}-{1}'.format(self.start_time, self.end_time)) + while time_end < self.times.size - 1 and self.times[time_end + 1] < self.end_time: + time_end += 1 + self.dimension_indexes['time'] = (time_start, time_end) + + def _download_url(self, url): + temp = TempFile.get() + Utils.execute_shell_command(['nccopy', url, temp]) + if not Utils.check_netcdf_file(temp): + raise THREDDSError('Can not retrieve {0} from server'.format(url)) + return temp + + def _get_subset_url(self): + var_slice = self.var + dimensions_slice = '' + + for dimension in self.var_dimensions: + slice_index = self._get_slice_index(self.dimension_indexes[dimension]) + var_slice += slice_index + if dimension == 'ensemble': + dimension = 'realization' + dimensions_slice += '{0}{1},'.format(dimension, slice_index) + + return '{0}?{1}{2}'.format(self.thredds_path, dimensions_slice, var_slice) + + def _get_slice_index(self, index_tuple): + return '[{0[0]}:1:{0[1]}]'.format(index_tuple) + + + diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 440c4c30c67d1cde271e43b09e66a474ce7b7d8d..7e545c6d507cffdcdac456b70709e8ecbd26749f 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -130,7 +130,9 @@ class Utils(object): error = True if error: + Log.info('First attemp to rename failed. Using secondary rename method for netCDF') Utils._rename_vars_by_creating_new_file(dic_names, filepath, temp) + Log.info('Rename done') Utils.move_file(temp, filepath) @@ -151,11 +153,11 @@ class Utils(object): @staticmethod def _rename_vars_by_creating_new_file(dic_names, filepath, temp): - Log.debug('Using secondary rename method for netCDF') original_handler = Utils.openCdf(filepath) new_handler = Utils.openCdf(temp, 'w') for attribute in original_handler.ncattrs(): - setattr(new_handler, attribute, getattr(original_handler, attribute)) + original = getattr(original_handler, attribute) + setattr(new_handler, attribute, Utils.convert_to_ASCII_if_possible(original)) for dimension in original_handler.dimensions.keys(): Utils.copy_dimension(original_handler, new_handler, dimension, new_names=dic_names) for variable in original_handler.variables.keys(): @@ -163,6 +165,17 @@ class Utils(object): original_handler.close() new_handler.close() + @staticmethod + def convert_to_ASCII_if_possible(string, encoding='ascii'): + if isinstance(string, basestring): + try: + return string.encode(encoding) + except UnicodeEncodeError: + if u'Bretonnière' in string: + string = string.replace(u'Bretonnière', 'Bretonniere') + return Utils.convert_to_ASCII_if_possible(string, encoding) + return string + @staticmethod def _rename_vars_directly(dic_names, filepath, handler, must_exist, rename_dimension): for old_name, new_name in dic_names.items(): @@ -419,7 +432,8 @@ class Utils(object): return original_var = source.variables[variable] new_var = destiny.createVariable(new_name, original_var.datatype, translated_dimensions) - new_var.setncatts({k: original_var.getncattr(k) for k in original_var.ncattrs()}) + new_var.setncatts({k: Utils.convert_to_ASCII_if_possible(original_var.getncattr(k)) + for k in original_var.ncattrs()}) new_var[:] = original_var[:] @staticmethod diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 70eb8bb0f9cc0d1c4898f2c694293e6f4207919d..faf7c74a601680593b1ea80c47952248249b1365 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -110,7 +110,7 @@ class Domains(object): seaIce = Domain('seaice') ocean = Domain('ocean') landIce = Domain('landIce') - atmos = Domain('seaice') + atmos = Domain('atmos') land = Domain('land') diff --git a/test/unit/test_averagesection.py b/test/unit/test_averagesection.py index e9b4816cf976b6d9f8e2659ba4e87c5904033e9f..01d32184a19c4686015f9aeb24e95347c860ea41 100644 --- a/test/unit/test_averagesection.py +++ b/test/unit/test_averagesection.py @@ -5,6 +5,8 @@ from earthdiagnostics.box import Box from earthdiagnostics.ocean.averagesection import AverageSection from mock import Mock +from earthdiagnostics.variable import Domains, Domain + class TestAverageSection(TestCase): @@ -19,18 +21,18 @@ class TestAverageSection(TestCase): self.box.max_lon = 0 self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.psi = AverageSection(self.data_manager, '20000101', 1, 1, 'domain', 'var', self.box) + self.psi = AverageSection(self.data_manager, '20000101', 1, 1, Domains.ocean, 'var', self.box) def test_generate_jobs(self): jobs = AverageSection.generate_jobs(self.diags, ['psi', 'var', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, 'ocean', 'var', self.box)) - self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, 'ocean', 'var', self.box)) + self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', self.box)) + self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', self.box)) - jobs = AverageSection.generate_jobs(self.diags, ['psi', 'var', '0', '0', '0', '0', 'domain']) + jobs = AverageSection.generate_jobs(self.diags, ['psi', 'var', '0', '0', '0', '0', 'ocean']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, 'domain', 'var', self.box)) - self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, 'domain', 'var', self.box)) + self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', self.box)) + self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', self.box)) with self.assertRaises(Exception): AverageSection.generate_jobs(self.diags, ['psi']) @@ -39,4 +41,4 @@ class TestAverageSection(TestCase): def test_str(self): self.assertEquals(str(self.psi), 'Average section Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0E ' - 'Variable: domain:var') + 'Variable: ocean:var') diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py index 9fd8bb387dcca55354d1661510eded81ccd41b2f..0d0fadb85cec7192220fde39d2d72856b94976c4 100644 --- a/test/unit/test_cdftools.py +++ b/test/unit/test_cdftools.py @@ -1,5 +1,8 @@ # coding=utf-8 from unittest import TestCase + +import os + from earthdiagnostics.cdftools import CDFTools import mock @@ -11,33 +14,37 @@ class TestCDFTools(TestCase): # noinspection PyTypeChecker def test_run(self): + def mock_exists(path, access=None): + """ + Function for os.path.exists mock + :param path: path to check + :type path: str + :return: true if path does not start with 'bad' + :rtype: bool + """ + return not os.path.basename(path.startswith('bad')) + with mock.patch('os.path.exists') as exists_mock: - def mock_exists(path): - """ - Function for os.path.exists mock - :param path: path to check - :type path: str - :return: true if path does not start with 'bad' - :rtype: bool - """ - return not path.startswith('bad') - exists_mock.side_effect = mock_exists - with mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') as execute_mock: - execute_mock.return_value = ['Command output'] - with self.assertRaises(ValueError): - self.cdftools.run('badcommand', input='input_file', output='output_file') - with self.assertRaises(ValueError): - self.cdftools.run('command', input='badinput_file', output='output_file') - with self.assertRaises(ValueError): - self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') - with self.assertRaises(ValueError): - self.cdftools.run('command', input='input_file', output='input_file') - with self.assertRaises(Exception): - self.cdftools.run('command', input='input_file', output='badoutput_file') - - self.cdftools.run('command', input='input_file', output='output_file') - self.cdftools.run('command', input='input_file') - self.cdftools.run('command', input=None) - self.cdftools.run('command', input=['input_file', 'input_file2']) - self.cdftools.run('command', input='input_file', options='-o -p') - self.cdftools.run('command', input='input_file', options=('-o', '-p')) + with mock.patch('os.access') as access_mock: + exists_mock.side_effect = mock_exists + access_mock.side_effect = mock_exists + + with mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') as execute_mock: + execute_mock.return_value = ['Command output'] + with self.assertRaises(ValueError): + self.cdftools.run('badcommand', input='input_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='badinput_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='input_file', output='input_file') + with self.assertRaises(Exception): + self.cdftools.run('command', input='input_file', output='badoutput_file') + + self.cdftools.run('command', input='input_file', output='output_file') + self.cdftools.run('command', input='input_file') + self.cdftools.run('command', input=None) + self.cdftools.run('command', input=['input_file', 'input_file2']) + self.cdftools.run('command', input='input_file', options='-o -p') + self.cdftools.run('command', input='input_file', options=('-o', '-p')) diff --git a/test/unit/test_cutsection.py b/test/unit/test_cutsection.py index 060c4a4adab76a7c3ad7ca58329f42dc1fba0fac..6f4d4aa276b24d58fb267ddecef37ca03c64e6a6 100644 --- a/test/unit/test_cutsection.py +++ b/test/unit/test_cutsection.py @@ -5,6 +5,8 @@ from earthdiagnostics.box import Box from earthdiagnostics.ocean.cutsection import CutSection from mock import Mock +from earthdiagnostics.variable import Domains + class TestCutSection(TestCase): @@ -19,18 +21,18 @@ class TestCutSection(TestCase): self.box.max_lon = 0 self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.psi = CutSection(self.data_manager, '20000101', 1, 1, 'domain', 'var', True, 0) + self.psi = CutSection(self.data_manager, '20000101', 1, 1, Domains.atmos, 'var', True, 0) def test_generate_jobs(self): jobs = CutSection.generate_jobs(self.diags, ['psi', 'var', 'true', '10']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, 'ocean', 'var', True, 10)) - self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, 'ocean', 'var', True, 10)) + self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', True, 10)) + self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', True, 10)) - jobs = CutSection.generate_jobs(self.diags, ['psi', 'var', 'false', '0', 'domain']) + jobs = CutSection.generate_jobs(self.diags, ['psi', 'var', 'false', '0', 'atmos']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, 'domain', 'var', False, 0)) - self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, 'domain', 'var', False, 0)) + self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', False, 0)) + self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', False, 0)) with self.assertRaises(Exception): CutSection.generate_jobs(self.diags, ['psi']) @@ -38,5 +40,5 @@ class TestCutSection(TestCase): CutSection.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: domain:var ' + self.assertEquals(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' 'Zonal: True Value: 0') diff --git a/test/unit/test_interpolate.py b/test/unit/test_interpolate.py index 9a092cfdfca8254081920ad56a6dba1e28ec8328..480a230d86ea5bf6f58a9db10a4559f09ca4ba66 100644 --- a/test/unit/test_interpolate.py +++ b/test/unit/test_interpolate.py @@ -4,6 +4,8 @@ from unittest import TestCase from earthdiagnostics.ocean.interpolate import Interpolate from mock import Mock +from earthdiagnostics.variable import Domains + class TestInterpolate(TestCase): @@ -15,29 +17,29 @@ class TestInterpolate(TestCase): self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.diags.config.experiment.model_version = 'model_version' - self.interpolate = Interpolate(self.data_manager, '20000101', 1, 1, 'domain', 'var', 'grid', 'model_version', - False) + self.interpolate = Interpolate(self.data_manager, '20000101', 1, 1, Domains.atmos, 'var', 'grid', + 'model_version', False) def test_generate_jobs(self): jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, 'ocean', 'var', 'grid', + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', 'grid', 'model_version', False)) - self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, 'ocean', 'var', 'grid', + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', 'grid', 'model_version', False)) - jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'domain']) + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, 'domain', 'var', 'grid', + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'grid', 'model_version', False)) - self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, 'domain', 'var', 'grid', + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'grid', 'model_version', False)) - jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'domain', 'true']) + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, 'domain', 'var', 'grid', + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'grid', 'model_version', True)) - self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, 'domain', 'var', 'grid', + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'grid', 'model_version', True)) with self.assertRaises(Exception): @@ -48,5 +50,5 @@ class TestInterpolate(TestCase): def test_str(self): self.assertEquals(str(self.interpolate), 'Interpolate Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: domain:var Target grid: grid Invert lat: False ' + 'Variable: atmos:var Target grid: grid Invert lat: False ' 'Model: model_version') diff --git a/test/unit/test_monthlymean.py b/test/unit/test_monthlymean.py index 669490898301aff7e1c6c43ca77cfedae3255ce3..128d411fe59f0c68201838df9c6a2e12db53d32b 100644 --- a/test/unit/test_monthlymean.py +++ b/test/unit/test_monthlymean.py @@ -5,6 +5,8 @@ from earthdiagnostics.box import Box from earthdiagnostics.general.monthlymean import MonthlyMean from mock import Mock +from earthdiagnostics.variable import Domains + class TestMonthlyMean(TestCase): @@ -19,24 +21,26 @@ class TestMonthlyMean(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, 'domain', 'var', 'freq', '') + self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, Domains.ocean, 'var', 'freq', '') def test_generate_jobs(self): - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'domain']) + jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'ocean']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, 'domain', 'var', 'day', '')) - self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, 'domain', 'var', 'day', '')) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', 'day', '')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', 'day', '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'domain', 'freq']) + jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'atmos', 'freq']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, 'domain', 'var', 'freq', '')) - self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, 'domain', 'var', 'freq', '')) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'freq', '')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'freq', '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'domain', 'freq', 'grid']) + jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'seaice', 'freq', 'grid']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, 'domain', 'var', 'freq', 'grid')) - self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, 'domain', 'var', 'freq', 'grid')) + self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, Domains.seaIce, 'var', 'freq', + 'grid')) + self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, Domains.seaIce, 'var', 'freq', + 'grid')) with self.assertRaises(Exception): MonthlyMean.generate_jobs(self.diags, ['psi']) @@ -46,4 +50,4 @@ class TestMonthlyMean(TestCase): def test_str(self): self.assertEquals(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: domain:var') + 'Variable: ocean:var') diff --git a/test/unit/test_rewrite.py b/test/unit/test_rewrite.py index 3cd0a7441cd99de6d08a979ae137111fac4a5389..13adc6857340a57bfd0de0c65a6d021897d90537 100644 --- a/test/unit/test_rewrite.py +++ b/test/unit/test_rewrite.py @@ -5,6 +5,8 @@ from earthdiagnostics.box import Box from earthdiagnostics.general.rewrite import Rewrite from mock import Mock +from earthdiagnostics.variable import Domains + class TestRewrite(TestCase): @@ -19,14 +21,19 @@ class TestRewrite(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.mixed = Rewrite(self.data_manager, '20000101', 1, 1, 'domain', 'var') + self.mixed = Rewrite(self.data_manager, '20000101', 1, 1, Domains.atmos, 'var', 'grid') def test_generate_jobs(self): - jobs = Rewrite.generate_jobs(self.diags, ['psi', 'var', 'domain']) + jobs = Rewrite.generate_jobs(self.diags, ['psi', 'var', 'atmos']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, Domains.atmos, 'var', 'original')) + self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, Domains.atmos, 'var', 'original')) + + jobs = Rewrite.generate_jobs(self.diags, ['psi', 'var', 'ocean', 'grid']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, 'domain', 'var')) - self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, 'domain', 'var')) + self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, Domains.ocean, 'var', 'grid')) + self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, Domains.ocean, 'var', 'grid')) with self.assertRaises(Exception): Rewrite.generate_jobs(self.diags, ['psi']) @@ -36,4 +43,4 @@ class TestRewrite(TestCase): def test_str(self): self.assertEquals(str(self.mixed), 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: domain:var') + 'Variable: atmos:var Grid: grid') diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 603001129c7350292b575767075ff3fe10d16e39..70664ab41837731c0925b21b35b0f64376ad8f9f 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -58,53 +58,3 @@ class TestUtils(TestCase): rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True, False), mock.call('file', {'old': 'new'}, False, True))) - def test_rename_variables(self): - mock_handler = mock.Mock() - mock_handler.variables = dict() - mock_handler.dimensions = dict() - mock_handler.variables['old'] = mock.Mock() - mock_handler.variables['old_var'] = mock.Mock() - mock_handler.dimensions['old'] = mock.Mock() - mock_handler.ncattrs.return_value = 'attribute' - mock_handler.attribute = 'value' - - with mock.patch('earthdiagnostics.utils.Utils.openCdf') as opencdf_mock: - with mock.patch('shutil.copyfile'): - with mock.patch('earthdiagnostics.utils.Utils.move_file'): - opencdf_mock.return_value = mock_handler - Utils.rename_variables('file', {'old': 'old_var'}) - Utils.rename_variables('file', {'old2': 'new'}, False) - Utils.rename_variables('file', {'old2': 'new', 'old': 'new'}, False) - Utils.rename_variables('file', {'old': 'new'}, False, True) - Utils.rename_variables('file', {'old_var': 'new'}, False, True) - - with self.assertRaises(ValueError): - Utils.rename_variables('file', {'new': 'new'}) - with self.assertRaises(Exception): - Utils.rename_variables('file', {'old2': 'new'}) - with self.assertRaises(Exception): - Utils.rename_variables('file', {'old2': 'new', 'old': 'new'}) - with self.assertRaises(Exception): - Utils.rename_variables('file', {'old_var': 'new'}, rename_dimension=True) - - def test_convert2netcdf4(self): - mock_handler = mock.Mock() - - with mock.patch('earthdiagnostics.utils.Utils.openCdf') as opencdf_mock: - with mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') as execute_mock: - with mock.patch('earthdiagnostics.utils.TempFile.get') as tempfile_mock: - with mock.patch('shutil.move'): - tempfile_mock.return_value = 'tempfile' - opencdf_mock.return_value = mock_handler - mock_handler.file_format = 'NETCDF4' - Utils.convert2netcdf4('file') - - mock_handler.file_format = 'OTHER' - Utils.convert2netcdf4('file2') - execute_mock.assert_called_with(['nccopy', '-4', '-d4', '-s', 'file2', 'tempfile']) - - mock_handler.file_format = 'NETCDF4' - Utils.convert2netcdf4('file3') - execute_mock.assert_called_with(['nccopy', '-4', '-d4', '-s', 'file3', 'tempfile']) - - self.assertEqual(execute_mock.call_count, 2) diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 67d68dc6f38f78ee29cfb3b0e4a101d0cc6cf5c5..ffc3d037acb845a4ed1ef315849029ab053400fe 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -1,27 +1,28 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.variable import Variable +from earthdiagnostics.variable import Variable, Domains class TestVariable(TestCase): def test__init__(self): - variable = Variable('alias:alias2,name,standard_name,long_name,domain,basin,units,' - 'valid_min,valid_max'.split(',')) + variable = Variable('alias:alias2,name,standard_name,long_name,ocean,basin,units,' + 'valid_min,valid_max,grid'.split(',')) self.assertEqual(variable.short_name, 'name') self.assertEqual(variable.standard_name, 'standard_name') self.assertEqual(variable.long_name, 'long_name') - self.assertEqual(variable.domain, 'domain') + self.assertEqual(variable.domain, Domains.ocean) self.assertEqual(variable.basin, None) self.assertEqual(variable.units, 'units') self.assertEqual(variable.valid_min, 'valid_min') self.assertEqual(variable.valid_max, 'valid_max') + self.assertEqual(variable.grid, 'grid') def test_get_variable(self): Variable._dict_variables = dict() - variable = Variable('alias:alias2,name,standard_name,long_name,domain,basin,units,valid_min,' - 'valid_max'.split(',')) + variable = Variable('alias:alias2,name,standard_name,long_name,atmos,basin,units,valid_min,' + 'valid_max,grid'.split(',')) Variable._dict_variables['var'] = variable self.assertIs(Variable.get_variable('var'), variable) self.assertIsNone(Variable.get_variable('novar'))