diff --git a/.gitignore b/.gitignore index cafa800b40cfb7157c94ee50969e873bfa644429..7352c474ab455518a6160f26706828691e6a3064 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ *.pyc .idea/* doc/build/* +test/report/* *.err *.out *.nc .coverage -htmlcov \ No newline at end of file +htmlcov +.pytest_cache +prof diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdc3755229b9f14f9814b69c9093ca6ade6c2064..c5b843a22f1041193551e84ad2f3fac189c06f20 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,19 +1,50 @@ before_script: - export GIT_SSL_NO_VERIFY=1 - - git submodule sync --recursive - - git submodule update --init --recursive - export PATH="$HOME/miniconda2/bin:$PATH" +stages: + - prepare + - test + - report + - clean + +cache: + paths: + - test/report + +prepare: + stage: prepare + script: + - conda update conda + test_python2: + stage: test script: - - conda env update -f environment.yml -n earthdiagnostics2 python=2 - - source activate earthdiagnostics - - coverage run -m unittest discover + - git submodule sync --recursive + - git submodule update --init --recursive + - conda env update -f environment.yml -n earthdiagnostics2 python=2.7 + - source activate earthdiagnostics2 + - python run_test.py test_python3: + stage: test + script: + - git submodule sync --recursive + - git submodule update --init --recursive + - conda env update -f environment.yml -n earthdiagnostics3 python=3.6 + - source activate earthdiagnostics3 + - python run_test.py + +report_codacy: + stage: report + script: + - source activate earthdiagnostics3 + - pip install codacy-coverage --upgrade + - python-codacy-coverage -r test/report/python3/coverage.xml + +clean: + stage: clean script: - - conda env update -f environment.yml -n earthdiagnostics3 python=3 - - source activate earthdiagnostics - - coverage run -m unittest discover - - coverage xml - - python-codacy-coverage -r coverage.xml + - conda clean --all --yes + + diff --git a/bin/earthdiags b/bin/earthdiags index 174528e3210ded69220e9aae3d5e05a614f09d90..200086ee283ea3c8b0eb15435d3a2d347dbc238d 100644 --- a/bin/earthdiags +++ b/bin/earthdiags @@ -18,7 +18,7 @@ def main(): """ Entry point for the Earth Diagnostics """ - if not EarthDiags.parse_args(): + if not EarthDiags.parse_args(sys.argv[1:]): os._exit(1) os._exit(0) diff --git a/diags.conf b/diags.conf index 6ea8d8c74fe2dca09fb218993b74041ee3c3292a..64dc8269cdee95d1bd35d4d52d236d018205fcb3 100644 --- a/diags.conf +++ b/diags.conf @@ -1,26 +1,26 @@ [DIAGNOSTICS] # Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) -DATA_ADAPTOR = CMOR +DATA_ADAPTOR = OBSRECON # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use -DATA_DIR = /esnas:/esarchive +DATA_DIR = /esarchive # Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) -DATA_TYPE = exp +DATA_TYPE = recon # CMORization type to use. Important also for THREDDS as it affects variable name conventions. # Options: SPECS (default), PRIMAVERA, CMIP6 -DATA_CONVENTION = PRIMAVERA +DATA_CONVENTION = SPECS # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = regmean,ocean,thetao +DIAGS = interpcdo,atmos,prlr,r240x121,bilinear,False,,False # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = mon +FREQUENCY = weekly # Path to CDFTOOLS binaries CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir @@ -73,11 +73,11 @@ SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = EC-Earth-Consortium -MODEL = EC-Earth3-HR -NAME = historical +INSTITUTE = gloh2o +MODEL = mswep +NAME = wekkly_means # Model version: Available versions -MODEL_VERSION =Ec3.2_O25L75 +MODEL_VERSION = # Atmospheric output timestep in hours ATMOS_TIMESTEP = 3 # Ocean output timestep in hours @@ -91,9 +91,10 @@ OCEAN_TIMESTEP = 3 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = a0n8 -STARTDATES = 19900101 -MEMBERS = fc0 +EXPID = mswep +# STARTDATES = {19970101,20161231,1D} +STARTDATES = 19970322 20010425 +MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 CHUNKS = 1 diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 1e2d718bddf49812b2a010c1701eca705c08b3ec..42099f32ab84ffa4db1c9c4ceae56e473e766dea 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -4,6 +4,8 @@ import glob import os import shutil import uuid +import traceback +import pygrib from datetime import datetime from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months @@ -63,6 +65,11 @@ class Cmorizer(object): 'time_counter_bounds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': self.lat_name, 'nav_lon': self.lon_name, 'x': 'i', 'y': 'j'} + @property + def path_icm(self): + """Path to the ICM file""" + return os.path.join(self.config.scratch_dir, 'ICM') + def cmorize_ocean(self): """Cmorize ocean files from MMO files""" if not self.cmor.ocean: @@ -119,7 +126,7 @@ class Cmorizer(object): def _cmorize_nc_files(self): nc_files = glob.glob(os.path.join(self.cmor_scratch, '*.nc')) - for filename in nc_files: + for filename in self._filter_files(nc_files): self._cmorize_nc_file(filename) self._clean_cmor_scratch() @@ -156,11 +163,12 @@ class Cmorizer(object): Log.debug('Moving file {0}', filepath) shutil.move(filepath, filepath.replace('/backup/', '/')) zip_files = glob.glob(os.path.join(self.cmor_scratch, '*.gz')) - for zip_file in self._filter_files(zip_files): - try: - Utils.unzip(zip_file) - except Utils.UnzipException as ex: - Log.error('File {0} could not be unzipped: {1}', tarfile, ex) + if zip_files: + for zip_file in self._filter_files(zip_files): + try: + Utils.unzip(zip_file) + except Utils.UnzipException as ex: + Log.error('File {0} could not be unzipped: {1}', tarfile, ex) def _clean_cmor_scratch(self): if os.path.exists(self.cmor_scratch): @@ -181,7 +189,7 @@ class Cmorizer(object): Utils.cdo.mergetime(input=gg_files, output=merged_gg) for filename in sh_files + gg_files: os.remove(filename) - tar_startdate = tarfile[0:-4].split('_')[5].split('-') + tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') shutil.move(merged_gg, os.path.join(self.cmor_scratch, 'MMAGG_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) shutil.move(merged_sh, os.path.join(self.cmor_scratch, 'MMASH_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) @@ -216,7 +224,7 @@ class Cmorizer(object): self._cmorize_nc_files() Log.result('Atmospheric file {0}/{1} finished'.format(count, len(tar_files))) except Exception as ex: - Log.error('Could not cmorize atmospheric file {0}: {1}', count, ex) + Log.error('Could not cmorize atmospheric file {0}: {1}\n {2}', count, ex, traceback.format_exc()) count += 1 @@ -235,7 +243,13 @@ class Cmorizer(object): for grid in ('SH', 'GG'): Log.info('Processing {0} variables', grid) - if not os.path.exists(self._get_original_grib_path(chunk_start, grid)): + first_grib = self._get_original_grib_path(chunk_start, grid) + if not os.path.exists(first_grib): + continue + var_list = Utils.cdo.showvar(input=first_grib)[0] + codes = {int(var.replace('var', '')) for var in var_list.split()} + if not codes.intersection(self.config.cmor.get_requested_codes()): + Log.info('No requested variables found in {0}. Skipping...', grid) continue self._cmorize_grib_file(chunk_end, chunk_start, grid) except Exception as ex: @@ -256,7 +270,7 @@ class Cmorizer(object): self._obtain_atmos_timestep(gribfile) full_file = self._get_monthly_grib(current_date, gribfile, grid) - if not self._unpack_grib(full_file, gribfile, grid): + if not self._unpack_grib(full_file, gribfile, grid, current_date.month): os.remove(gribfile) return @@ -278,29 +292,29 @@ class Cmorizer(object): self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, '{0}hr'.format(self.atmos_timestep)) - def _unpack_grib(self, full_file, gribfile, grid): + def _unpack_grib(self, full_file, gribfile, grid, month): Log.info('Unpacking... ') # remap on regular Gauss grid codes = self.cmor.get_requested_codes() if 228 in codes: - codes.update(142, 143) + codes.update((142, 143)) codes_str = ','.join([str(code) for code in codes]) try: if grid == 'SH': - Utils.cdo.splitparam(input='-sp2gpl -selcode,{0} {1} '.format(codes_str, full_file), + Utils.cdo.splitparam(input='-sp2gpl -selmon,{2} -selcode,{0} {1} '.format(codes_str, full_file, month), output=gribfile + '_', options='-f nc4') else: - Utils.cdo.splitparam(input='-selcode,{0} {1}'.format(codes_str, full_file), + Utils.cdo.splitparam(input='-selmon,{2} -selcode,{0} {1}'.format(codes_str, full_file, month), output=gribfile + '_', options='-R -f nc4') # total precipitation (remove negative values) if 228 in codes: Utils.cdo.setcode(228, input='-setmisstoc,0 -setvrange,0,Inf ' - '-add {0}_{{142,143}}.128.nc'.format(gribfile), + '-add {0}_142.128.nc {0}_143.128.nc'.format(gribfile), output='{0}_228.128.nc'.format(gribfile), options='-f nc4') return True @@ -308,13 +322,14 @@ class Cmorizer(object): Log.info('No requested codes found in {0} file'.format(grid)) return False finally: - Utils.remove_file('ICM') + Utils.remove_file(self.path_icm) def _get_monthly_grib(self, current_date, gribfile, grid): prev_gribfile = self._get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) if os.path.exists(prev_gribfile): self._merge_grib_files(current_date, prev_gribfile, gribfile) - full_file = 'ICM' + + full_file = self.path_icm else: full_file = gribfile return full_file @@ -335,15 +350,10 @@ class Cmorizer(object): def _get_atmos_timestep(self, gribfile): Log.info('Getting timestep...') - import pygrib grib_handler = pygrib.open(gribfile) dates = set() - try: - while True: - mes = grib_handler.next() - dates.add(mes.analDate) - except StopIteration: - pass + for mes in grib_handler: + dates.add(mes.validDate) dates = list(dates) dates.sort() atmos_timestep = dates[1] - dates[0] @@ -359,37 +369,37 @@ class Cmorizer(object): os.remove(filename) return - Utils.convert2netcdf4(filename) + # Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) Utils.rename_variables(filename, self.alt_coord_names, False, True) handler = Utils.open_cdf(filename) Cmorizer._remove_valid_limits(handler) self._add_common_attributes(handler, frequency) self._update_time_variables(handler) - handler.sync() + + variables = handler.variables.keys() + handler.close() Log.info('Splitting file {0}', filename) - for variable in handler.variables.keys(): + for variable in variables: if variable in Cmorizer.NON_DATA_VARIABLES: continue try: - self.extract_variable(filename, handler, frequency, variable) + self.extract_variable(filename, frequency, variable) except Exception as ex: Log.error('Variable {0} can not be cmorized: {1}', variable, ex) Log.result('File {0} cmorized!', filename) - handler.close() os.remove(filename) @staticmethod - def _remove_valid_limits(filename): - handler = Utils.open_cdf(filename) + def _remove_valid_limits(handler): for variable in handler.variables.keys(): var = handler.variables[variable] if 'valid_min' in var.ncattrs(): del var.valid_min if 'valid_max' in var.ncattrs(): del var.valid_max - handler.close() + handler.sync() def _get_nc_file_frequency(self, filename): file_parts = os.path.basename(filename).split('_') @@ -409,14 +419,13 @@ class Cmorizer(object): variables = Utils.get_file_variables(filename) return self.cmor.any_required(variables) - def extract_variable(self, file_path, handler, frequency, variable): + def extract_variable(self, file_path, frequency, variable): """ Extract a variable from a file and creates the CMOR file Parameters ---------- file_path:str - handler: netCDF4.Dataset frequency: Frequency variable: str @@ -437,7 +446,7 @@ class Cmorizer(object): Utils.nco.ncks(input=file_path, output=temp, options=('-v {0}'.format(variable),)) self._rename_level_variables(temp, var_cmor) - self._add_coordinate_variables(handler, temp) + self._add_coordinate_variables(file_path, temp) if alias.basin is None: region = None @@ -504,13 +513,15 @@ class Cmorizer(object): raise Exception('File {0} start date is not a valid chunk start date'.format(file_path)) return chunk - def _add_coordinate_variables(self, handler, temp): + def _add_coordinate_variables(self, file_path, temp): handler_cmor = Utils.open_cdf(temp) + handler = Utils.open_cdf(file_path, 'r') Utils.copy_variable(handler, handler_cmor, self.lon_name, False) Utils.copy_variable(handler, handler_cmor, self.lat_name, False) if 'time' in handler_cmor.dimensions.keys(): Utils.copy_variable(handler, handler_cmor, 'leadtime', False) handler_cmor.close() + handler.close() @staticmethod def _rename_level_variables(temp, var_cmor): @@ -523,19 +534,18 @@ class Cmorizer(object): if var_cmor.domain == ModelingRealms.atmos: Utils.rename_variables(temp, {'depth': 'plev'}, False, True) - @staticmethod - def _merge_grib_files(current_month, prev_gribfile, gribfile): + def _merge_grib_files(self, current_month, prev_gribfile, gribfile): Log.info('Merging data from different files...') - fd = open('rules_files', 'w') + rules_path = os.path.join(self.config.scratch_dir, 'rules_files') + fd = open(rules_path, 'w') fd.write('if (dataDate >= {0.year}{0.month:02}01) {{ write ; }}\n'.format(current_month)) fd.close() # get first timestep for each month from previous file (if possible) - if os.path.exists('ICM'): - os.remove('ICM') - Utils.execute_shell_command('grib_filter -o ICM rules_files ' - '{0} {1}'.format(os.path.basename(prev_gribfile), - os.path.basename(gribfile))) - os.remove('rules_files') + if os.path.exists(self.path_icm): + os.remove(self.path_icm) + Utils.execute_shell_command('grib_filter -o {2} {3} ' + '{0} {1}'.format(prev_gribfile, gribfile, self.path_icm, rules_path)) + os.remove(rules_path) Utils.remove_file(prev_gribfile) def _ungrib_vars(self, gribfile, month, frequency): @@ -548,21 +558,19 @@ class Cmorizer(object): continue new_units = None - cdo_operator = '-selmon,{0}'.format(month) + cdo_operator = '' cdo_operator = self._get_time_average(cdo_operator, frequency, var_code) - cdo_operator = self._fix_time_shift(cdo_operator, var_code) - cdo_operator, new_units = self._change_units(cdo_operator, new_units, var_code) levels = self.config.cmor.get_levels(frequency, var_code) if levels: cdo_operator = "{0} -sellevel,{1}".format(cdo_operator, levels) - Utils.execute_shell_command('cdo -t ecmwf setreftime,{0} ' - '{1} {2}_{3}.128.nc ' - '{2}_{3}_{4}.nc'.format(cdo_reftime, cdo_operator, - gribfile, var_code, frequency)) + Utils.cdo.setreftime(cdo_reftime, + input='{2} {0}_{1}.128.nc '.format(gribfile, var_code, cdo_operator), + output='{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency), + options='-t ecmwf') h_var_file = '{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency) handler = Utils.open_cdf(h_var_file) @@ -594,10 +602,10 @@ class Cmorizer(object): if var_code == 201: cdo_operator = "-monmean -daymax {0}".format(cdo_operator) elif var_code == 202: - cdo_operator = "-monmean -daymax {0}".format(cdo_operator) + cdo_operator = "-monmean -daymin {0}".format(cdo_operator) else: cdo_operator = "-monmean {0} ".format(cdo_operator) - if frequency == Frequencies.daily: + elif frequency == Frequencies.daily: if var_code == 201: cdo_operator = "-daymax {0} ".format(cdo_operator) elif var_code == 202: @@ -622,7 +630,7 @@ class Cmorizer(object): elif var_code in (144, 182, 205, 228): # precipitation/evaporation/runoff new_units = "kg m-2 s-1" - cdo_operator = "-mulc,1000 -divc,{0}".format(self.experiment.atmos_timestep * 3600) + cdo_operator = "-mulc,1000 -divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) return cdo_operator, new_units def _merge_and_cmorize_atmos(self, chunk_start, chunk_end, grid, frequency): @@ -665,8 +673,8 @@ class Cmorizer(object): startdate = parse_date(self.startdate) leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate for time in leadtime] - for lt in range(0, len(leadtime)): - var[lt] = leadtime[lt].days + for lt, lead in enumerate(leadtime): + var[lt] = lead.days def _add_common_attributes(self, handler, frequency): cmor = self.config.cmor diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 931f4ec73cc6dd87b85b89264f126b98056d392f..9d2fa84f22fd3f90888f4773449c7abe0958c411 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -496,8 +496,25 @@ class CMORManager(DataManager): return for startdate, member in self.experiment.get_member_list(): - if not self._unpack_cmor_files(startdate, member): - self._cmorize_member(startdate, member) + Log.info('Checking data for startdate {0} member {1}', startdate, member) + if not self.config.cmor.force: + cmorized = False + for chunk in range(1, self.experiment.num_chunks + 1): + if not self.config.cmor.chunk_cmorization_requested(chunk): + Log.debug('Skipping chunk {0}', chunk) + continue + if not self.config.cmor.force_untar: + Log.debug('Checking chunk {0}...', chunk) + for domain in (ModelingRealms.atmos, ModelingRealms.ocean, ModelingRealms.seaIce): + if self.is_cmorized(startdate, member, chunk, domain): + Log.debug('Chunk {0} ready', chunk) + continue + if self._unpack_chunk(startdate, member, chunk): + cmorized = True + if cmorized: + Log.info('Startdate {0} member {1} ready', startdate, member) + return + self._cmorize_member(startdate, member) def is_cmorized(self, startdate, member, chunk, domain): """ @@ -564,34 +581,14 @@ class CMORManager(DataManager): def _cmorize_member(self, startdate, member): start_time = datetime.now() member_str = self.experiment.get_member_str(member) - Log.info('CMORizing startdate {0} member {1}. Starting at {0}', startdate, member_str, start_time) + Log.info('CMORizing startdate {0} member {1}. Starting at {2}', startdate, member_str, start_time) cmorizer = Cmorizer(self, startdate, member) cmorizer.cmorize_ocean() cmorizer.cmorize_atmos() Log.result('CMORized startdate {0} member {1}! Elapsed time: {2}\n\n', startdate, member_str, datetime.now() - start_time) - def _unpack_cmor_files(self, startdate, member): - if self.config.cmor.force: - return False - cmorized = False - for chunk in range(1, self.experiment.num_chunks + 1): - if not self.config.cmor.force_untar: - if self.is_cmorized(startdate, member, chunk, ModelingRealms.atmos) or \ - self.is_cmorized(startdate, member, chunk, ModelingRealms.ocean): - cmorized = True - continue - - if self._unpack_chunk(startdate, member, chunk): - cmorized = True - if cmorized: - Log.info('Startdate {0} member {1} ready', startdate, member) - return cmorized - def _unpack_chunk(self, startdate, member, chunk): - if not self.config.cmor.chunk_cmorization_requested(chunk): - return True - filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar.gz') if len(filepaths) > 0: Log.info('Unzipping cmorized data for {0} {1} {2}...', startdate, member, chunk) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 7ef152465b3aff73c1be1d6cade4b18a70f4a06e..734f94db7904708d40a44a51e492ce821b65cde5 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -6,6 +6,7 @@ import six from bscearth.utils.config_parser import ConfigParser from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str, add_years, add_months, add_days from bscearth.utils.log import Log +import bscearth.utils.path from earthdiagnostics import cdftools from earthdiagnostics.frequency import Frequency, Frequencies @@ -109,9 +110,13 @@ class Config(object): ---------- path: str """ + config_file_path = bscearth.utils.path.expand_path(path) + if not os.path.isfile(config_file_path): + Log.critical('Configuration file {0} can not be found', config_file_path) + raise ValueError('Configuration file {0} can not be found'.format(config_file_path)) parser = ConfigParser() parser.optionxform = str - parser.read(path) + parser.read(config_file_path) # Read diags config self.data_adaptor = parser.get_choice_option('DIAGNOSTICS', 'DATA_ADAPTOR', ('CMOR', 'THREDDS', 'OBSRECON'), diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index a244ca5c542cced734bafc4fc032410ac5fccf88..e4679e7469245902b05742e9accb822d39357cec 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -26,6 +26,12 @@ class Basin(object): def __str__(self): return self._name + def __repr__(self): + return str(self) + + def __hash__(self): + return hash(str(self)) + @property def name(self): """ diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 8bd25b4decfe645ab8e076a3a9ec18909d5c5698..086273e146a64142d8062dcd5575d73631bc0dbe 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -6,6 +6,7 @@ import shutil from datetime import datetime import iris +import iris.coords import numpy as np from bscearth.utils.log import Log @@ -238,8 +239,8 @@ class DataFile(Publisher): self.lon_name = 'longitude' self.lat_name = 'latitude' else: - self.lon_name = 'longitude' - self.lat_name = 'latitude' + self.lon_name = 'lon' + self.lat_name = 'lat' Utils.convert2netcdf4(self.local_file) if rename_var: @@ -290,7 +291,7 @@ class DataFile(Publisher): if diagnostic in self._modifiers: self._modifiers.remove(diagnostic) if region is not None: - self.region = region.name + self.region = region else: self.region = None self.local_file = local_file @@ -304,7 +305,8 @@ class DataFile(Publisher): def _correct_metadata(self): handler = Utils.open_cdf(self.local_file) var_handler = handler.variables[self.final_name] - coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name}, set(handler.variables.keys())) + coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name, 'leadtime', 'region', 'time_centered'}, + set(handler.variables.keys())) var_handler.coordinates = ' '.join(coords) if not self.cmor_var: handler.close() @@ -399,10 +401,7 @@ class DataFile(Publisher): regions = handler.variables['region'][...].tolist() if len(regions) > 1: ordered_regions = sorted(regions) - print(regions) - print(ordered_regions) new_indexes = [regions.index(region) for region in ordered_regions] - print(new_indexes) for var in handler.variables.values(): if 'region' not in var.dimensions: @@ -423,34 +422,32 @@ class DataFile(Publisher): self._fix_values_metadata(var_type, temp) Utils.nco.ncks(input=temp, output=temp, options=['--mk_rec_dmn region']) - handler = Utils.open_cdf(temp) - var_handler = handler.variables[self.final_name] - if hasattr(var_handler, 'valid_min'): - del var_handler.valid_min - if hasattr(var_handler, 'valid_max'): - del var_handler.valid_max - handler.sync() cubes = iris.load(self.local_file) for cube in cubes: if self.final_name == cube.var_name: - value = cube.data + value = cube break - if isinstance(value, np.ma.MaskedArray): - value = np.ma.getdata(value) - var_region = handler.variables['region'] - basin_index = np.where(var_region[:] == self.region) - if len(basin_index[0]) == 0: - var_region[var_region.shape[0]] = self.region - basin_index = var_region.shape[0] - 1 - else: - basin_index = basin_index[0][0] - handler.variables[self.final_name][..., basin_index] = np.multiply(np.ones(value.shape), value) - handler.close() + for index_region, region in enumerate(value.coord('region').points): + handler = Utils.open_cdf(temp) + region_slice = value.data[index_region, ...] + original_regions = handler.variables['region'][...] + var = handler.variables[self.final_name] + if region in original_regions: + var[np.where(original_regions == region)[0][0], ...] = region_slice + else: + var[original_regions.shape[0], ...] = region_slice + handler.variables[-1] = region + handler.close() + + # handler.close() Utils.move_file(temp, self.local_file) def _add_region_dimension_to_var(self): handler = Utils.open_cdf(self.local_file) + if 'region' in handler.variables: + handler.close() + return handler.createDimension('region') var_region = handler.createVariable('region', str, 'region') var_region[0] = self.region @@ -619,7 +616,12 @@ class NetCDFFile(DataFile): if not self.local_file: self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) - Utils.copy_file(self.remote_file, self.local_file) + try: + Utils.copy_file(self.remote_file, self.local_file, retrials=1) + except Utils.CopyException: + Utils.get_file_hash(self.remote_file, use_stored=False, save=True) + Utils.copy_file(self.remote_file, self.local_file, retrials=2) + if self.data_convention == 'meteofrance': Log.debug('Converting variable names from meteofrance convention') alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 4090dcd606b0054cf5af16694a3a1b1b38241060..099ef4f1781d81b43e343a57c1c1c6ad6c6722d3 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -92,7 +92,7 @@ class Diagnostic(Publisher): if file_generated.has_modifiers(): Log.warning('Can not skip diagnostics run when data is going to be modified: {0}'.format(self)) return False - return False + return True def __repr__(self): """Full string representation. Defaults to str""" @@ -428,6 +428,9 @@ class Diagnostic(Publisher): return len([request.storage_status != StorageStatus.READY or request.local_status != LocalStatus.READY for request in self._requests]) + def _different_type(self, other): + return type(self) is not type(other) + class DiagnosticOption(object): """Class to manage string options for the diagnostic""" @@ -608,6 +611,14 @@ class DiagnosticListFrequenciesOption(DiagnosticOption): super(DiagnosticListFrequenciesOption, self).__init__(name, default_value) def parse(self, option_value): + """ + Parse option value + + Returns + ------- + List of Frequency + + """ option_value = self._check_default(option_value) if isinstance(option_value, (tuple, list)): return option_value @@ -738,6 +749,32 @@ class DiagnosticFrequencyOption(DiagnosticOption): return Frequency.parse(self._check_default(option_value)) +class DiagnosticBasinListOption(DiagnosticOption): + """Class to parse list of basins options""" + + def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value: str + + Returns + ------- + Basin + + """ + option_value = self._check_default(option_value) + basins = [] + for value in option_value.split(':'): + basin = Basins().parse(value) + if basin is None: + raise DiagnosticOptionError('Basin {0} not recognized'.format(value)) + basins.append(basin) + return basins + + class DiagnosticBasinOption(DiagnosticOption): """Class to parse basin options""" diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 23107feb04d545a7bb2dd9ffcbf0d5a67fcf6934..86a5e6bc2b24a4ef5381003421cf9db12ca5b641 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -3,6 +3,7 @@ """Entry point for EarthDiagnostics""" import argparse import os +import sys import shutil import tempfile from distutils.spawn import find_executable @@ -19,7 +20,7 @@ from earthdiagnostics.constants import Basins from earthdiagnostics.obsreconmanager import ObsReconManager from earthdiagnostics.threddsmanager import THREDDSManager from earthdiagnostics.utils import TempFile, Utils -from work_manager import WorkManager +from earthdiagnostics.work_manager import WorkManager class EarthDiags(object): @@ -45,25 +46,37 @@ class EarthDiags(object): else: version = pkg_resources.require("earthdiagnostics")[0].version - def __init__(self, config_file): - Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) + def __init__(self): + self.time = dict() + self.data_manager = None + self.threads = None + self.had_errors = False self.config = Config() + + def read_config(self, config_file): + """ + Read config file and initialize earthdiagnostics + + Parameters + ---------- + config_file + + Returns + ------- + + """ + Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) self.config.parse(config_file) os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' - TempFile.scratch_folder = self.config.scratch_dir cdftools.path = self.config.cdftools_path self._create_dic_variables() - self.time = dict() - self.data_manager = None - self.threads = None - self.had_errors = False Log.debug('Diags ready') Log.info('Running diags for experiment {0}, startdates {1}, members {2}', self.config.experiment.expid, self.config.experiment.startdates, self.config.experiment.members) @staticmethod - def parse_args(): + def parse_args(args): """ Entry point for the Earth Diagnostics. @@ -92,31 +105,24 @@ class EarthDiags(object): parser.add_argument('-f', '--configfile', default='diags.conf', type=str) - args = parser.parse_args() + args = parser.parse_args(args) if args.doc: - Log.info('Opening documentation...') - doc_path = os.path.join('http://earthdiagnostics.readthedocs.io/en/latest') - Utils.execute_shell_command(('xdg-open', doc_path)) - Log.result('Documentation opened!') - return True + return EarthDiags.open_documentation() Log.set_console_level(args.logconsole) Log.set_file_level(args.logfile) if Log.console_handler.level <= Log.DEBUG: Utils.cdo.debug = True - Utils.nco.debug = False # This is due to a bug in nco. Must change when it's solved + Utils.nco.debug = True Utils.cdo.CDO = find_executable('cdo') if args.logfilepath: Log.set_file(bscearth.utils.path.expand_path(args.logfilepath)) - config_file_path = bscearth.utils.path.expand_path(args.configfile) - if not os.path.isfile(config_file_path): - Log.critical('Configuration file {0} can not be found', config_file_path) - return False try: - diags = EarthDiags(config_file_path) + diags = EarthDiags() + diags.read_config(args.configfile) if args.clean: result = diags.clean() elif args.report: @@ -130,6 +136,22 @@ class EarthDiags(object): return result + @staticmethod + def open_documentation(): + """ + Open Earthdiagnostics online documentation + + Returns + ------- + bool: + True if successful + """ + Log.info('Opening documentation...') + doc_path = os.path.join('http://earthdiagnostics.readthedocs.io/en/latest') + Utils.execute_shell_command(('xdg-open', doc_path)) + Log.result('Documentation opened!') + return True + def _create_dic_variables(self): self.dic_variables = dict() self.dic_variables['x'] = 'i' @@ -385,7 +407,7 @@ class EarthDiags(object): # Small size differences can be due to the renaming of variables reference_size = os.stat(source).st_size delta_size = abs(reference_size - os.stat(destiny).st_size) - if delta_size < 2048 or delta_size / reference_size < 1/1000: + if delta_size < 2048 or delta_size / reference_size < 1 / 1000: Log.info('File {0} already exists', destiny) return True @@ -414,7 +436,7 @@ class EarthDiags(object): def main(): """Main for earthdiagnostics""" - if not EarthDiags.parse_args(): + if not EarthDiags.parse_args(sys.argv[1:]): exit(1) diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index f80e971892844be202b224e1b9a507c56dac0b18..1e8c8a508b39a517520c8f51e559b80f05e85be9 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -1,11 +1,12 @@ # coding=utf-8 """Set attributtes in netCDF files""" -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, \ - DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.general.fix_file import FixFile from earthdiagnostics.utils import Utils +from earthdiagnostics.diagnostic import DiagnosticDomainOption, DiagnosticVariableOption, DiagnosticOption, \ + DiagnosticComplexStrOption -class Attribute(Diagnostic): +class Attribute(FixFile): """ Set the value of an attribute @@ -34,13 +35,7 @@ class Attribute(Diagnostic): def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, attributte_name, attributte_value): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid + FixFile.__init__(self, data_manager, startdate, member, chunk, domain, variable, grid) self.attributte_name = attributte_name self.attributte_value = attributte_value @@ -50,6 +45,9 @@ class Attribute(Diagnostic): 'Grid: {0.grid}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.grid == other.grid and \ self.attributte_name == other.attributte_name and self.attributte_value == other.attributte_value @@ -78,16 +76,6 @@ class Attribute(Diagnostic): options['value'])) return job_list - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) - def compute(self): """Run the diagnostic""" variable_file = self.variable_file.local_file diff --git a/earthdiagnostics/general/fix_file.py b/earthdiagnostics/general/fix_file.py new file mode 100644 index 0000000000000000000000000000000000000000..837302920ee8db2191d250628d491c35d459904a --- /dev/null +++ b/earthdiagnostics/general/fix_file.py @@ -0,0 +1,84 @@ +# coding=utf-8 +"""Base diagnostic for fixing files""" +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption + + +class FixFile(Diagnostic): + """ + Base diagnostic class for diagnostic that read and reqrite the same files + + Must be derived + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + """ + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + + _STR_PREFIX = None + + def __str__(self): + return '{0._STR_PREFIX} Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) + + def __eq__(self, other): + if self._different_type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.grid == self.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ + options_available = cls._get_options(diags) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(cls(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'])) + return job_list + + @classmethod + def _get_options(cls, diags): + return [DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticOption('grid', '')] + + def request_data(self): + """Request data required by the diagnostic""" + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index b8b311466f93c46a37ddb03169b186f8f83aed4a..64904c387dc2ea00d099c08835698648e4ff44a2 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -49,6 +49,8 @@ class Module(Diagnostic): self.componentv, self.module, self.grid) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.componentu == other.componentu and \ self.componentv == other.componentv and self.module == other.module and self.grid == other.grid diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 1a9162d948e63221f1217be94294b8b0f1fdf338..fcae8e514310fa9dc924f0fef5b1757a8c408a11 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -47,6 +47,8 @@ class Relink(Diagnostic): 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.move_old == other.move_old and \ self.grid == other.grid diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 17d05df88969749e450cb7a096f23b29b00714dc..1b59e1b24f70aa8984aba44d7c68c69cc991715f 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -28,6 +28,8 @@ class RelinkAll(Diagnostic): return 'Relink all output Startdate: {0}'.format(self.startdate) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate @classmethod diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index db19ee029a3c2cb150670cf4f25d81813bf8a321..26cab98c0ae4be8b3505a2fa9999e246b7d4f04e 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -1,9 +1,9 @@ # coding=utf-8 """Rewrite netCDF file""" -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.general.fix_file import FixFile -class Rewrite(Diagnostic): +class Rewrite(FixFile): """ Rewrites files without doing any calculations. @@ -30,54 +30,7 @@ class Rewrite(Diagnostic): alias = 'rewrite' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid - - def __str__(self): - return 'Rewrite output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, - self.grid) - - def __eq__(self, other): - return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.grid == self.grid - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, grid - :type options: list[str] - :return: - """ - options_available = (DiagnosticDomainOption(), - DiagnosticVariableOption(diags.data_manager.config.var_manager), - DiagnosticOption('grid', '')) - options = cls.process_options(options, options_available) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['grid'])) - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + _STR_PREFIX = 'Rewrite output' def compute(self): """Run the diagnostic""" diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index b7d27de28fe8d5ff1f0ffbc92f87b41146217f1c..de904de8af7553e4f40894826c7dd2a500df88e2 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -5,12 +5,13 @@ import math import numpy as np from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, DiagnosticVariableOption, \ +from earthdiagnostics.general.fix_file import FixFile +from earthdiagnostics.diagnostic import DiagnosticDomainOption, DiagnosticVariableOption, \ DiagnosticFloatOption, DiagnosticBoolOption, DiagnosticListFrequenciesOption, DiagnosticOption from earthdiagnostics.utils import Utils -class Scale(Diagnostic): +class Scale(FixFile): """ Scales a variable by the given value also adding at offset @@ -40,13 +41,7 @@ class Scale(Diagnostic): def __init__(self, data_manager, startdate, member, chunk, value, offset, domain, variable, grid, min_limit, max_limit, frequency, apply_mask): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid + FixFile.__init__(self, data_manager, startdate, member, chunk, domain, variable, grid) self.value = value self.offset = offset self.min_limit = min_limit @@ -62,6 +57,8 @@ class Scale(Diagnostic): 'Frequency: {0.frequency} Apply mask: {0.apply_mask}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ self.apply_mask == other.apply_mask and self.value == other.value and self.offset == other.offset @@ -96,16 +93,6 @@ class Scale(Diagnostic): options['apply_mask'])) return job_list - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, frequency=self.frequency, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, frequency=self.frequency) - def compute(self): """Run the diagnostic""" variable_file = self.variable_file.local_file diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 239c2df1807530f92769b7f7b19e991ea1560b5e..3a6c38f1d17b49eae443ff10fce792af1e791e1c 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -45,6 +45,8 @@ class SelectLevels(Diagnostic): self.grid, self.box.min_depth, self.box.max_depth) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.grid == self.grid diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 6f73d379e37f94e8bb1236045ce3bd93e366762e..90af9df6eae46fc52963aa60b817ac2705c61b5a 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -1,13 +1,13 @@ # coding=utf-8 """Convert i j files to lon lat when there is no interpolation required""" import numpy as np - -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ +from earthdiagnostics.general.fix_file import FixFile +from earthdiagnostics.diagnostic import DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption from earthdiagnostics.utils import Utils, TempFile -class SimplifyDimensions(Diagnostic): +class SimplifyDimensions(FixFile): """ Convert i j files to lon lat when there is no interpolation required @@ -29,13 +29,7 @@ class SimplifyDimensions(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, data_convention): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid + FixFile.__init__(self, data_manager, startdate, member, chunk, domain, variable, grid) if data_convention in ('cmip6', 'primavera'): self.lon_name = 'longitude' self.lat_name = 'latitude' @@ -49,6 +43,8 @@ class SimplifyDimensions(Diagnostic): self.grid) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.grid == self.grid @@ -77,16 +73,6 @@ class SimplifyDimensions(Diagnostic): diags.config.data_convention)) return job_list - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.simplified = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) - def compute(self): """Run the diagnostic""" handler = Utils.open_cdf(self.variable_file.local_file) diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 0f0b53229fffd0084c334d6ebbe676cf906b17ec..2addbe5e168c7345de90a535524b696b7e2876a9 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -46,6 +46,8 @@ class TimeMean(Diagnostic): 'Variable: {0.domain}:{0.variable} Original frequency: {0.frequency} Grid: {0.grid}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ self.grid == other.grid and self._target_frequency == other._target_frequency diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index 4cc5f3945d4b305f4975e8da72f9fbe1614cfdc2..602fc5cd72a1a33c07cca0698704670c35392d70 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -49,6 +49,8 @@ class VerticalMeanMetersIris(Diagnostic): self.box = box def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 0f20d28dac39bcd0c6e2cc50d0bd7fc3f6cca047..d94a724b7856424ee5a1c76773a647e32e58d103 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -56,6 +56,8 @@ class AreaMoc(Diagnostic): self.box = box def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.basin == other.basin and self.box == other.box diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 27eac1aa5bc981084b7ef348f84722e0a0118b86..aa879381194280d6357b2dd36bef5668fc602c73 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -50,6 +50,8 @@ class AverageSection(Diagnostic): self.grid = grid def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.box == other.box diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 08523f34c1b44884c6b45f71bad3dd5a2a0d30b0..8a82d654d17fc796bb9cdcc9c6c6718938de7679 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -46,6 +46,8 @@ class ConvectionSites(Diagnostic): return 'Convection sites Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index 2690e6b5196e252cd77a3ae63594c16d037724d4..1059e4435a3c958f5839d06473fc284504c6d3da 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -60,6 +60,8 @@ class CutSection(Diagnostic): self.box.min_lat = self.value def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.zonal == other.zonal and \ self.value == other.value diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index 942a24676d1467113f16270c974fb315f6d83763..38d22f9f6fc826f293658bd3b357cee111061f41 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -43,6 +43,8 @@ class Gyres(Diagnostic): self.var_vsftbarot = None def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 476c639f242bae59cd8ed03fff04a040e14596ff..a93329159c1d2a790d24735876e328ea3c89e2ce 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -52,7 +52,7 @@ class HeatContent(Diagnostic): self.max_level = max_level def __eq__(self, other): - if not isinstance(other, HeatContent): + if self._different_type(other): return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.basin == other.basin and self.mxloption == other.mxloption diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 88a74a7ca7d3f93dac2862a529d1547e3eeb84c5..135f9f4b1bb2af78a46a0ba5e786dc76f7f9d3c7 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -81,29 +81,7 @@ class HeatContentLayer(Diagnostic): @classmethod def _compute_weights(cls, box): - handler = Utils.open_cdf('mesh_zgr.nc') - # mask = Utils.get_mask(options['basin']) - mask = handler.variables['tmask'][:] - if 'e3t' in handler.variables: - e3t = handler.variables['e3t'][:] - elif 'e3t_0' in handler.variables: - e3t = handler.variables['e3t_0'][:] - else: - raise Exception('e3t variable can not be found') - if 'gdepw' in handler.variables: - depth = handler.variables['gdepw'][:] - elif 'gdepw_0' in handler.variables: - depth = handler.variables['gdepw_0'][:] - else: - raise Exception('gdepw variable can not be found') - e3t_3d = e3t.shape != depth.shape - if e3t_3d: - mask = e3t_3d * mask - else: - e3t = e3t[0, :] - while len(depth.shape) < 4: - depth = np.expand_dims(depth, -1) - handler.close() + depth, e3t, mask = cls._get_mesh_info() def calculate_weight(e3t_point, depth_point, mask_point): """Calculate the weight for each cell""" @@ -123,6 +101,12 @@ class HeatContentLayer(Diagnostic): calc = np.vectorize(calculate_weight, otypes='f') weight = calc(e3t, depth, mask) + max_level, min_level = cls._get_used_levels(weight) + weight = weight[:, min_level:max_level, :] + return max_level, min_level, weight + + @classmethod + def _get_used_levels(cls, weight): # Now we will reduce to the levels with any weight != 0 to avoid loading too much data on memory levels = weight.shape[1] min_level = 0 @@ -131,8 +115,34 @@ class HeatContentLayer(Diagnostic): max_level = min_level while max_level < (levels - 1) and weight[:, max_level + 1, :].any(): max_level += 1 - weight = weight[:, min_level:max_level, :] - return max_level, min_level, weight + return max_level, min_level + + @classmethod + def _get_mesh_info(cls): + handler = Utils.open_cdf('mesh_zgr.nc') + # mask = Utils.get_mask(options['basin']) + mask = handler.variables['tmask'][:] + if 'e3t' in handler.variables: + e3t = handler.variables['e3t'][:] + elif 'e3t_0' in handler.variables: + e3t = handler.variables['e3t_0'][:] + else: + raise Exception('e3t variable can not be found') + if 'gdepw' in handler.variables: + depth = handler.variables['gdepw'][:] + elif 'gdepw_0' in handler.variables: + depth = handler.variables['gdepw_0'][:] + else: + raise Exception('gdepw variable can not be found') + e3t_3d = e3t.shape != depth.shape + if e3t_3d: + mask = e3t_3d * mask + else: + e3t = e3t[0, :] + while len(depth.shape) < 4: + depth = np.expand_dims(depth, -1) + handler.close() + return depth, e3t, mask def request_data(self): """Request data required by the diagnostic""" diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 56f478445889e22ce00395c2a6c09049ba16d7c2..76e9d78a142b29de9a88c1110278e4f630730fbf 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -62,6 +62,8 @@ class Interpolate(Diagnostic): self.original_grid = original_grid def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version and self.domain == other.domain and \ self.variable == other.variable and self.grid == other.grid and \ diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 5a09eaf3174905af3ec4a2b868cfc5405fb9ae39..eccdf7eaa17cc3cbc008a5d32456876a3b7bc71d 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -65,6 +65,9 @@ class InterpolateCDO(Diagnostic): self.weights = weights def __eq__(self, other): + if self._different_type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version and self.domain == other.domain and \ self.variable == other.variable and self.mask_oceans == other.mask_oceans and self.grid == other.grid and \ @@ -127,6 +130,21 @@ class InterpolateCDO(Diagnostic): @classmethod def compute_weights(cls, method, target_grid, sample_file, weights): + """ + Compute weights for interpolation from sample file + + Parameters + ---------- + method: int + Interpolation method + target_grid: str + Grid to intepolate to. Can be anything understand by CDO + sample_file: str + Path to a file containing original mesh information + weights: + Path to the file to store the weights + + """ if method == InterpolateCDO.BILINEAR: Utils.cdo.genbil(target_grid, input=sample_file, output=weights) elif method == InterpolateCDO.BICUBIC: @@ -138,6 +156,15 @@ class InterpolateCDO(Diagnostic): @classmethod def get_sample_grid_file(cls): + """ + Get a sample grid file + + Create a sample grid file from the definition in the masks file + + Returns + ------- + str + """ temp = TempFile.get() handler = Utils.open_cdf('mask.nc') @@ -316,6 +343,7 @@ class ComputeWeights(Diagnostic): return 'Computing weights for CDO interpolation: Method {0.method} Target grid: {0.grid}'.format(self) def compute(self): + """Compute weights""" InterpolateCDO.compute_weights(self.method, self.grid, self.sample_data.local_file, self.weights_file) def request_data(self): diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index 01062c11d7db87301933e9e28eb084939681b01b..451d9d2a648a4bcf0cb27d9638a8d3ae11cab05d 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -37,6 +37,8 @@ class MaskLand(Diagnostic): self.grid = grid def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index eba04a78977fe9779503ae04b0943d35ff39daa8..0e13d71f7adf585a7d52f6772287f7fb84985bb9 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -56,6 +56,8 @@ class MaxMoc(Diagnostic): 'Basin: {4}'.format(self.startdate, self.member, self.year, self.box, self.basin.name) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.year == other.year and \ self.box == other.box and self.basin == other.basin diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index fa03a558bc8fdd789939ccb562b1b271d1aedefb..529d71084a7020ac2e091bd83411608c8640a490 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -40,6 +40,8 @@ class MixedLayerHeatContent(Diagnostic): self.generated_vars = ['scvertsum'] def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk def __str__(self): diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index a6f7b4559ee8c209361cd050f67decce5ed93a59..170815fe87c77dfc63699384b0bcfba0f5806606 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -44,6 +44,8 @@ class MixedLayerSaltContent(Diagnostic): self.chunk) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk @classmethod diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index a300e8b113484709ab2fa2ec75786a86bb4ea3d8..644b4f38fd3475b70c5ab63989246e8483894a1d 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -47,6 +47,8 @@ class Moc(Diagnostic): return 'MOC Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk @classmethod diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index 1c733c83ebb514bcfc3cb61d6a89d07f3ea7ab58..2022daaac41b425b3dc951c934aea87b13d50561 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -32,6 +32,8 @@ class Mxl(Diagnostic): self.chunk = chunk def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk def __str__(self): diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index 03107eda951397285a7fb177b5b3a58c74d94958..b9b32f024c06ed670193f72bb46d8dd1737aa4fb 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -38,6 +38,8 @@ class Psi(Diagnostic): self.chunk = chunk def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk def __str__(self): diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index fa41fcf161a1ee5cc4563b4ab70fac2b2176a0c8..12121f525154fb497ec01369db12e089e1ffbdc1 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -64,7 +64,7 @@ class RegionMean(Diagnostic): self.lon_name = 'lon' def __eq__(self, other): - if type(self) is not type(other): + if self._different_type(other): return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable @@ -156,7 +156,7 @@ class RegionMean(Diagnostic): return cell.point - 1 in j_indexes def selected_level(cell): - return lev_limits[0] <= cell.point <= lev_limits[1] + return lev_limits[0] <= cell.point - 1 <= lev_limits[1] data = data.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) if has_levels: @@ -201,7 +201,7 @@ class RegionMean(Diagnostic): def _load_data(self): def add_i_j(cube, field, filename): if cube.var_name != self.variable: - return + raise iris.exceptions.IgnoreCubeException() if not cube.coords('i'): index = field.dimensions.index('i') i = np.arange(1, field.shape[index] + 1) @@ -218,9 +218,23 @@ class RegionMean(Diagnostic): lev = iris.coords.AuxCoord(i, var_name='lev') cube.add_aux_coord(lev, index) + coords = [] + handler = Utils.open_cdf(self.variable_file.local_file) + for variable in handler.variables: + if variable in ('time', 'lev', 'lat', 'lon', 'latitude', 'longitude'): + coords.append(variable) + + handler.variables[self.variable].coordinates = ' '.join(coords) + handler.close() + data = iris.load_cube(self.variable_file.local_file, - iris.AttributeConstraint(short_name=self.variable), callback=add_i_j) + + if data.coords('model_level_number'): + coord = data.coord('model_level_number') + coord.standard_name = 'depth' + coord.long_name = 'depth' + return data def _fix_file_metadata(self): @@ -300,7 +314,7 @@ class ComputeWeights(Diagnostic): self.box = box def __eq__(self, other): - if type(self) is not type(other): + if self._different_type(other): return False return self.weights_file == other.weights_file and self.basin == other.basin and \ self.grid_point == other.grid_point and self.box != other.box @@ -346,12 +360,12 @@ class ComputeWeights(Diagnostic): return cell.point - 1 in j_indexes def selected_level(cell): - return min_level <= cell.point <= max_level + return min_level <= cell.point - 1 <= max_level e1_small = e1.extract(iris.Constraint(i=selected_i, j=selected_j)) e2_small = e2.extract(iris.Constraint(i=selected_i, j=selected_j)) e3_small = e3.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) - mask_small = mask_small[min_level:max_level, ...] + mask_small = mask_small[min_level:max_level + 1, ...] mask_small = e3_small * mask_small e_small = e1_small * e2_small diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index 6ba7a4e310461d84a36aa706ca8fb4bd237eccba..afb452242a39c524f808eacfec05ca57ef142279 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -58,8 +58,12 @@ class RegionSum(Diagnostic): self.lon_name = 'lon' def __eq__(self, other): + if self._different_type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.box == other.box and self.variable == other.variable + self.box == other.box and self.variable == other.variable and \ + self.grid_point == other.grid_point and self.grid == other.grid and self.basin == other.basin def __str__(self): return 'Region sum Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Variable: {0.variable} ' \ diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index b00a30b913d0bb9f686a68e336b7e6e53a2845ac..21e2e053310ae93a9e481f49f51d88be7cc452c2 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -45,6 +45,8 @@ class Rotation(Diagnostic): self.tempTemplate = None def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variableu == other.variableu and \ self.variablev == other.variablev and self.executable == other.executable diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 53b1c7281a0a2c79760a6476e44bf934978f0ac7..d60d2bb7fb5b7d018ea9667300f17fdc6b1824da 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,13 +1,16 @@ # coding=utf-8 """Compute the sea ice extent , area and volume in both hemispheres or a specified region""" -import os - -import netCDF4 +import six import numpy as np + +import iris +import iris.analysis +import iris.coords +import iris.util from bscearth.utils.log import Log from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticBoolOption +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinListOption, DiagnosticBoolOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile @@ -27,7 +30,7 @@ class Siasiesiv(Diagnostic): chunk: init domain: ModellingRealm variable: str - basin: Basin + basin: list of Basin mask: numpy.array omit_vol: bool """ @@ -39,22 +42,26 @@ class Siasiesiv(Diagnostic): e2t = None gphit = None - def __init__(self, data_manager, startdate, member, chunk, basin, mask, var_manager, omit_vol): + def __init__(self, data_manager, startdate, member, chunk, masks, var_manager, omit_vol): Diagnostic.__init__(self, data_manager) - self.basin = basin self.startdate = startdate self.member = member self.chunk = chunk - self.mask = mask + self.masks = masks self.generated = {} self.var_manager = var_manager self.omit_volume = omit_vol self.sic_varname = self.var_manager.get_variable('sic').short_name self.sit_varname = self.var_manager.get_variable('sit').short_name + self.results = {} + for var in ('siarean', 'siareas', 'sivoln', 'sivols', 'siextentn', 'siextents'): + self.results[var] = {} + def __str__(self): return 'Siasiesiv Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ - 'Basin: {0.basin} Omit volume: {0.omit_volume}'.format(self) + 'Basins: {1} Omit volume: {0.omit_volume}'.format(self, + ','.join(str(basin) for basin in self.masks.keys())) @classmethod def generate_jobs(cls, diags, options): @@ -67,25 +74,26 @@ class Siasiesiv(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticBasinOption('basin', Basins().Global), + options_available = (DiagnosticBasinListOption('basins', [Basins().Global]), DiagnosticBoolOption('omit_volume', False)) options = cls.process_options(options, options_available) - if options['basin'] is None: - Log.error('Basin not recognized') + if not options['basins']: + Log.error('Basins not recognized') return () - mask = np.asfortranarray(Utils.get_mask(options['basin'])) + masks = {} + for basin in options['basins']: + masks[basin] = Utils.get_mask(basin) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask, + job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, masks, diags.config.var_manager, options['omit_volume'])) - mesh_handler = Utils.open_cdf('mesh_hgr.nc') - Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) - Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) - Siasiesiv.gphit = np.asfortranarray(mesh_handler.variables['gphit'][0, :]) - mesh_handler.close() + + e1t = iris.load_cube('mesh_hgr.nc', 'e1t') + e2t = iris.load_cube('mesh_hgr.nc', 'e2t') + Siasiesiv.area = e1t * e2t return job_list @@ -110,57 +118,76 @@ class Siasiesiv(Diagnostic): self._declare_var('siextentn') def _declare_var(self, var_name): - self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, self.startdate, self.member, - self.chunk, region=self.basin.name) + self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, + self.startdate, self.member, self.chunk, + region=self.masks.keys()) def compute(self): """Run the diagnostic""" - import earthdiagnostics.cdftoolspython as cdftoolspython - - sic_handler = Utils.open_cdf(self.sic.local_file) - Utils.convert_units(sic_handler.variables[self.sic_varname], '1.0') - sic = np.asfortranarray(sic_handler.variables[self.sic_varname][:]) - timesteps = sic_handler.dimensions['time'].size - sic_handler.close() + handler = Utils.open_cdf(self.sic.local_file) + handler.variables[self.sic_varname].coordinates = 'time lat lon leadtime time_centered' + handler.close() + sic = iris.load_cube(self.sic.local_file) + if sic.units.origin == '%' and sic.data.max() < 2: + sic.units = '1.0' + sic.convert_units('1.0') + sic *= Siasiesiv.area.data + extent = sic.data > 0.15 - if self.omit_volume: - sit = sic - else: - sit_handler = Utils.open_cdf(self.sit.local_file) - sit = np.asfortranarray(sit_handler.variables[self.sit_varname][:]) - sit_handler.close() + if not self.omit_volume: + handler = Utils.open_cdf(self.sit.local_file) + handler.variables[self.sit_varname].coordinates = 'time lat lon leadtime time_centered' + handler.close() + sit = iris.load_cube(self.sic.local_file) - result = np.empty((8, timesteps)) - for t in range(0, timesteps): - result[:, t] = cdftoolspython.icediag.icediags(Siasiesiv.e1t, Siasiesiv.e2t, self.mask, - Siasiesiv.gphit, sit[t, :], sic[t, :]) + for basin, mask in six.iteritems(self.masks): + self.results['siarean'][basin] = self.sum(sic, mask, north=True) + self.results['siareas'][basin] = self.sum(sic, mask, north=False) - self._extract_variable_and_rename(result[5, :], 'siareas', '10^9 m2') - self._extract_variable_and_rename(result[7, :], 'siextents', '10^9 m2') + if not self.omit_volume: + volume = sic * sit + self.results['sivoln'][basin] = self.sum(volume, mask, north=True) + self.results['sivols'][basin] = self.sum(volume, mask, north=False) - self._extract_variable_and_rename(result[1, :], 'siarean', '10^9 m2') - self._extract_variable_and_rename(result[3, :], 'siextentn', '10^9 m2') + extent_mask = mask * extent + self.results['siextentn'][basin] = self.sum(sic, extent_mask, north=True) + self.results['siextents'][basin] = self.sum(sic, extent_mask, north=False) - if not self.omit_volume: - self._extract_variable_and_rename(result[4, :], 'sivols', '10^9 m3') - self._extract_variable_and_rename(result[0, :], 'sivoln', '10^9 m3') + self.save() - def _extract_variable_and_rename(self, values, cmor_name, units): + def sum(self, data, mask, north=True): + if north: + condition = data.coord('latitude').points > 0 + else: + condition = data.coord('latitude').points < 0 + weights = iris.util.broadcast_to_shape(condition, data.shape, data.coord_dims('latitude')) * mask + return data.collapsed(('latitude', 'longitude'), iris.analysis.SUM, weights=weights) + + def save(self): + for var, basins in six.iteritems(self.results): + results = iris.cube.CubeList() + for basin, result in six.iteritems(basins): + result.var_name = var + if var.startswith('sivol'): + result.units = 'm^3' + else: + result.units = 'm^2' + result.add_aux_coord(iris.coords.AuxCoord(basin.name, var_name='region')) + results.append(result) + if not results: + continue + self._save_file(results.merge_cube(), self.generated[var], basins) + + def _save_file(self, data, generated_file, basins): temp = TempFile.get() - reference_handler = Utils.open_cdf(self.sic.local_file) - os.remove(temp) - handler = netCDF4.Dataset(temp, 'w') - - Utils.copy_variable(reference_handler, handler, 'time', add_dimensions=True) - Utils.copy_variable(reference_handler, handler, 'time_bnds', add_dimensions=True, must_exist=False) - Utils.copy_variable(reference_handler, handler, 'leadtime', add_dimensions=True, must_exist=False) - reference_handler.close() - - new_var = handler.createVariable(cmor_name, float, 'time', fill_value=1.0e20) - new_var.units = units - new_var.short_name = cmor_name - new_var.valid_min = 0.0 - new_var[:] = values - new_var.valid_max = np.max(values) + region = data.coord('region').points + data.remove_coord('region') + iris.save(data, temp, zlib=True) + Utils.rename_variable(temp, 'dim0', 'region', False, True) + handler = Utils.open_cdf(temp) + var = handler.createVariable('region2', str, ('region',)) + var[...] = region handler.close() - self.generated[cmor_name].set_local_file(temp) + Utils.rename_variable(temp, 'region2', 'region', True, False) + + generated_file.set_local_file(temp, region=basins.keys()) diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 64a933ed8f677d3293464a558373923237d18de1..32ee7460994a41ed37245e888780faa581041cd8 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -41,10 +41,10 @@ class VerticalGradient(Diagnostic): self.chunk = chunk self.variable = variable self.box = box - self.required_vars = [variable] - self.generated_vars = [variable + 'vmean'] def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index cf05d5eb6b4bef756d69ad00e2ce26ee8da5a271..795dc456a2573a299e7e6af8dc9f09788eaa39ad 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -42,10 +42,10 @@ class VerticalMean(Diagnostic): self.chunk = chunk self.variable = variable self.box = box - self.required_vars = [variable] - self.generated_vars = [variable + 'vmean'] def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index ee9d2084256f3c3b924f28ec6d2d60d465ad0df9..18d9ce456849447faed1bf0d18747af60faf0622 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -47,6 +47,8 @@ class VerticalMeanMeters(Diagnostic): self.grid_point = grid_point def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 8050f7ef9bfa39aa21d50eb09ad3549bc9f47f16..b4585b233bebb018e4c0dea79b9db4a26f65eb46 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -47,6 +47,8 @@ class ClimatologicalPercentile(Diagnostic): self.leadtime_files = {} def __eq__(self, other): + if self._different_type(other): + return False return self.domain == other.domain and self.variable == other.variable and \ self.start_year == other.start_year and self.end_year == other.end_year and \ self.forecast_month == other.forecast_month diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index ad1b25f397c33b8e065f342afd5751759a882054..ca1c77dfd0960a546b2ac694945337ccd470e83b 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -48,6 +48,8 @@ class DaysOverPercentile(Diagnostic): self.lon_coord = None def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.domain == other.domain and \ self.variable == other.variable and self.start_year == other.start_year and \ self.end_year == other.end_year diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index e577d90b1b2c501351a70de688400e49ebb1da8e..e96a95f7a1430021c6a3f5b37fde92e25980d450 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -92,6 +92,8 @@ class Discretize(Diagnostic): self._bins = value def __eq__(self, other): + if self._different_type(other): + return False return self.domain == other.domain and self.variable == other.variable and self.num_bins == other.num_bins and \ self.min_value == other.min_value and self.max_value == other.max_value and \ self.startdate == other.startdate diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 9e3575bd9ea14d1a468d8b006feaf7095352bf8f..bbd33dd6f85773572b629379399bb65ac0b82f43 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -39,6 +39,8 @@ class MonthlyPercentile(Diagnostic): self.percentiles = percentiles def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.percentiles == other.percentiles diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index b6606524453ee5785d197da28d259a8f1c23b32b..be37da89bca569bbffa6cc115a76f07c3bb88571 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -5,14 +5,15 @@ from datetime import datetime from time import strptime import iris +from iris.coords import DimCoord + import netCDF4 import numpy as np from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date from bscearth.utils.log import Log from cf_units import Unit -from iris.coords import DimCoord -from datafile import DataFile, StorageStatus, LocalStatus +from earthdiagnostics.datafile import DataFile, StorageStatus, LocalStatus from earthdiagnostics.datamanager import DataManager from earthdiagnostics.utils import TempFile, Utils from earthdiagnostics.variable import VariableType @@ -25,6 +26,7 @@ class THREDDSManager(DataManager): Parameters ---------- config: Config + """ def __init__(self, config): @@ -47,9 +49,11 @@ class THREDDSManager(DataManager): # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN, possible_versions=None): - """" + """ Check if a file exists in the storage + Creates a THREDDSSubset and checks if it is accesible + Parameters ---------- domain: ModelingRealm @@ -64,7 +68,8 @@ class THREDDSManager(DataManager): Returns ------- - bool + THREDDSSubset + """ aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) @@ -280,6 +285,7 @@ class THREDDSSubset(DataFile): var: str start_time: datetime end_time: datetime + """ def __init__(self, thredds_path, file_path, var, start_time, end_time): diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index bf44e4b442316ee4b4347ca1ef9221de9a11f927..a570f39a7eaa897b69ca385b0772f99808e07907 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -42,7 +42,7 @@ class Utils(object): nco = Nco() """An instance of Nco class ready to be used""" - cdo = Cdo() + cdo = Cdo(env=os.environ) """An instance of Cdo class ready to be used""" @staticmethod @@ -79,7 +79,7 @@ class Utils(object): mask_handler = Utils.open_cdf('mask.nc') mask = mask_handler.variables['tmask'][0, 0, :] mask_handler.close() - return mask + return np.squeeze(mask) @staticmethod def setminmax(filename, variable_list): @@ -341,7 +341,7 @@ class Utils(object): while hash_original != hash_destiny: if retrials == 0: - raise Exception('Can not copy {0} to {1}'.format(source, destiny)) + raise Utils.CopyException('Can not copy {0} to {1}'.format(source, destiny)) Log.debug('Copying... {0}', datetime.datetime.now()) shutil.copyfile(source, destiny) Log.debug('Hashing copy ... {0}', datetime.datetime.now()) @@ -554,24 +554,12 @@ class Utils(object): @classmethod def _is_compressed_netcdf4(cls, filetoconvert): - is_compressed = True - handler = Utils.open_cdf(filetoconvert) - if not handler.file_format == 'NETCDF4': - is_compressed = False - handler.close() - else: - handler.close() - ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) - ncdump_result = ncdump_result[0].replace('\t', '').split('\n') - for var in handler.variables: - if not '{0}:_DeflateLevel = 4 ;'.format(var) in ncdump_result: - is_compressed = False - break - if not '{0}:_Shuffle = "true" ;'.format(var) in ncdump_result: - is_compressed = False - break - return is_compressed - + ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) + ncdump_result = ncdump_result[0].replace('\t', '').split('\n') + if any(':_Shuffle = "true"' in line for line in ncdump_result) and \ + any(':_DeflateLevel = 4' in line for line in ncdump_result): + return True + return False # noinspection PyPep8Naming @staticmethod @@ -877,6 +865,11 @@ class Utils(object): pass + class CopyException(Exception): + """Exception raised when copy fails""" + + pass + class TempFile(object): """Class to manage temporal files""" diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index fee14f52f230571f751303649bf6a5fe55edec50..36efe266db94b500b63533ed0f9a15defa50a1a3 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -280,7 +280,7 @@ class VariableManager(object): if row[1].value in excel.sheetnames: table_data[row[1].value] = (Frequency(row[2].value), 'Date missing') for sheet_name in excel.sheetnames: - sheet = excel.get_sheet_by_name(sheet_name) + sheet = excel[sheet_name] if sheet.title == 'Primday': pass if sheet['A1'].value != 'Priority': diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 385a5525e2c4cac75a510964b5d5c8d474525ca2..daec40612d2e4a11ed0870ffecf04947d39f1f02 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -83,17 +83,19 @@ class WorkManager(object): self.uploader = ThreadPoolExecutor(self.config.parallel_uploads) self.executor = ThreadPoolExecutor(self.threads) + self.lock = threading.Lock() + self.lock.acquire() + for job in self.jobs: job.request_data() job.declare_data_generated() job.subscribe(self, self._job_status_changed) - job.check_is_ready() - - if self.config.skip_diags_done: - for job in self.jobs: + if self.config.skip_diags_done: if job.can_skip_run(): Log.info('Diagnostic {0} already done. Skipping !', job) job.status = DiagnosticStatus.COMPLETED + continue + job.check_is_ready() for file_object in self.data_manager.requested_files.values(): file_object.subscribe(self, self._file_object_status_changed) @@ -101,10 +103,6 @@ class WorkManager(object): self.downloader.submit(file_object) self.downloader.start() - self.lock = threading.Lock() - self.lock.acquire() - - # self.check_completion() self.lock.acquire() self.downloader.shutdown() @@ -362,8 +360,9 @@ class Downloader(object): return time.sleep(0.01) continue - self._downloads.sort(key=cmp_to_key(Downloader._prioritize)) - datafile = self._downloads[0] + downloads = self._downloads[:100] + downloads.sort(key=cmp_to_key(Downloader._prioritize)) + datafile = downloads[0] self._downloads.remove(datafile) datafile.download() except Exception as ex: @@ -383,7 +382,7 @@ class Downloader(object): def _prioritize(datafile1, datafile2): waiting = Downloader._suscribers_waiting(datafile1) - Downloader._suscribers_waiting(datafile2) if waiting: - return -waiting + return waiting suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) if suscribers: @@ -393,12 +392,12 @@ class Downloader(object): if datafile2.size is None: return 0 else: - return -1 + return 1 elif datafile2.size is None: - return 1 + return -1 size = datafile1.size - datafile2.size if size: - return -size + return size return 0 def shutdown(self): diff --git a/environment.yml b/environment.yml index 14d620055e705e0214f91ecc4d3b850a6bf96835..42dcdbbe63415a11d105b89a57ebd06736da32de 100644 --- a/environment.yml +++ b/environment.yml @@ -5,10 +5,10 @@ channels: - conda-forge dependencies: -- iris +- iris<2 - netcdf4 - numpy -- cdo +- cdo=1.9.2 - nco - python-cdo - coverage @@ -20,6 +20,9 @@ dependencies: - mock - cmake - coverage +- pytest +- pytest-cov +- pycodestyle - pip: - bscearth.utils @@ -27,4 +30,4 @@ dependencies: - nco - exrex - xxhash - - codacy-coverage + - pytest-profiling diff --git a/launch_diags.sh b/launch_diags.sh index df9a004b1826525927f211e3cb86959793db7736..1ff20d775f46951304c507c5c74a3ad7f8fb971d 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -7,7 +7,7 @@ -PATH_TO_CONF_FILE=~rfernand/run_earthdiagnostics/m04p/diags.conf +PATH_TO_CONF_FILE=~/earthdiags_confs/ruben/diags_a0dc_DEBUG.conf PATH_TO_DIAGNOSTICS=~jvegas/PycharmProjects/earthdiagnostics PATH_TO_CONDAENV=/home/Earth/jvegas/.conda/envs/earthdiagnostics3/ diff --git a/run_test.py b/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..cec7ec80ed57a8abc1fe8b81301f904eef2be094 --- /dev/null +++ b/run_test.py @@ -0,0 +1,24 @@ +# coding=utf-8 +"""Script to run the tests for EarthDiagnostics and generate the code coverage report""" + +import os +import sys +import pytest +work_path = os.path.abspath(os.path.join(os.path.dirname(__file__))) +os.chdir(work_path) +print(work_path) + + +version = sys.version_info[0] +report_dir = 'test/report/python{}'.format(version) +errno = pytest.main([ + 'test', + '--ignore=test/report', + '--cov=earthdiagnostics', + '--cov-report=term', + '--cov-report=html:{}/coverage_html'.format(report_dir), + '--cov-report=xml:{}/coverage.xml'.format(report_dir), + '--profile', + '--profile-svg', +]) +sys.exit(errno) diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py new file mode 100644 index 0000000000000000000000000000000000000000..0f5fb8ea7d8ddf3efa9dbfb0b22d532c12813403 --- /dev/null +++ b/test/integration/test_cmorizer.py @@ -0,0 +1,531 @@ +from earthdiagnostics.cmorizer import Cmorizer +from earthdiagnostics.utils import TempFile, Utils +from bscearth.utils import log + +from unittest import TestCase +from mock import Mock +import os +import tempfile +import shutil +import iris +import iris.cube +from iris.coords import DimCoord +import tarfile +import numpy as np +import six + + +class TestCmorizer(TestCase): + + def _get_variable_and_alias(self, variable): + mock_alias = Mock() + mock_alias.basin = None + mock_alias.grid = None + + mock_variable = self._get_variable(variable) + + return mock_alias, mock_variable + + def _get_variable(self, variable, silent=False): + mock_variable = Mock() + mock_variable.short_name = variable + mock_variable.domain = 'domain' + + return mock_variable + + def _get_file_path(self, *args, **kwargs): + return os.path.join(self.tmp_dir, args[3], '{0[3]}.nc'.format(args)) + + def _get_file_path_grib(self, *args, **kwargs): + return os.path.join(self.tmp_dir, args[3], str(args[6]), '{0[3]}.nc'.format(args)) + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + + self.data_manager = Mock() + self.data_manager.get_file_path = self._get_file_path + self.data_manager.is_cmorized.return_value = False + self.data_manager.config.data_dir = os.path.join(self.tmp_dir, 'data') + self.data_manager.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') + TempFile.scratch_folder = self.data_manager.config.scratch_dir + self.data_manager.config.data_convention = 'data_convention' + + self.data_manager.config.var_manager.get_variable_and_alias = self._get_variable_and_alias + self.data_manager.config.var_manager.get_variable = self._get_variable + self.data_manager.variable_list = self.data_manager.config.var_manager + + self.data_manager.config.experiment.expid = 'expid' + self.data_manager.config.experiment.model = 'model' + self.data_manager.config.experiment.experiment_name = 'experiment_name' + self.data_manager.config.experiment.num_chunks = 1 + self.data_manager.config.experiment.chunk_size = 1 + self.data_manager.config.experiment.institute = 'institute' + self.data_manager.config.experiment.get_member_str.return_value = 'member' + self.data_manager.config.experiment.atmos_timestep = 6 + + self.data_manager.config.cmor.force = False + self.data_manager.config.cmor.ocean = True + self.data_manager.config.cmor.atmosphere = True + self.data_manager.config.cmor.use_grib = True + self.data_manager.config.cmor.filter_files = '' + self.data_manager.config.cmor.associated_experiment = 'associated_experiment' + self.data_manager.config.cmor.initialization_method = 'initialization_method' + self.data_manager.config.cmor.initialization_description = 'initialization_description' + self.data_manager.config.cmor.physics_version = 'physics_version' + self.data_manager.config.cmor.physics_description = 'physics_description' + self.data_manager.config.cmor.initialization_description = 'initialization_description' + self.data_manager.config.cmor.associated_model = 'initialization_description' + self.data_manager.config.cmor.source = 'source' + self.data_manager.config.cmor.get_requested_codes.return_value = {228, 142, 143, 201, 202, 129, 169, 180} + self.data_manager.config.cmor.get_variables.return_value = {228, 142, 143, 201, 202, 129, 169, 180} + self.data_manager.config.cmor.get_levels.return_value = None + + os.makedirs(self.data_manager.config.data_dir) + os.makedirs(self.data_manager.config.scratch_dir) + + def create_ocean_files(self, filename, tar_name, gzip=False, backup=False): + coord_data = np.array([1, 2], np.float) + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', + units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', + units='degrees_east') + time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', + units='days since 1950-01-01') + depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') + + var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') + var1.add_dim_coord(time, 0) + var1.add_dim_coord(lat, 1) + var1.add_dim_coord(lon, 2) + + var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') + time.bounds = np.array([[0.5, 1.5], [1.5, 2.5]], np.float) + var2.add_dim_coord(time, 0) + var2.add_dim_coord(lat, 1) + var2.add_dim_coord(lon, 2) + var2.add_dim_coord(depth, 3) + + folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', + 'outputs') + os.makedirs(folder_path) + file_path = os.path.join(folder_path, filename) + iris.save((var1, var2), file_path, zlib=True) + + if gzip: + import subprocess + process = subprocess.Popen(('gzip', file_path), stdout=subprocess.PIPE) + comunicate = process.communicate() + file_path = "{0}.gz".format(file_path) + filename = "{0}.gz".format(filename) + if process.returncode != 0: + raise Exception('Can not compress: {0}'.format(comunicate)) + + if backup: + filename = os.path.join('backup', filename) + + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') + tar.add(file_path, arcname=filename, recursive=False) + tar.close() + os.remove(file_path) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_skip_ocean_cmorization(self): + self.data_manager.config.cmor.ocean = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertLogs([record for record in cmd.records if + record.message == 'Skipping ocean cmorization due to configuration']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_skip_atmos_cmorization(self): + self.data_manager.config.cmor.atmosphere = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + self.assertTrue([record for record in cmd.records if + record.message == 'Skipping atmosphere cmorization due to configuration']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_skip_when_cmorized(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.is_cmorized.return_value = True + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if + record.message == 'No need to unpack file 1/1']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_skip_when_not_requested(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.chunk_cmorization_requested.return_value = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if + record.message == 'No need to unpack file 1/1']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_force(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.is_cmorized.return_value = True + self.data_manager.config.cmor.force = True + self._test_ocean_cmor() + + def test_ocean_cmorization_no_files(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_ocean_cmorization_not_vars_requested(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.any_required.return_value = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_no_vars_recognized(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + + def not_recognized(*args): + return None, None + self.data_manager.config.var_manager.get_variable_and_alias = not_recognized + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_var2_not_requested(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + + def _reject_var2(cmor_var): + return cmor_var.short_name != 'var2' + + self.data_manager.config.cmor.cmorize = _reject_var2 + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def _test_ocean_cmor(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_ocean_cmorization(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_with_filter(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.filter_files = 'expid' + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_with_bad_filter(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.filter_files = 'badfilter' + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_gzip(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', gzip=True) + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_backup(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', backup=True) + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_PPO(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'PPO_expid_1D_xx_19900101_19900131.tar') + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_diags(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'diags_expid_1D_xx_19900101_19900131.tar') + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_atmos_cmorization(self): + self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + self._test_atmos_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def _test_atmos_cmor(self): + + if six.PY3: + + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + + try: + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + except AssertionError: + print(cmd) + raise + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + + def create_mma_files(self, filename, tar_name, gzip=False): + coord_data = np.array([0, 1], np.float) + folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', + 'outputs') + filepath_gg, filename_gg = self._create_file(coord_data, folder_path, filename.replace('??', 'GG'), gzip) + filepath_sh, filename_sh = self._create_file(coord_data, folder_path, filename.replace('??', 'SH'), gzip) + + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') + tar.add(filepath_gg, arcname=filename_gg, recursive=False) + tar.add(filepath_sh, arcname=filename_sh, recursive=False) + tar.close() + os.remove(filepath_gg) + os.remove(filepath_sh) + + def _create_file(self, coord_data, folder_path, filename, gzip): + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', + units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', + units='degrees_east') + time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', + units='days since 1950-01-01') + depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') + var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') + var1.add_dim_coord(time, 0) + var1.add_dim_coord(lat, 1) + var1.add_dim_coord(lon, 2) + var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') + var2.add_dim_coord(time, 0) + var2.add_dim_coord(lat, 1) + var2.add_dim_coord(lon, 2) + var2.add_dim_coord(depth, 3) + if not os.path.isdir(folder_path): + os.makedirs(folder_path) + file_path = os.path.join(folder_path, filename) + iris.save((var1, var2), file_path, zlib=True) + if gzip: + import subprocess + process = subprocess.Popen(('gzip', file_path), stdout=subprocess.PIPE) + comunicate = process.communicate() + file_path = "{0}.gz".format(file_path) + filename = "{0}.gz".format(filename) + if process.returncode != 0: + raise Exception('Can not compress: {0}'.format(comunicate)) + return file_path, filename + + def test_skip_when_not_requested_mma(self): + self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + self.data_manager.config.cmor.chunk_cmorization_requested.return_value = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + self.assertTrue([record for record in cmd.records if + record.message == 'No need to unpack file 1/1']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_force_mma(self): + self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + self.data_manager.is_cmorized.return_value = True + self.data_manager.config.cmor.force = True + self._test_atmos_cmor() + + def test_atmos_cmorization_no_mma_files(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + + def create_grib_files(self, filename, month): + filename = filename.format(month) + coord_data = np.array([0, 1], np.float) + folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', + 'outputs') + self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), [142, 143, 129, 169, 180], + month) + self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [201, 202], + month) + + def _create_file_for_grib(self, coord_data, folder_path, filename, codes, month): + month -= 1 + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', + units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', + units='degrees_east') + time_data = np.array([0.25, 0.50], np.float) + month * 31 + time = DimCoord(time_data, standard_name='time', long_name='time', var_name='time', + units='days since 1990-01-01') + vars = [] + for code in codes: + var = iris.cube.Cube(np.ones((2, 2, 2), np.float) * code, + long_name='Variable {}'.format(code), + var_name='var{}'.format(code)) + var.data[0, ...] += time_data[0] + var.data[1, ...] += time_data[1] + var.add_dim_coord(time, 0) + var.add_dim_coord(lat, 1) + var.add_dim_coord(lon, 2) + var.attributes['table'] = np.int32(128) + var.attributes['code'] = np.int32(code) + vars.append(var) + + if not os.path.isdir(folder_path): + os.makedirs(folder_path) + file_path = os.path.join(folder_path, filename) + iris.save(vars, file_path, zlib=True, local_keys=('table', 'code')) + Utils.cdo.settaxis('1990-0{}-01,06:00,6hour'.format(month + 1), + input=file_path, + output=file_path.replace('.nc', '.grb'), + options='-f grb2') + os.remove(file_path) + + def test_grib_cmorization(self): + self.data_manager.config.experiment.chunk_size = 2 + self.data_manager.get_file_path = self._get_file_path_grib + self.create_grib_files('ICM??expid+19900{}.nc', 1) + self.create_grib_files('ICM??expid+19900{}.nc', 2) + self._test_atmos_cmor() + variables = { + 'CP': 143, + 'EWSS': 180, + 'LSP': 142, + 'MN2T': 202, + 'MX2T': 201, + 'SSRD': 169, + 'TP': 228, + 'Z': 129 + } + for var, code in six.iteritems(variables): + self.assertTrue(os.path.isdir(os.path.join(self.tmp_dir, var))) + base_data = np.ones((2, 2, 2), np.float) * code + factor = 1 + month_offsets = np.array((0.375, 31.375)) + daily_offsets = np.array((0.375, 31.375)) + hourly_offsets = np.array((0.25, 0.5, 31.25, 31.5)) + if code == 129: + factor = 9.81 + elif code in (180, 169): + factor = 6 * 3600.0 + elif code == 228: + base_data = np.ones((2, 2, 2), np.float) * (142 + 143) + month_offsets *= 2 + daily_offsets *= 2 + hourly_offsets *= 2 + factor = 6 * 3600.0 / 1000 + if code == 201: + month_offsets = np.array((0.5, 31.5)) + daily_offsets = np.array((0.5, 31.5)) + elif code == 202: + month_offsets = np.array((0.25, 31.25)) + daily_offsets = np.array((0.25, 31.25)) + base_data /= factor + month_offsets /= factor + daily_offsets /= factor + hourly_offsets /= factor + + monthly = iris.load_cube(os.path.join(self.tmp_dir, var, 'mon', '{}.nc'.format(var))) + self._test_data(monthly, base_data, month_offsets, var, 'Month') + + daily = iris.load_cube(os.path.join(self.tmp_dir, var, 'day', '{}.nc'.format(var))) + self._test_data(daily, base_data, daily_offsets, var, 'Day') + + hourly = iris.load_cube(os.path.join(self.tmp_dir, var, '6hr', '{}.nc'.format(var))) + self._test_data(hourly, base_data, hourly_offsets, var, 'Hour') + + def _test_data(self, data, base_data, offsets, var, freq): + self.assertEqual(data.coord('time').shape, (len(offsets),)) + for x, offset in enumerate(offsets): + self.assertTrue(np.allclose(data.data[x, ...], base_data + offset), + '{} {} data wrong for {}'.format(freq, x, var)) diff --git a/test/run_test.py b/test/run_test.py deleted file mode 100644 index 12614344898e370d45c3c0b4c2849bfe73ac8884..0000000000000000000000000000000000000000 --- a/test/run_test.py +++ /dev/null @@ -1,31 +0,0 @@ -# coding=utf-8 -""" -Script to run the tests for EarthDiagnostics and generate the code coverage report -""" - - -import os -import sys -work_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -os.chdir(work_path) -print(work_path) -import coverage -import unittest - -cov = coverage.Coverage() -cov.start() -if len(sys.argv) == 1: - suite = unittest.TestLoader().discover('.') -else: - suite = unittest.TestLoader().discover('.', pattern=sys.argv[1]) - - -unittest.TextTestRunner(verbosity=2).run(suite) -cov.stop() - -cov.save() -cov.report() -cov.html_report() - - - diff --git a/test/unit/general/test_attribute.py b/test/unit/general/test_attribute.py index 5616b347327da016bd16340d9543c307e9c0a72b..3e2cce9b1265305094206e8db41fd72bb1330c65 100644 --- a/test/unit/general/test_attribute.py +++ b/test/unit/general/test_attribute.py @@ -50,5 +50,6 @@ class TestAttribute(TestCase): def test_str(self): mixed = Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'att', 'value') - self.assertEqual(str(mixed), 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 ' - 'Variable: atmos:var Attributte: att:value Grid: grid') + self.assertEqual(str(mixed), + 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Attributte: att:value Grid: grid') diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py index 8fa092c634374c89a8fc4f1eb060cee3aab1f2ec..426e893116d07d26deee340e3a8a3ae40dca5e0f 100644 --- a/test/unit/general/test_dailymean.py +++ b/test/unit/general/test_dailymean.py @@ -51,5 +51,6 @@ class TestDailyMean(TestCase): def test_str(self): mixed = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') - self.assertEqual(str(mixed), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + self.assertEqual(str(mixed), + 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_module.py b/test/unit/general/test_module.py index 218223b9f12e8ce9ecdc0f1613f5b698fe32f7dd..39549556dbb7de169ac2344cef559516dfb40963 100644 --- a/test/unit/general/test_module.py +++ b/test/unit/general/test_module.py @@ -50,5 +50,6 @@ class TestModule(TestCase): def test_str(self): mixed = Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'varu', 'varv', 'varmodule', 'grid') - self.assertEqual(str(mixed), 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' - 'Variables: atmos:varu,varv,varmodule Grid: grid') + self.assertEqual(str(mixed), + 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' + 'Variables: atmos:varu,varv,varmodule Grid: grid') diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py index adc75cabdd11aa21322d7254d2aecb93df40cc3b..5672c4622025ca1e28523898a7473a49b877984f 100644 --- a/test/unit/general/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -51,5 +51,6 @@ class TestMonthlyMean(TestCase): MonthlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + self.assertEqual(str(self.mixed), + 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_relink.py b/test/unit/general/test_relink.py index 970bb0ead44a43d94186e66ed979d12143f67d11..d2810bd1b3256359a4a8fa6378f6ed40339e8a55 100644 --- a/test/unit/general/test_relink.py +++ b/test/unit/general/test_relink.py @@ -57,5 +57,6 @@ class TestRelink(TestCase): def test_str(self): mixed = Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', True, 'grid') - self.assertEqual(str(mixed), 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' - 'Variable: ocean:var Grid: grid') + self.assertEqual(str(mixed), + 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/general/test_rewrite.py b/test/unit/general/test_rewrite.py index 4a6fd038e538e97fd0e20ede8a214885f289dc15..ed0cb80f9eff75167726c689f5ffc67bfc7b612e 100644 --- a/test/unit/general/test_rewrite.py +++ b/test/unit/general/test_rewrite.py @@ -47,5 +47,5 @@ class TestRewrite(TestCase): Rewrite.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: atmos:var Grid: grid') + self.assertEqual(str(self.mixed), + 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var Grid: grid') diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py index 23e1f7b7be192429b26cc07d0342f28cb34b3f74..92321f3959c1e10dc37ca515adbb954ef7eca9fd 100644 --- a/test/unit/general/test_scale.py +++ b/test/unit/general/test_scale.py @@ -75,5 +75,6 @@ class TestScale(TestCase): def test_str(self): mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, Frequencies.three_hourly, False) - self.assertEqual(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' - 'Variable: atmos:var Frequency: 3hr Apply mask: False') + self.assertEqual(str(mixed), + 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' + 'Variable: atmos:var Frequency: 3hr Apply mask: False') diff --git a/test/unit/general/test_select_levels.py b/test/unit/general/test_select_levels.py index bca61711b2bba7669207baa4d962106cd1bdccdc..146f0744345326290028499e94f5c62f6535e6fa 100644 --- a/test/unit/general/test_select_levels.py +++ b/test/unit/general/test_select_levels.py @@ -62,5 +62,6 @@ class TestSelectLevels(TestCase): def test_str(self): mixed = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 20) - self.assertEqual(str(mixed), 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Levels: 0-20 Grid: grid') + self.assertEqual(str(mixed), + 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Levels: 0-20 Grid: grid') diff --git a/test/unit/general/test_simplify_dimensions.py b/test/unit/general/test_simplify_dimensions.py index 5a36288e1c878e7e1aa38bcff9863bdd68933b5a..348f7053b6a05f9833488dc6d9cfa07504aea621 100644 --- a/test/unit/general/test_simplify_dimensions.py +++ b/test/unit/general/test_simplify_dimensions.py @@ -51,6 +51,8 @@ class TestSimplifyDimensions(TestCase): SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid', 'extra']) def test_str(self): - mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'convention') - self.assertEqual(str(mixed), 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Grid: grid') + mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + 'grid', 'convention') + self.assertEqual(str(mixed), + 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Grid: grid') diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py index ca972cac9a4ae91b57b6c10860530ce451f168ed..49f6a1b5e71c9c8075567ffa91eed4012cf2ff0d 100644 --- a/test/unit/general/test_verticalmeanmetersiris.py +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -71,5 +71,6 @@ class TestVerticalMeanMetersIris(TestCase): box.min_depth = 0 box.max_depth = 100 mixed = VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', box) - self.assertEqual(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Box: 0-100m') + self.assertEqual(str(mixed), + 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Box: 0-100m') diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py index 36b48eaf8fca6a9ea1c83787c83db9342dc8ff38..5c0d7c037633e8d239d7bab2204d9151d114979b 100644 --- a/test/unit/general/test_yearlymean.py +++ b/test/unit/general/test_yearlymean.py @@ -51,5 +51,6 @@ class TestYearlyMean(TestCase): YearlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + self.assertEqual(str(self.mixed), + 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/ocean/test_areamoc.py b/test/unit/ocean/test_areamoc.py index 648ce675f8c77d943bd77f3849294b43ac84a4f6..cb7e7e7ed9e912e520faaaf987a2da914b02eff3 100644 --- a/test/unit/ocean/test_areamoc.py +++ b/test/unit/ocean/test_areamoc.py @@ -8,7 +8,7 @@ from mock import Mock, patch class TestAreaMoc(TestCase): - + def setUp(self): self.data_manager = Mock() self.diags = Mock() @@ -53,5 +53,5 @@ class TestAreaMoc(TestCase): AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.psi), 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 ' - 'Basin: Atlantic') + self.assertEqual(str(self.psi), + 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 Basin: Atlantic') diff --git a/test/unit/ocean/test_averagesection.py b/test/unit/ocean/test_averagesection.py index a5905589db9c993aac9ae483973e8255edcefcd1..5b00d64b67456dcccf0372ef864f988d7c6a8525 100644 --- a/test/unit/ocean/test_averagesection.py +++ b/test/unit/ocean/test_averagesection.py @@ -51,5 +51,6 @@ class TestAverageSection(TestCase): def test_str(self): diag = AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', self.box, 'grid') - self.assertEqual(str(diag), 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' - 'Variable: ocean:var Grid: grid') + self.assertEqual(str(diag), + 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/ocean/test_cutsection.py b/test/unit/ocean/test_cutsection.py index e27102a0b1107e1f985e141cc8d39cf0383d32c9..c88bc6641d1bc78c09d2097a8b7c60862d2fffe3 100644 --- a/test/unit/ocean/test_cutsection.py +++ b/test/unit/ocean/test_cutsection.py @@ -51,5 +51,6 @@ class TestCutSection(TestCase): CutSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' - 'Zonal: True Value: 0') + self.assertEqual(str(self.psi), + 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' + 'Zonal: True Value: 0') diff --git a/test/unit/ocean/test_heatcontent.py b/test/unit/ocean/test_heatcontent.py index a2387016bff607bf9f668ea4a8a0bed66ba69b42..517f1d3036f702d9b8d9862d6902dc35b9775371 100644 --- a/test/unit/ocean/test_heatcontent.py +++ b/test/unit/ocean/test_heatcontent.py @@ -44,5 +44,5 @@ class TestHeatContent(TestCase): def test_str(self): diag = HeatContent(self.data_manager, '20010101', 0, 0, Basins().Global, -1, self.box, 1, 20) - self.assertEqual(str(diag), 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100 ' - 'Basin: Global') + self.assertEqual(str(diag), + 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100 Basin: Global') diff --git a/test/unit/ocean/test_interpolate.py b/test/unit/ocean/test_interpolate.py index 8bd284dc6f382dd6ebeeea2b23b5c88d21f83b94..7a348c255a9a5ef92cec230d0c1c48817ed3a22f 100644 --- a/test/unit/ocean/test_interpolate.py +++ b/test/unit/ocean/test_interpolate.py @@ -71,6 +71,7 @@ class TestInterpolate(TestCase): def test_str(self): diag = Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'model_version', True, 'original_grid') - self.assertEqual(str(diag), 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Target grid: grid Invert lat: True Model: model_version ' - 'Original grid: original_grid') + self.assertEqual(str(diag), + 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Target grid: grid Invert lat: True Model: model_version ' + 'Original grid: original_grid') diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index fccf47de39cee86b4a77ee18543246375ce2a2e2..73c6d39e01c5c6e729b4067173054654ea85d2b7 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -88,6 +88,7 @@ class TestInterpolate(TestCase): def test_str(self): diag = InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'atmos_grid', 'model_version', False, 'orig', None) - self.assertEqual(str(diag), 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' - 'Target grid: atmos_grid Original grid: orig Mask ocean: False ' - 'Model: model_version') + self.assertEqual(str(diag), + 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' + 'Target grid: atmos_grid Original grid: orig Mask ocean: False ' + 'Model: model_version') diff --git a/test/unit/ocean/test_maxmoc.py b/test/unit/ocean/test_maxmoc.py index 3224050ecb03509171c9497d29f4569b1e560866..2605e0cc18696b008c72e216ecd5127ea694f049 100644 --- a/test/unit/ocean/test_maxmoc.py +++ b/test/unit/ocean/test_maxmoc.py @@ -63,5 +63,5 @@ class TestMaxMoc(TestCase): MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.maxmoc), 'Max moc Startdate: 20000101 Member: 1 Year: 2000 ' - 'Box: 0.0N0m Basin: Global') + self.assertEqual(str(self.maxmoc), + 'Max moc Startdate: 20000101 Member: 1 Year: 2000 Box: 0.0N0m Basin: Global') diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index 2ee865fa474025d884c0dceb508fa2000036756a..313456fe3b51ec3095b0284c9fee5510c0f8827a 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -98,5 +98,6 @@ class TestRegionMean(TestCase): diag = RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box, False, 'file', True, Basins().Global) - self.assertEqual(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var ' - 'Box: 1-10 Save 3D: False Save variance: True') + self.assertEqual(str(diag), + 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Box: 1-10 ' + 'Save 3D: False Save variance: True') diff --git a/test/unit/ocean/test_rotation.py b/test/unit/ocean/test_rotation.py index 08c61fb1203f151306ca46ce601cf1c90bb70ca6..4cc90e47ebbc97f0f94d91c53742b43c1f3fdce2 100644 --- a/test/unit/ocean/test_rotation.py +++ b/test/unit/ocean/test_rotation.py @@ -43,4 +43,5 @@ class TestMixedLayerHeatContent(TestCase): Rotation.generate_jobs(self.diags, ['diagnostic', 'ocean', 'varu', 'varv', 'exe', 'extra']) def test_str(self): - self.assertEqual(str(self.mixed), 'Rotate variables Startdate: 20000101 Member: 1 Chunk: 1 Variables: ocean:varu , ocean:varv') + self.assertEqual(str(self.mixed), + 'Rotate variables Startdate: 20000101 Member: 1 Chunk: 1 Variables: ocean:varu , ocean:varv') diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py index 59ac3043f13d5431f2d1ea745a70da3d49883255..5ad93a2688c343c96973c5ab3fdb58091ffe0719 100644 --- a/test/unit/ocean/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -17,8 +17,8 @@ class TestSiasiesiv(TestCase): self.mask = Mock() self.var_manager = Mock() - self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask, self.var_manager, False) + self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, {Basins().Global: []}, self.var_manager, False) def test_str(self): self.assertEqual(str(self.psi), - 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global Omit volume: False') + 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basins: Global Omit volume: False') diff --git a/test/unit/ocean/test_verticalmean.py b/test/unit/ocean/test_verticalmean.py index f6ae18c2cd6b6801dd73da3a71c26819dd85b38a..3b25f50f94417353552cd7e751947f292d4acf34 100644 --- a/test/unit/ocean/test_verticalmean.py +++ b/test/unit/ocean/test_verticalmean.py @@ -53,5 +53,5 @@ class TestVerticalMean(TestCase): VerticalMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' - 'Box: 0-100') + self.assertEqual(str(self.mixed), + 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var Box: 0-100') diff --git a/test/unit/ocean/test_verticalmeanmeters.py b/test/unit/ocean/test_verticalmeanmeters.py index ff6580a2b9547a714c7fa7eb86b60d7dc87ced24..315e6a360bf0006c98f82a7e33272c80dcb69d22 100644 --- a/test/unit/ocean/test_verticalmeanmeters.py +++ b/test/unit/ocean/test_verticalmeanmeters.py @@ -60,5 +60,5 @@ class TestVerticalMeanMeters(TestCase): VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Box: 0-100m') + self.assertEqual(str(self.mixed), + 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 Variable: ocean:var Box: 0-100m') diff --git a/test/unit/statistics/test_climatologicalpercentile.py b/test/unit/statistics/test_climatologicalpercentile.py index 3a905cbb743cbcd189d222ed514d96c5085db3a8..385174b6b0222bdb1cd4242741a25cc13c8a8573 100644 --- a/test/unit/statistics/test_climatologicalpercentile.py +++ b/test/unit/statistics/test_climatologicalpercentile.py @@ -38,5 +38,5 @@ class TestClimatologicalPercentile(TestCase): diagnostic = ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, 11, self.diags.config.experiment) - self.assertEqual(str(diagnostic), 'Climatological percentile Variable: ocean:var Period: 2000-2001 ' - 'Forecast month: 11') + self.assertEqual(str(diagnostic), + 'Climatological percentile Variable: ocean:var Period: 2000-2001 Forecast month: 11') diff --git a/test/unit/statistics/test_daysoverpercentile.py b/test/unit/statistics/test_daysoverpercentile.py index afc1a03ec546decb3f54248098c99026cdb9e76a..caf59553e5a2f32a3788472c724d9c1b717800e4 100644 --- a/test/unit/statistics/test_daysoverpercentile.py +++ b/test/unit/statistics/test_daysoverpercentile.py @@ -30,5 +30,5 @@ class TestDaysOverPercentile(TestCase): def test_str(self): diagnostic = DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, '20001101', 11) - self.assertEqual(str(diagnostic), 'Days over percentile Startdate: 20001101 Variable: ocean:var ' - 'Climatology: 2000-2001') + self.assertEqual(str(diagnostic), + 'Days over percentile Startdate: 20001101 Variable: ocean:var Climatology: 2000-2001') diff --git a/test/unit/statistics/test_discretize.py b/test/unit/statistics/test_discretize.py index 872c510944c4e67c29fb0c2b78dc2c70e6f791ae..78a714793774464cb0acb8918d4e7584d8c65355 100644 --- a/test/unit/statistics/test_discretize.py +++ b/test/unit/statistics/test_discretize.py @@ -50,11 +50,11 @@ class TestClimatologicalPercentile(TestCase): def test_str(self): diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, 10, 40) - self.assertEqual(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' - 'Range: [10, 40]') + self.assertEqual(str(diagnostic), + 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 Range: [10, 40]') diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, float('nan'), float('nan')) - self.assertEqual(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' - 'Range: [None, None]') + self.assertEqual(str(diagnostic), + 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 Range: [None, None]') diff --git a/test/unit/statistics/test_monthlypercentile.py b/test/unit/statistics/test_monthlypercentile.py index 8127d5a4cab83216867dbedfb9e41b2b2f70d14c..d0e4da482d2da1fed31b771b313c30afd794d57a 100644 --- a/test/unit/statistics/test_monthlypercentile.py +++ b/test/unit/statistics/test_monthlypercentile.py @@ -38,4 +38,4 @@ class TestMonthlyPercentile(TestCase): def test_str(self): self.assertEqual(str(self.diagnostic), 'Monthly percentile Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Percentiles: 10, 90') + 'Variable: ocean:var Percentiles: 10, 90') diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 6a32c3674e1d621236fbd2c95f65a71e03b8d398..9c32c184f7ef58a73bb9f17321091eba9d517c39 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -28,6 +28,7 @@ class TestCMORManager(TestCase): self.config.experiment.calendar = 'standard' self.config.experiment.atmos_timestep = 3 self.config.experiment.ocean_timestep = 6 + self.config.experiment.get_member_str.return_value = 'r1i1p1' self.config.cmor.initialization_number = 1 self.config.cmor.version = '' @@ -41,6 +42,7 @@ class TestCMORManager(TestCase): self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) self.config.data_dir = self.tmp_dir + self.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') def tearDown(self): shutil.rmtree(self.tmp_dir) @@ -105,6 +107,7 @@ class TestCMORManager(TestCase): os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + def test_get_file_path_specs_empty_time_info(self): cmor_manager = CMORManager(self.config) with self.assertRaises(ValueError): @@ -165,8 +168,8 @@ class TestCMORManager(TestCase): 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_1998.nc')) - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, - 1, 'mon', year='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', + cmor_var, 1, 'mon', year='1998') def test_get_file_path_raise_incompatible_date_info(self): cmor_manager = CMORManager(self.config) @@ -232,7 +235,8 @@ class TestCMORManager(TestCase): self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expnameS19900101/' 'r2i1p1f1/Omon/var/ocean_grid/version/' - 'var_Omon_model_expnameS19900101_r2i1p1f1_ocean_grid_198901-198912.nc')) + 'var_Omon_model_expnameS19900101_r2i1p1f1_ocean_grid_' + '198901-198912.nc')) def test_get_file_path_primavera_no_cmor(self): self._configure_primavera() @@ -314,8 +318,8 @@ class TestCMORManager(TestCase): 'var/ocean_grid/version/' 'var_Omon_model_expname_r2i1p1f1_ocean_grid_1998.nc')) - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, - 1, 'mon', year='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', + cmor_var, 1, 'mon', year='1998') def test_get_file_path_primavera_date_str(self): self._configure_primavera() @@ -346,7 +350,7 @@ class TestCMORManager(TestCase): self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1, possible_versions=('version1', 'version2'))) isfile.return_value = False - self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1, possible_versions=('version1', 'version2'))) def test_get_file_path_meteofrance(self): @@ -529,7 +533,6 @@ class TestCMORManager(TestCase): original_cmor_path = os.path.join(self.config.data_dir, self.config.experiment.expid, 'original_files', 'cmorfiles') os.makedirs(original_cmor_path) - self.config.experiment.get_member_str.return_value = 'r1i1p1' self.config.experiment.get_chunk_start_str.return_value = '20000101' cmor_prefix = 'CMORT_{0}_{1}_{2}_{3}-'.format(self.config.experiment.expid, '20000101', 'r1i1p1', '20000101') @@ -797,8 +800,6 @@ class TestCMORManager(TestCase): datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) self.assertEqual(datafile.remote_file, '/path/to/file') - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') def test_request_year(self, mock_get_file_path): self.config.experiment.get_year_chunks.return_value = (1, 2) @@ -833,6 +834,3 @@ class TestMergeYear(TestCase): MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998)) self.assertEqual(MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998, 'grid'), MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998, 'grid')) - - - diff --git a/test/unit/test_config.py b/test/unit/test_config.py index b1ee9671af1d38017c22320ca37567adf74e3b8a..8f7a610dce1f6c04915ca4620177528e51f7394e 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -204,10 +204,10 @@ class TestCMORConfig(TestCase): self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.six_hourly), {128: None, - 129: '1', - 130: '1,2', - 131: '1,2,3,4,5,6,7,8,9', - 132: '0,5'}) + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) self.assertEqual(config.get_levels(Frequencies.six_hourly, 128), None) self.assertEqual(config.get_levels(Frequencies.six_hourly, 129), '1') @@ -222,10 +222,10 @@ class TestCMORConfig(TestCase): self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.daily), {128: None, - 129: '1', - 130: '1,2', - 131: '1,2,3,4,5,6,7,8,9', - 132: '0,5'}) + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) self.assertEqual(config.get_levels(Frequencies.daily, 128), None) self.assertEqual(config.get_levels(Frequencies.daily, 129), '1') @@ -240,10 +240,10 @@ class TestCMORConfig(TestCase): self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.monthly), {128: None, - 129: '1', - 130: '1,2', - 131: '1,2,3,4,5,6,7,8,9', - 132: '0,5'}) + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) self.assertEqual(config.get_levels(Frequencies.monthly, 128), None) self.assertEqual(config.get_levels(Frequencies.monthly, 129), '1') @@ -355,8 +355,8 @@ class TestExperimentConfig(TestCase): config = ExperimentConfig() config.parse_ini(self.mock_parser) self.assertEqual(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', - u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', - u'20020801', u'20021101']) + u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', + u'20020801', u'20021101']) def test_auto_startdates(self): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') @@ -483,7 +483,8 @@ class TestConfig(TestCase): with mock.patch('earthdiagnostics.config.ExperimentConfig', new=mock_new_exp): with mock.patch('earthdiagnostics.config.CMORConfig'): with mock.patch('earthdiagnostics.config.THREDDSConfig'): - config.parse('path') + with mock.patch('os.path.isfile'): + config.parse('path') def test_diags(self): config = Config() @@ -491,6 +492,11 @@ class TestConfig(TestCase): self._parse(config) self.assertEqual(config.get_commands(), (['diag1', 'diag2,opt1,opt2'])) + def test_file_not_found(self): + config = Config() + with self.assertRaises(ValueError): + config.parse('path') + def test_parse(self): config = Config() self._parse(config) @@ -524,8 +530,3 @@ class TestConfig(TestCase): namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'earthdiagnostics/CDFTOOLS_primavera.namlist')) self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) - - - - - diff --git a/test/unit/test_data_manager.py b/test/unit/test_data_manager.py index 00d03b8c1edae697eb2aea6586e00a565466f33a..cb4c7af87f9f957182847c9bba20e06bf34693ef 100644 --- a/test/unit/test_data_manager.py +++ b/test/unit/test_data_manager.py @@ -65,5 +65,3 @@ class TestDataManager(TestCase): box.get_lat_str.return_value = '_lat' box.get_depth_str.return_value = '_depth' self.assertEqual(self.data_manager._get_final_var_name(box, 'var'), 'var_lon_lat_depth') - - diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index b1ff6c82e7fef19c093785783716f0f9fb3e40e1..2ee054807a3df7094d43e33eb9e5fb3bb7a72414 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -366,3 +366,19 @@ class TestDiagnostic(TestCase): def test_empty_process_options(self): self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) + + +class TestDiagnosticBasinOption(TestCase): + + @patch.object(Basins, 'parse') + def test_not_recognized(self, parse_mock): + parse_mock.return_value = 'var1' + diag = DiagnosticBasinOption('basin') + self.assertEqual('var1', diag.parse('var1')) + + @patch.object(Basins, 'parse') + def test_not_recognized(self, parse_mock): + parse_mock.return_value = None + diag = DiagnosticBasinOption('basin') + with self.assertRaises(DiagnosticOptionError): + diag.parse('bad') diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py index 95468b71af8911089fd5083b6f05a1ef04baa881..c1f3a0ccf120168fa1ac1187a45c3d13c8f63b38 100644 --- a/test/unit/test_earthdiags.py +++ b/test/unit/test_earthdiags.py @@ -1,15 +1,130 @@ # coding=utf-8 from unittest import TestCase +import os +import tempfile from earthdiagnostics.earthdiags import EarthDiags +from earthdiagnostics.utils import TempFile, Utils +from earthdiagnostics import cdftools +from bscearth.utils.log import Log -# -# class TestEarthDiags(TestCase): -# -# def setUp(self): -# self.earthdiags = EarthDiags('path/to/conf') -# -# self.earthdiags.config() -# -# def test_clean(self): -# self.earthdiags.parse_args() +import mock +from mock import Mock + + +class TestEarthDiags(TestCase): + + def setUp(self): + self.earthdiags = EarthDiags() + + def test_clean(self): + self.earthdiags.config.scratch_dir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.earthdiags.config.scratch_dir, 'extra_folder')) + + self.assertTrue(os.path.isdir(self.earthdiags.config.scratch_dir)) + self.earthdiags.clean() + self.assertFalse(os.path.isdir(self.earthdiags.config.scratch_dir)) + + def test_clean_link(self): + temp_folder = tempfile.mkdtemp() + scratch_dir = tempfile.mktemp() + os.symlink(temp_folder, scratch_dir) + os.makedirs(os.path.join(temp_folder, 'extra_folder')) + self.earthdiags.config.scratch_dir = scratch_dir + + self.assertTrue(os.path.islink(self.earthdiags.config.scratch_dir)) + self.assertTrue(os.path.isdir(temp_folder)) + self.earthdiags.clean() + self.assertFalse(os.path.islink(self.earthdiags.config.scratch_dir)) + self.assertFalse(os.path.isdir(temp_folder)) + + def test_read_config(self): + self.earthdiags.config = Mock() + self.earthdiags.config.scratch_dir = 'scratch_path' + self.earthdiags.config.cdftools_path = 'cdftools_path' + self.earthdiags.config.data_convention = 'SPECS' + + self.earthdiags.read_config('config/path') + + self.earthdiags.config.parse.assert_called_once_with('config/path') + self.assertEqual(os.environ['HDF5_USE_FILE_LOCKING'], 'FALSE') + self.assertEqual(self.earthdiags.config.scratch_dir, TempFile.scratch_folder) + self.assertEqual(self.earthdiags.config.cdftools_path, cdftools.path) + self.assertDictEqual(self.earthdiags.dic_variables, + {'nav_lat': 'lat', + 'nav_lev': 'lev', + 'nav_lon': 'lon', + 't': 'time', + 'time_counter': 'time', + 'x': 'i', + 'y': 'j', + 'z': 'lev'}) + + def test_read_config_primavera(self): + self.earthdiags.config = Mock() + self.earthdiags.config.scratch_dir = 'scratch_path' + self.earthdiags.config.cdftools_path = 'cdftools_path' + self.earthdiags.config.data_convention = 'PRIMAVERA' + + self.earthdiags.read_config('config/path') + + self.earthdiags.config.parse.assert_called_once_with('config/path') + self.assertEqual(os.environ['HDF5_USE_FILE_LOCKING'], 'FALSE') + self.assertEqual(self.earthdiags.config.scratch_dir, TempFile.scratch_folder) + self.assertEqual(self.earthdiags.config.cdftools_path, cdftools.path) + self.assertDictEqual(self.earthdiags.dic_variables, + {'nav_lat': 'latitude', + 'nav_lev': 'lev', + 'nav_lon': 'longitude', + 't': 'time', + 'time_counter': 'time', + 'x': 'i', + 'y': 'j', + 'z': 'lev'}) + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.open_documentation') + def test_parse_args_open_doc(self, open_doc): + EarthDiags.parse_args(['--doc']) + open_doc.assert_called_once() + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.clean') + def test_parse_args_clean(self, clean, read_config): + EarthDiags.parse_args(['--clean']) + clean.assert_called_once() + read_config.assert_called_once_with('diags.conf') + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.report') + def test_parse_args_report(self, report, read_config): + EarthDiags.parse_args(['--report']) + report.assert_called_once() + read_config.assert_called_once_with('diags.conf') + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.run') + def test_parse_args_run(self, run, read_config): + EarthDiags.parse_args([]) + run.assert_called_once() + read_config.assert_called_once_with('diags.conf') + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.run') + def test_log_levels(self, run, read_config): + log_file = tempfile.mktemp() + for level in ('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', + 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'): + EarthDiags.parse_args(['-lc', level, '-lf', level, '-log', log_file]) + self.assertEqual(getattr(Log, level), Log.console_handler.level) + self.assertEqual(getattr(Log, level), Log.file_handler.level) + if os.path.isfile(log_file): + os.remove(log_file) + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.run') + def test_log_cdo_nco(self, run, read_config): + EarthDiags.parse_args(['-lc', 'DEBUG']) + run.assert_called_once() + read_config.assert_called_once_with('diags.conf') + self.assertTrue(Utils.cdo.debug) + self.assertTrue(Utils.nco.debug) diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py index b92069e348857283a1bae08b7994181810170ed4..be4241816efb2155e1bd4deb19a4f98ca96c4b3e 100644 --- a/test/unit/test_frequency.py +++ b/test/unit/test_frequency.py @@ -40,4 +40,3 @@ class TestFrequency(TestCase): def test_get_climatology(self): self.assertEqual(Frequency('clim').folder_name(VariableType.STATISTIC), 'clim') self.assertEqual(Frequency('clim').folder_name(VariableType.MEAN), 'clim') - diff --git a/test/unit/test_lint.py b/test/unit/test_lint.py new file mode 100644 index 0000000000000000000000000000000000000000..03bdeb4bb9eb527d7d3daca63791790409762fc9 --- /dev/null +++ b/test/unit/test_lint.py @@ -0,0 +1,32 @@ +""" Lint tests """ +import os +import textwrap +import unittest + +import pycodestyle # formerly known as pep8 + + +class TestLint(unittest.TestCase): + + def test_pep8_conformance(self): + """Test that we conform to PEP-8.""" + check_paths = [ + 'earthdiagnostics', + 'test', + ] + exclude_paths = [ + ] + + print("PEP8 check of directories: {}\n".format(', '.join(check_paths))) + + # Get paths wrt package root + package_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + for paths in (check_paths, exclude_paths): + for i, path in enumerate(paths): + paths[i] = os.path.join(package_root, path) + + style = pycodestyle.StyleGuide() + style.options.exclude.extend(exclude_paths) + style.options.max_line_length = 120 + + self.assertEqual(style.check_files(check_paths).total_errors, 0) diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 3752346f9d46106d6338064eec46d6b8a37ecf11..103bcdc387f20a47a40f1870d24fe9f17568dd39 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -49,7 +49,3 @@ class TestModellingRealm(TestCase): def test_get_6hrplev(self): self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev') - - - - diff --git a/test/unit/test_obsreconmanager.py b/test/unit/test_obsreconmanager.py index 05d98f3527e9627a16607068f40a58af60163eb5..b04cc568dfdf7965466195b2af68cb8c01c99947 100644 --- a/test/unit/test_obsreconmanager.py +++ b/test/unit/test_obsreconmanager.py @@ -136,7 +136,3 @@ class TestObsReconManager(TestCase): cmor_manager = ObsReconManager(self.config) datafile = cmor_manager.request_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) self.assertEqual(datafile.remote_file, '/path/to/file') - - - - diff --git a/test/unit/test_publisher.py b/test/unit/test_publisher.py index a91248322496017267a5de99c41f79642c9f8a3b..f884f46adf8b91465d192da92a67950306c6d2e8 100644 --- a/test/unit/test_publisher.py +++ b/test/unit/test_publisher.py @@ -45,4 +45,3 @@ class TestPublisher(TestCase): pub.subscribe(suscriber2, callback=suscriber.callback) self.assertFalse(pub.only_suscriber(suscriber)) self.assertFalse(pub.only_suscriber(suscriber2)) - diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 2162cab9b43f653e2f30cfeee4be8d3f6ef6b7ef..3b274e3f1075d34f70d6d1253570d79f09268af6 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -57,4 +57,3 @@ class TestUtils(TestCase): Utils.rename_variable('file', 'old', 'new', False, True) rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True, False), mock.call('file', {'old': 'new'}, False, True))) - diff --git a/test/unit/test_variable_type.py b/test/unit/test_variable_type.py index 5b98e6aef1ab19b87b19da8040fbe6ba818f5b46..cb3dfa0549a7f4aedd0774b9f158577094d2914f 100644 --- a/test/unit/test_variable_type.py +++ b/test/unit/test_variable_type.py @@ -15,6 +15,3 @@ class TestVariableType(TestCase): def test_bad_one(self): with self.assertRaises(ValueError): VariableType.to_str('bad type') - - - diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index ad5c88b64a2073169d534892f99a2041f3f9ae73..121e56173ad00343d13137681d41e29e59fc67e4 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -1,8 +1,11 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.work_manager import Downloader, WorkManager -from earthdiagnostics.diagnostic import Diagnostic from mock import Mock +import six +from earthdiagnostics.work_manager import Downloader, WorkManager +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticStatus +from earthdiagnostics.datafile import DataFile, StorageStatus, LocalStatus +from bscearth.utils import log class TestDownloader(TestCase): @@ -25,16 +28,186 @@ class TestDownloader(TestCase): self.downloader.submit(datafile) self.downloader.start() self.downloader.shutdown() - datafile.download.assert_called_once() + def test_download_fails(self): + datafile = Mock() -class TestWorkManager(TestCase): + def _download(): + raise Exception('Error') + datafile.download = _download + self.downloader.submit(datafile) + if six.PY3: + with self.assertLogs(log.Log.log, log.Log.CRITICAL): + self.downloader.start() + self.downloader.shutdown() + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_smaller_first(self): + small_file = self._create_datafile_mock(size=1) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=10) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 1', + 'INFO:bscearth.utils:Suscribers: () Size: 10']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_not_none_first(self): + small_file = self._create_datafile_mock(size=None) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=10) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 10', + 'INFO:bscearth.utils:Suscribers: () Size: None']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_not_none_second(self): + small_file = self._create_datafile_mock(size=1) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=None) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 1', + 'INFO:bscearth.utils:Suscribers: () Size: None']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_both_none(self): + small_file = self._create_datafile_mock(size=None) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=None) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: None', + 'INFO:bscearth.utils:Suscribers: () Size: None']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_can_not_order(self): + small_file = self._create_datafile_mock(size=1) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=1) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 1', + 'INFO:bscearth.utils:Suscribers: () Size: 1']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_more_suscribers_first(self): + no_suscribers = self._create_datafile_mock() + self.downloader.submit(no_suscribers) + suscriber = Mock() + one_suscriber = self._create_datafile_mock(suscribers=(suscriber, )) + self.downloader.submit(one_suscriber) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: ({0},) Size: 0'.format(str(suscriber)), + 'INFO:bscearth.utils:Suscribers: () Size: 0']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_more_suscribers_waiting_first(self): + + class StatusDiag(Diagnostic): + def __init__(self, data_manager, pending): + super(StatusDiag, self).__init__(data_manager) + self.pending = pending + + def __str__(self): + return str(self.pending) + + def pending_requests(self): + return self.pending + + suscriber = StatusDiag(Mock(), 1) + suscriber.status = DiagnosticStatus.WAITING + one_waiting_suscriber = self._create_datafile_mock(suscribers=(suscriber,)) + self.downloader.submit(one_waiting_suscriber) + suscriber = StatusDiag(Mock(), 2) + suscriber.status = DiagnosticStatus.FAILED + one_failed_suscriber = self._create_datafile_mock(suscribers=(suscriber, )) + self.downloader.submit(one_failed_suscriber) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: (2,) Size: 0', + 'INFO:bscearth.utils:Suscribers: (1,) Size: 0']) + else: + self.downloader.start() + self.downloader.shutdown() + + def _create_datafile_mock(self, size=0, suscribers=tuple()): + data_mock = Mock() + data_mock.size = size + data_mock.suscribers = suscribers + + def _download(): + log.Log.info('Suscribers: {0.suscribers} Size: {0.size}'.format(data_mock)) + data_mock.download = _download + + return data_mock +class MockFile(DataFile): + + def download(self): + self.local_status = LocalStatus.READY + + def upload(self): + self.storage_status = StorageStatus.READY + + def clean_local(self): + pass + + +class TestWorkManager(TestCase): + def setUp(self): self.config = Mock() self.data_manager = Mock() + self.config.max_cores = 1 + self.config.parallel_uploads = 1 + self.data_manager.requested_files = {} + self.data_manager.request_chunk = self._request_chunk + self.data_manager.declare_chunk = self._declare_chunk self.work_manager = WorkManager(self.config, self.data_manager) def test_prepare_job_list(self): @@ -52,3 +225,156 @@ class TestWorkManager(TestCase): self.config.get_commands.return_value = ['diag1', 'baddiag', 'diag1,badoption'] Diagnostic.register(Diag1) self.work_manager.prepare_job_list() + + def test_run(self): + self.config.max_cores = -1 + + class EmptyDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Null diag' + + def compute(self): + pass + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + self.work_manager.jobs = [EmptyDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) + + def test_diag_can_be_skipped(self): + self.config.skip_diags_done = True + + class SkippedDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Skipped diag' + + def compute(self): + raise Exception('Diagnostic should not be computed') + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + def can_skip_run(self): + return True + + self.work_manager.jobs = [SkippedDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) + + def test_diag_can_be_skipped_but_no(self): + self.config.skip_diags_done = False + + class SkippedDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Skipped diag' + + def compute(self): + raise Exception('Diagnostic should not be computed') + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + def can_skip_run(self): + return True + + self.work_manager.jobs = [SkippedDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.FAILED) + + def test_failed_run(self): + class FailDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Fail diag' + + def compute(self): + raise Exception('Must fail') + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + def can_skip_run(self): + return False + + self.work_manager.jobs = [FailDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.FAILED) + + def _request_chunk(self, *args, **kwargs): + req = MockFile() + req.storage_status = StorageStatus.READY + req.local_status = LocalStatus.PENDING + req.remote_file = 'requested' + self.data_manager.requested_files[req.remote_file] = req + return req + + def _declare_chunk(self, *args, **kwargs): + req = MockFile() + req.remote_file = 'generated' + self.data_manager.requested_files[req.remote_file] = req + req.storage_status = StorageStatus.PENDING + req.local_status = LocalStatus.NOT_REQUESTED + return req + + def test_run_data(self): + self.data_manager.config.max_cores = -1 + + class DataDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Data diag' + + def compute(self): + self.declared.local_status = LocalStatus.READY + + def request_data(self): + self.request = self.request_chunk(None, None, None, None, None) + + def declare_data_generated(self): + self.declared = self.declare_chunk(None, None, None, None, None) + + self.work_manager.jobs = [DataDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) + + def test_run_empty(self): + self.work_manager.jobs = [] + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.work_manager.run() + self.assertTrue([record for record in cmd.records if + record.message == 'No diagnostics to run']) + else: + self.work_manager.run()