diff --git a/.codacy.yml b/.codacy.yml index 445a627b511319070406c4ab67202651e780aea8..3e730e5f2f81b1a6fdeaf83ae87f1e092dfb3218 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -5,9 +5,6 @@ engines: coverage: enabled: true - exclude_paths: [ - 'tests', - ] metrics: enabled: true duplication: @@ -20,6 +17,5 @@ engines: exclude_paths: [ 'doc/**', - 'test/**', 'earthdiagnostics/cmor_tables/**', ] diff --git a/.coveragerc b/.coveragerc index 6c77e847caf3215fbe9907faf6eda073d963b626..95b76079e4a83d97539bc0fd6adc2ca0d946ac78 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,6 @@ [run] branch = True -source = earthdiagnostics +source = earthdiagnostics test [html] title = Coverage report for EarthDiagnostics diff --git a/VERSION b/VERSION index e2859adcb157b1787daaca9a0489d008bbfbab4b..7a113d6c3a110bd924a12ac1cac92cf8f78e11fc 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc9 +3.0.0rc10 diff --git a/earthdiagnostics/cdftoolspython.so b/earthdiagnostics/cdftoolspython.so deleted file mode 100755 index 2d3916295e882a4723b0fdda32ceab0ed0d26105..0000000000000000000000000000000000000000 Binary files a/earthdiagnostics/cdftoolspython.so and /dev/null differ diff --git a/earthdiagnostics/cmor_tables/default.csv b/earthdiagnostics/cmor_tables/default.csv index fd13cc4fa08ba52ef6288fb180cdf81cf86598d4..9d562abff63d9aa8d003e8fd1109b67bc047bfc1 100644 --- a/earthdiagnostics/cmor_tables/default.csv +++ b/earthdiagnostics/cmor_tables/default.csv @@ -1,4 +1,5 @@ Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables +hfxout,hfxout,,Ocean surface heat flux,ocean,,W m-2,,,,, iiceages:siage:iice_otd,ageice,age_of_sea_ice,Age of sea ice,seaIce,,,,,, al,al,surface_albedo,Albedo,atmos,,,,,, bgfrcsal,bgfrcsal,change_over_time_in_heat_content_from_forcing,Change over time in salt content from forcing,ocean,,,,,, @@ -345,4 +346,4 @@ zqsb,hfsso,surface_downward_sensible_heat_flux,Surface Downward Sensible Heat Fl zqlw,rlntds,surface_net_downward_longwave_flux,Surface Net Downward Longwave Radiation,ocean,,W m-2,,,, var78,tclw,total_column_liquid_water,Total column liquid water,atmos,,kg m-2,,,, var79,tciw,total_column_ice_water,Total column ice water,atmos,,kg m-2,,,, -rho,rhopoto,sea_water_potential_density,Sea Water Potential Density,ocean,,kg m-3,,,, \ No newline at end of file +rho,rhopoto,sea_water_potential_density,Sea Water Potential Density,ocean,,kg m-3,,,, diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 87c2169ac64df3590c5fec0adf402829c4679e2b..7e80ebb04ebeb2150129ac1d3a85d47acf15c37a 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -50,23 +50,21 @@ class Cmorizer(object): def __init__(self, data_manager, startdate, member): self.data_manager = data_manager + self.startdate = startdate self.member = member self.config = data_manager.config self.experiment = self.config.experiment self.cmor = self.config.cmor + self.convetion = self.config.data_convention self.member_str = self.experiment.get_member_str(member) self.original_files_path = os.path.join(self.config.data_dir, self.experiment.expid, 'original_files', self.startdate, self.member_str, 'outputs') self.atmos_timestep = None self.cmor_scratch = str(os.path.join(self.config.scratch_dir, 'CMOR', self.startdate, self.member_str)) - if self.config.data_convention in ('primavera', 'cmip6'): - self.lon_name = 'longitude' - self.lat_name = 'latitude' - else: - self.lon_name = 'lon' - self.lat_name = 'lat' + self.lon_name = self.config.data_convention.lon_name + self.lat_name = self.config.data_convention.lat_name self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', 'time_counter_bounds': 'time_bnds', @@ -465,10 +463,11 @@ class Cmorizer(object): netcdf_file = NetCDFFile() netcdf_file.data_manager = self.data_manager netcdf_file.local_file = temp - netcdf_file.remote_file = self.data_manager.get_file_path(self.startdate, self.member, - var_cmor.domain, var_cmor.short_name, var_cmor, - None, frequency, - grid=alias.grid, year=None, date_str=date_str) + netcdf_file.remote_file = self.config.data_convention.get_file_path(self.startdate, self.member, + var_cmor.domain, var_cmor.short_name, + var_cmor, None, frequency, + grid=alias.grid, year=None, + date_str=date_str) netcdf_file.data_convention = self.config.data_convention netcdf_file.region = region @@ -494,7 +493,9 @@ class Cmorizer(object): cube.coord(lev_original).var_name = lev_target except iris.exceptions.CoordinateNotFoundError: pass + iris.save(cube, temp, zlib=True) + Utils.rename_variables(temp, {'dim1': 'j', 'dim2': 'i'}, must_exist=False, rename_dimension=True) def _set_coordinates_attribute(self, file_path, var_cmor, variable): handler = Utils.open_cdf(file_path) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 43122007251749b3e181b6091f3051130dbf5f7f..901d49d2726a680085647574d45b0851cbc6969d 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -229,7 +229,8 @@ class DataConvention(object): if not grid: grid = 'original' - + if domain is None: + pass variable_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, self.config.experiment.atmos_timestep) vargrid_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, @@ -630,7 +631,12 @@ class Cmor3Convention(DataConvention): frequency = self.config.var_manager.tables[table].frequency Log.debug('Creating links for table {0}', table) for var in os.listdir(os.path.join(path, member, table)): - domain = self.config.var_manager.get_variable(var, silent=True).domain + cmor_var = self.config.var_manager.get_variable(var, silent=True) + domain = None + if cmor_var is not None: + domain = cmor_var.domain + if domain is None: + domain = self.config.var_manager.tables[table].domain for grid in os.listdir(os.path.join(path, member, table, var)): if member_str is not None and member_str != member: continue diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 3a6c38f1d17b49eae443ff10fce792af1e791e1c..35fa33f45933348b6bf4db20a3224b8f15224c28 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -90,9 +90,16 @@ class SelectLevels(Diagnostic): def compute(self): """Run the diagnostic""" temp = TempFile.get() - - Utils.nco.ncks(input=self.variable_file, output=temp, - options=('-O -d lev,{0.min_depth},{0.max_depth}'.format(self.box),)) + handler = Utils.open_cdf(self.variable_file.local_file) + var_name = "" + for var in ('lev', 'plev'): + if var in handler.variables: + var_name = var + continue + handler.close() + + Utils.nco.ncks(input=self.variable_file.local_file, output=temp, + options=('-O -d {1},{0.min_depth},{0.max_depth}'.format(self.box, var_name),)) self.result.set_local_file(temp) @staticmethod diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index 7c9b607c4dd7d579af467c627fa3f16f0d4b2e38..ce91670d45c7f5483e484e0f24abc388044fc381 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -120,7 +120,7 @@ class ModelingRealm(object): """ table_name = self.get_table_name(frequency, data_convention) from earthdiagnostics.variable import CMORTable - return CMORTable(table_name, frequency, 'December 2013') + return CMORTable(table_name, frequency, 'December 2013', self) class ModelingRealms(object): diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 705aea3b566a7d478c3138a4cc365958f7712c1d..2468d5fda3eda613667ebb8b5630c27f1af4ed85 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -33,6 +33,7 @@ class VariableManager(object): self._dict_variables = {} self._dict_aliases = {} self.tables = {} + self.table_name = None def get_variable(self, original_name, silent=False): """ @@ -89,37 +90,38 @@ class VariableManager(object): table_name: str """ + self.table_name = table_name self._dict_variables = dict() - self._load_variable_list(table_name) + self._load_variable_list() self._load_missing_defaults() - self._load_known_aliases(table_name) + self._load_known_aliases() self.create_aliases_dict() - def _load_variable_list(self, table_name): + def _load_variable_list(self): - xlsx_path = self._get_xlsx_path(table_name) + xlsx_path = self._get_xlsx_path() if xlsx_path: self._load_xlsx(xlsx_path) return - json_folder = self._get_json_folder(table_name) + json_folder = self._get_json_folder() if os.path.isdir(json_folder): self._load_json(json_folder) return - csv_path = self._get_csv_path(table_name) + csv_path = self._get_csv_path(self.table_name) if os.path.isfile(csv_path): - self._load_file(table_name) + self._load_file(self.table_name) return - raise Exception('Data convention {0} unknown'.format(table_name)) + raise Exception('Data convention {0} unknown'.format(self.table_name)) def _get_csv_path(self, table_name): csv_table_path = os.path.join(self._cmor_tables_folder, '{0}.csv'.format(table_name)) return csv_table_path - def _get_json_folder(self, table_name): - json_folder = os.path.join(self._cmor_tables_folder, '{0}/Tables'.format(table_name)) + def _get_json_folder(self): + json_folder = os.path.join(self._cmor_tables_folder, '{0}/Tables'.format(self.table_name)) return json_folder def _load_file(self, csv_table_path, default=False): @@ -169,7 +171,8 @@ class VariableManager(object): table_id = data['Header']['table_id'][6:] table = CMORTable(table_id, Frequency(data['variable_entry'].values()[0]['frequency']), - data['Header']['table_date']) + data['Header']['table_date'], + ModelingRealms.parse(data['Header']['realm'])) self.tables[table_id] = table self._load_json_variables(data['variable_entry'], table) @@ -189,9 +192,9 @@ class VariableManager(object): except VariableJsonException: Log.error('Could not read variable {0}'.format(short_name)) - def _load_known_aliases(self, table_name): + def _load_known_aliases(self): self._load_alias_csv('default') - self._load_alias_csv(table_name) + self._load_alias_csv(self.table_name) def _load_alias_csv(self, filename): file_path = self._get_aliases_csv_path(filename) @@ -262,12 +265,12 @@ class VariableManager(object): for alias in cmor_var.known_aliases: self._dict_aliases[alias.alias] = (alias, cmor_var) - def _get_xlsx_path(self, table_name): - xlsx_table_path = os.path.join(self._cmor_tables_folder, '{0}.xlsx'.format(table_name)) + def _get_xlsx_path(self): + xlsx_table_path = os.path.join(self._cmor_tables_folder, '{0}.xlsx'.format(self.table_name)) if os.path.isfile(xlsx_table_path): return xlsx_table_path - xlsx_table_path = os.path.join(self._cmor_tables_folder, table_name, 'etc', '*.xlsx') + xlsx_table_path = os.path.join(self._cmor_tables_folder, self.table_name, 'etc', '*.xlsx') xlsx_table_path = glob.glob(xlsx_table_path) if len(xlsx_table_path) == 1: return xlsx_table_path[0] @@ -292,10 +295,11 @@ class VariableManager(object): def _load_xlsx_table(self, sheet, table_data): try: table_frequency, table_date = table_data[sheet.title] - table = CMORTable(sheet.title, table_frequency, table_date) + realm = self._read_realm_from_json(sheet.title) + table = CMORTable(sheet.title, table_frequency, table_date, realm) self.tables[sheet.title] = table for row in sheet.rows: - if row[0].value == 'Priority' or not row[5].value: + if row[0].value in ('Priority', 'rm') or not row[5].value: continue self._parse_xlsx_var_row(row, table) except Exception as ex: @@ -303,6 +307,18 @@ class VariableManager(object): import traceback traceback.print_exc() + def _read_realm_from_json(self, table_name): + for prefix in ('CMIP6', 'PRIMAVERA'): + json_path = os.path.join(self._get_json_folder(), '{0}_{1}.json'.format(prefix, table_name)) + if os.path.isfile(json_path): + with open(json_path) as json_file: + json_data = json_file.read() + data = json.loads(json_data) + # Cogemos el primer realm para las tablas que tienen varios + # Solo se usa al generar los links para una startdate concreta + return ModelingRealms.parse(data['Header']['realm'].split(' ')[0]) + return None + def _parse_xlsx_var_row(self, row, table): cmor_name = row[11].value if not cmor_name: @@ -487,7 +503,7 @@ class Variable(object): return table if self.domain: table_name = self.domain.get_table_name(frequency, data_convention) - return CMORTable(table_name, frequency, 'December 2013') + return CMORTable(table_name, frequency, 'December 2013', self.domain) raise ValueError('Can not get table for {0} and frequency {1}'.format(self, frequency)) @staticmethod @@ -554,16 +570,17 @@ class CMORTable(object): date: str """ - def __init__(self, name, frequency, date): + def __init__(self, name, frequency, date, domain): self.name = name self.frequency = frequency self.date = date + self.domain = domain def __str__(self): return self.name def __repr__(self): - return '{0.name} ({0.frequency}, {0.date})'.format(self) + return '{0.name} ({0.domain} {0.frequency}, {0.date})'.format(self) def __lt__(self, other): return self.name < other.name diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index abe27695a77bc23c5fc31fe4a8df778c9290eb42..bcb41e725374aa7dd0b8a39ed03e668e647b18c6 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -1,3 +1,4 @@ +"""Tests for earthdiagnostics.cmorizer module""" from earthdiagnostics.cmorizer import Cmorizer from earthdiagnostics.utils import TempFile, Utils from earthdiagnostics.data_convention import DataConvention @@ -18,6 +19,7 @@ import calendar class TestCmorizer(TestCase): + """Tests for Cmorizer class""" def _get_variable_and_alias(self, variable): mock_alias = Mock() @@ -36,22 +38,26 @@ class TestCmorizer(TestCase): return mock_variable def _get_file_path(self, *args, **kwargs): - return os.path.join(self.tmp_dir, args[3], '{0[3]}.nc'.format(args)) + return os.path.join(self.tmp_dir, args[3], '{0[3]}.nc'.format(args)) def _get_file_path_grib(self, *args, **kwargs): return os.path.join(self.tmp_dir, args[3], str(args[6]), '{0[3]}.nc'.format(args)) def setUp(self): + """Prepare tests""" self.tmp_dir = tempfile.mkdtemp() self.data_manager = Mock() - self.data_manager.get_file_path = self._get_file_path + self.data_manager.is_cmorized.return_value = False self.data_manager.config.data_dir = os.path.join(self.tmp_dir, 'data') self.data_manager.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') TempFile.scratch_folder = self.data_manager.config.scratch_dir self.data_manager.config.data_convention = create_autospec(DataConvention) self.data_manager.config.data_convention.name = 'data_convention' + self.data_manager.config.data_convention.lat_name = 'lat' + self.data_manager.config.data_convention.lon_name = 'lon' + self.data_manager.config.data_convention.get_file_path = self._get_file_path self.data_manager.config.var_manager.get_variable_and_alias = self._get_variable_and_alias self.data_manager.config.var_manager.get_variable = self._get_variable @@ -86,34 +92,39 @@ class TestCmorizer(TestCase): os.makedirs(self.data_manager.config.data_dir) os.makedirs(self.data_manager.config.scratch_dir) - def create_ocean_files(self, filename, tar_name, gzip=False, backup=False): - coord_data = np.array([1, 2], np.float) - lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', - units='degrees_north') - lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', - units='degrees_east') - time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', - units='days since 1950-01-01') - depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') + def _create_ocean_files(self, filename, tar_name, gzip=False, backup=False): + folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', + 'outputs') + file_path, filename = self._create_file(folder_path, filename, gzip) - var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') - var1.add_dim_coord(time, 0) - var1.add_dim_coord(lat, 1) - var1.add_dim_coord(lon, 2) + if backup: + filename = os.path.join('backup', filename) - var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') - time.bounds = np.array([[0.5, 1.5], [1.5, 2.5]], np.float) - var2.add_dim_coord(time, 0) - var2.add_dim_coord(lat, 1) - var2.add_dim_coord(lon, 2) - var2.add_dim_coord(depth, 3) + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') + tar.add(file_path, arcname=filename, recursive=False) + tar.close() + os.remove(file_path) + def _create_mma_files(self, filename, tar_name, gzip=False): folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', 'outputs') - os.makedirs(folder_path) + filepath_gg, filename_gg = self._create_file(folder_path, filename.replace('??', 'GG'), gzip) + filepath_sh, filename_sh = self._create_file(folder_path, filename.replace('??', 'SH'), gzip) + + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') + tar.add(filepath_gg, arcname=filename_gg, recursive=False) + tar.add(filepath_sh, arcname=filename_sh, recursive=False) + tar.close() + os.remove(filepath_gg) + os.remove(filepath_sh) + + def _create_file(self, folder_path, filename, gzip): + var1 = self._create_sample_cube('Variable 1', 'var1', threed=False, time_bounds=True) + var2 = self._create_sample_cube('Variable 2', 'var2', threed=True, time_bounds=True) + if not os.path.isdir(folder_path): + os.makedirs(folder_path) file_path = os.path.join(folder_path, filename) iris.save((var1, var2), file_path, zlib=True) - if gzip: import subprocess process = subprocess.Popen(('gzip', file_path), stdout=subprocess.PIPE) @@ -122,31 +133,86 @@ class TestCmorizer(TestCase): filename = "{0}.gz".format(filename) if process.returncode != 0: raise Exception('Can not compress: {0}'.format(comunicate)) + return file_path, filename - if backup: - filename = os.path.join('backup', filename) + def _create_sample_cube(self, long_name, var_name, threed, time_bounds): + coord_data = np.array([1, 2], np.float) + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', + units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', + units='degrees_east') + time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', + units='days since 1950-01-01') + if time_bounds: + time.bounds = np.array([[0.5, 1.5], [1.5, 2.5]], np.float) - tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') - tar.add(file_path, arcname=filename, recursive=False) - tar.close() - os.remove(file_path) + if threed: + data = np.random.rand(2, 2, 2, 2).astype(np.float) + depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') + else: + data = np.random.rand(2, 2, 2).astype(np.float) + + cube = iris.cube.Cube(data, long_name=long_name, var_name=var_name) + cube.add_dim_coord(time, 0) + cube.add_dim_coord(lat, 1) + cube.add_dim_coord(lon, 2) + if threed: + cube.add_dim_coord(depth, 3) + return cube def tearDown(self): + """Clean up after tests""" shutil.rmtree(self.tmp_dir) - def test_skip_ocean_cmorization(self): - self.data_manager.config.cmor.ocean = False + def _test_ocean_cmor(self, success=True, error=False, critical=False, warnings=False, message='', check_vars=None): + self._test_cmorization(success=success, error=error, critical=critical, warnings=warnings, message=message, + ocean=True, check_vars=check_vars) + + def _test_atmos_cmor(self, success=True, error=False, critical=False, warnings=False, message='', check_vars=None): + self._test_cmorization(success=success, error=error, critical=critical, warnings=warnings, message=message, + ocean=False, check_vars=check_vars) + + def _test_cmorization(self, success=True, error=False, critical=False, warnings=False, message='', ocean=True, + check_vars=None): + self._check_logs(critical, error, message, ocean, success, warnings) + if check_vars: + for variable, status in six.iteritems(check_vars): + if status: + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, variable, '{}.nc'.format(variable)))) + else: + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, variable, '{}.nc'.format(variable)))) + + def _check_logs(self, critical, error, message, ocean, success, warnings): if six.PY3: with self.assertLogs(log.Log.log) as cmd: cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertLogs([record for record in cmd.records if - record.message == 'Skipping ocean cmorization due to configuration']) + if ocean: + cmorizer.cmorize_ocean() + else: + cmorizer.cmorize_atmos() + if message: + self.assertTrue([record for record in cmd.records if record.message == message]) + else: + for level, value in six.iteritems({log.Log.RESULT: success, log.Log.ERROR: error, + log.Log.CRITICAL: critical, log.Log.WARNING: warnings}): + if value: + self.assertTrue([record for record in cmd.records if record.levelno == level]) + else: + self.assertFalse([record for record in cmd.records if record.levelno == level]) else: cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() + if ocean: + cmorizer.cmorize_ocean() + else: + cmorizer.cmorize_atmos() + + def test_skip_ocean_cmorization(self): + """Test ocean cmorization flag disabled option""" + self.data_manager.config.cmor.ocean = False + self._test_ocean_cmor(message='Skipping ocean cmorization due to configuration') def test_skip_atmos_cmorization(self): + """Test atmos cmorization flag disabled option""" self.data_manager.config.cmor.atmosphere = False if six.PY3: with self.assertLogs(log.Log.log) as cmd: @@ -159,279 +225,113 @@ class TestCmorizer(TestCase): cmorizer.cmorize_ocean() def test_skip_when_cmorized(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test cmorization skipped if already done""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') self.data_manager.is_cmorized.return_value = True - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue([record for record in cmd.records if - record.message == 'No need to unpack file 1/1']) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() + self._test_ocean_cmor(message='No need to unpack file 1/1') def test_skip_when_not_requested(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test cmorization skipped if chunk is not requested""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') self.data_manager.config.cmor.chunk_cmorization_requested.return_value = False - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue([record for record in cmd.records if - record.message == 'No need to unpack file 1/1']) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() + self._test_ocean_cmor(message='No need to unpack file 1/1') def test_force(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test cmorization force works""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') self.data_manager.is_cmorized.return_value = True self.data_manager.config.cmor.force = True self._test_ocean_cmor() def test_ocean_cmorization_no_files(self): - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() + """Test ocean cmorization report error if no input data""" + self._test_ocean_cmor(success=False, error=True) def test_ocean_cmorization_not_vars_requested(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test ocean cmorization report success if no vars qhere requested""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') self.data_manager.config.cmor.any_required.return_value = False - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + self._test_ocean_cmor(check_vars={'var1': False, 'var2': False}) def test_ocean_cmorization_no_vars_recognized(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test ocean cmorization report success if no vars where recognized""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') def not_recognized(*args): return None, None self.data_manager.config.var_manager.get_variable_and_alias = not_recognized - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + self._test_ocean_cmor(check_vars={'var1': False, 'var2': False}) def test_ocean_cmorization_var2_not_requested(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test ocean cmorization with var2 not recognized""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') def _reject_var2(cmor_var): return cmor_var.short_name != 'var2' self.data_manager.config.cmor.cmorize = _reject_var2 - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) - - def _test_ocean_cmor(self): - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() + self._test_ocean_cmor(check_vars={'var1': True, 'var2': False}) def test_ocean_cmorization(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') - self._test_ocean_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + """Test basic ocean cmorization""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self._test_ocean_cmor(check_vars={'var1': True, 'var2': True}) def test_ocean_cmorization_with_filter(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test ocean cmorization filtering files""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') self.data_manager.config.cmor.filter_files = 'expid' - self._test_ocean_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + self._test_ocean_cmor(check_vars={'var1': True, 'var2': True}) def test_ocean_cmorization_with_bad_filter(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + """Test ocean cmorization fails if a bad filter is added""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') self.data_manager.config.cmor.filter_files = 'badfilter' - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.WARNING]) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() - self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + self._test_ocean_cmor(warnings=True, check_vars={'var1': False, 'var2': False}) def test_ocean_cmorization_gzip(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', gzip=True) - self._test_ocean_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + """Test ocean cmorization if tars are also zipped""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', gzip=True) + self._test_ocean_cmor(check_vars={'var1': True, 'var2': True}) def test_ocean_cmorization_backup(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', backup=True) - self._test_ocean_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + """Test ocean cmorization when files are in backup path""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', backup=True) + self._test_ocean_cmor(check_vars={'var1': True, 'var2': True}) def test_ocean_cmorization_PPO(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'PPO_expid_1D_xx_19900101_19900131.tar') - self._test_ocean_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + """Test ocean cmorization when files are PPO""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'PPO_expid_1D_xx_19900101_19900131.tar') + self._test_ocean_cmor(check_vars={'var1': True, 'var2': True}) def test_ocean_cmorization_diags(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'diags_expid_1D_xx_19900101_19900131.tar') - self._test_ocean_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + """Test ocean cmorization when files are diags""" + self._create_ocean_files('expid_1d_19900101_19900131.nc', 'diags_expid_1D_xx_19900101_19900131.tar') + self._test_ocean_cmor(check_vars={'var1': True, 'var2': True}) def test_atmos_cmorization(self): - self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') - self._test_atmos_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) - - def _test_atmos_cmor(self): - - # if six.PY3: - # - # with self.assertLogs(log.Log.log) as cmd: - # cmorizer = Cmorizer(self.data_manager, '19900101', 0) - # cmorizer.cmorize_atmos() - # - # try: - # self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - # self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - # self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - # self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - # except AssertionError: - # print(cmd) - # raise - # else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_atmos() - - def create_mma_files(self, filename, tar_name, gzip=False): - coord_data = np.array([0, 1], np.float) - folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', - 'outputs') - filepath_gg, filename_gg = self._create_file(coord_data, folder_path, filename.replace('??', 'GG'), gzip) - filepath_sh, filename_sh = self._create_file(coord_data, folder_path, filename.replace('??', 'SH'), gzip) - - tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') - tar.add(filepath_gg, arcname=filename_gg, recursive=False) - tar.add(filepath_sh, arcname=filename_sh, recursive=False) - tar.close() - os.remove(filepath_gg) - os.remove(filepath_sh) - - def _create_file(self, coord_data, folder_path, filename, gzip): - lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', - units='degrees_north') - lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', - units='degrees_east') - time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', - units='days since 1950-01-01') - depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') - var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') - var1.add_dim_coord(time, 0) - var1.add_dim_coord(lat, 1) - var1.add_dim_coord(lon, 2) - var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') - var2.add_dim_coord(time, 0) - var2.add_dim_coord(lat, 1) - var2.add_dim_coord(lon, 2) - var2.add_dim_coord(depth, 3) - if not os.path.isdir(folder_path): - os.makedirs(folder_path) - file_path = os.path.join(folder_path, filename) - iris.save((var1, var2), file_path, zlib=True) - if gzip: - import subprocess - process = subprocess.Popen(('gzip', file_path), stdout=subprocess.PIPE) - comunicate = process.communicate() - file_path = "{0}.gz".format(file_path) - filename = "{0}.gz".format(filename) - if process.returncode != 0: - raise Exception('Can not compress: {0}'.format(comunicate)) - return file_path, filename + """Test basic atmos cmorization from nc""" + self._create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + self._test_atmos_cmor(check_vars={'var1': True, 'var2': True}) def test_skip_when_not_requested_mma(self): - self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + """Test atmos cmorization is skipped if chunk is not requested""" + self._create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') self.data_manager.config.cmor.chunk_cmorization_requested.return_value = False - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_atmos() - self.assertTrue([record for record in cmd.records if - record.message == 'No need to unpack file 1/1']) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() + self._test_atmos_cmor(message='No need to unpack file 1/1') def test_force_mma(self): - self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + """Test force atmos cmorization""" + self._create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') self.data_manager.is_cmorized.return_value = True self.data_manager.config.cmor.force = True self._test_atmos_cmor() def test_atmos_cmorization_no_mma_files(self): - if six.PY3: - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_atmos() - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - else: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_atmos() + """Test atmos cmorization report error if there are no files""" + self._test_atmos_cmor(success=False, error=True) - def create_grib_files(self, filename, month): + def _create_grib_files(self, filename, month): filename = filename.format(month) coord_data = np.array([0, 1], np.float) folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', @@ -451,7 +351,7 @@ class TestCmorizer(TestCase): time_data = np.arange(0.25, month_days + 0.25, 0.25, np.float) + month * 31 time = DimCoord(time_data, standard_name='time', long_name='time', var_name='time', units='days since 1990-01-01 00:00:00') - vars = [] + variables = [] for code in codes: var = iris.cube.Cube(np.ones((month_days * 4, 2, 2), np.float) * code, long_name='Variable {}'.format(code), @@ -463,12 +363,12 @@ class TestCmorizer(TestCase): var.add_dim_coord(lon, 2) var.attributes['table'] = np.int32(128) var.attributes['code'] = np.int32(code) - vars.append(var) + variables.append(var) if not os.path.isdir(folder_path): os.makedirs(folder_path) file_path = os.path.join(folder_path, filename) - iris.save(vars, file_path, zlib=True, local_keys=('table', 'code')) + iris.save(variables, file_path, zlib=True, local_keys=('table', 'code')) Utils.cdo.settaxis('1990-0{}-01,06:00,6hour'.format(month + 1), input=file_path, output=file_path.replace('.nc', '.grb'), @@ -476,10 +376,12 @@ class TestCmorizer(TestCase): os.remove(file_path) def test_grib_cmorization(self): + """Test atmos cmorization from grib""" + self.data_manager.config.data_convention.get_file_path = self._get_file_path_grib self.data_manager.config.experiment.chunk_size = 2 self.data_manager.get_file_path = self._get_file_path_grib - self.create_grib_files('ICM??expid+19900{}.nc', 1) - self.create_grib_files('ICM??expid+19900{}.nc', 2) + self._create_grib_files('ICM??expid+19900{}.nc', 1) + self._create_grib_files('ICM??expid+19900{}.nc', 2) self._test_atmos_cmor() variables = { 'CP': 143, diff --git a/test/unit/data_convention/test_primavera.py b/test/unit/data_convention/test_primavera.py index 2eb75af909e805fbb78ad127c20b3dcda064fd43..42fe37cb5627a60db0ee5e8ca3a873ceb4c71f04 100644 --- a/test/unit/data_convention/test_primavera.py +++ b/test/unit/data_convention/test_primavera.py @@ -1,3 +1,4 @@ +"""Tests for PRIMAVERA Convention""" import os import shutil import tempfile @@ -12,8 +13,10 @@ from earthdiagnostics.data_convention import PrimaveraConvention class TestPrimaveraConvention(TestCase): + """Tests for PRIMAVERA convetion class""" def setUp(self): + """Prepare tests""" self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, 'expid')) self.config = Mock() @@ -37,22 +40,27 @@ class TestPrimaveraConvention(TestCase): self.convention = PrimaveraConvention('name', self.config) def tearDown(self): + """Cleanup""" shutil.rmtree(self.tmp_dir) def test_get_startdate_path(self): + """Test get startdate path""" self.assertEqual(self.convention.get_startdate_path('19900101'), os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name')) def test_experiment_name(self): + """Test get expriment name""" self.assertEqual(self.convention.experiment_name('19900101'), 'experiment_name') def test_experiment_name_append(self): + """Test get expriment name when appending startdate""" self.config.cmor.append_startdate = True self.assertEqual(self.convention.experiment_name('19900101'), 'experiment_nameS19900101') def test_get_cmor_folder_path(self): + """Test get cmor foilder path""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -64,6 +72,7 @@ class TestPrimaveraConvention(TestCase): 'r2i1p1f1/Omon/var/ocean_grid/version')) def test_get_cmor_folder_path_no_cmor_var(self): + """Test get cmor folder path when not passing cmor_var""" file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', Frequencies.monthly, None, None) self.assertEqual(file_path, @@ -71,6 +80,7 @@ class TestPrimaveraConvention(TestCase): 'r2i1p1f1/Omon/var/ocean_grid/version')) def test_get_cmor_folder_path_atmos(self): + """Test get cmor foilder path for the atmos""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -82,6 +92,7 @@ class TestPrimaveraConvention(TestCase): 'r2i1p1f1/Omon/var/atmos_grid/version')) def test_get_cmor_folder_path_custom_grid(self): + """Test get cmor foilder path for a custom grid""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -93,6 +104,7 @@ class TestPrimaveraConvention(TestCase): 'r2i1p1f1/Omon/var/grid/version')) def test_get_cmor_folder_path_no_cmor(self): + """Test get cmor folder path with no cmor_var""" frequency = Mock() frequency.__str__ = Mock() frequency.__str__.return_value = 'mon' @@ -104,6 +116,7 @@ class TestPrimaveraConvention(TestCase): 'r2i1p1f1/Omon/var/ocean_grid/version')) def test_get_file_path_no_version_primavera(self): + """Test get cmor folder path with no version""" self.config.cmor.version = '' cmor_var = Mock() omon = Mock() @@ -113,6 +126,7 @@ class TestPrimaveraConvention(TestCase): self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', 'grid', cmor_var) def test_get_filename(self): + """Test get_filename""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -123,6 +137,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') def test_get_filename_no_cmor_var(self): + """Test get_filename not passing cmor_var""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -133,6 +148,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') def test_get_filename_daily(self): + """Test get_filename for daily frequency""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -144,6 +160,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_19900101-19900131.nc') def test_get_filename_6hourly(self): + """Test get_filename for 6hourly files""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -155,6 +172,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001010000-199001311800.nc') def test_get_filename_3hourly(self): + """Test get_filename for 3hourly files""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -166,6 +184,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001010000-199001312100.nc') def test_get_filename_atmos(self): + """Test get_filename for atmos""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -176,6 +195,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_atmos_grid_199001-199001.nc') def test_get_filename_grid(self): + """Test get_filename for a custom grid""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -186,6 +206,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_grid_199001-199001.nc') def test_get_filename_year(self): + """Test get_filename for a whole year""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -196,6 +217,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_1990.nc') def test_get_filename_date_Str(self): + """Test get_filename passing date_Str""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -206,6 +228,7 @@ class TestPrimaveraConvention(TestCase): 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_date_str.nc') def test_get_filename_no_date_info(self): + """Test get_filename with no date info raises ValueError""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -217,6 +240,7 @@ class TestPrimaveraConvention(TestCase): @mock.patch('os.path.isfile') def test_is_cmorized(self, mock_is_file): + """Test is cmorized""" mock_is_file.return_value = True cmor_var = Mock() omon = Mock() @@ -230,6 +254,7 @@ class TestPrimaveraConvention(TestCase): @mock.patch('os.path.isfile') def test_is_not_cmorized(self, mock_is_file): + """Test is cmorized false""" mock_is_file.return_value = False cmor_var = Mock() omon = Mock() @@ -242,6 +267,7 @@ class TestPrimaveraConvention(TestCase): self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) def test_is_cmorized_false_not_member_folder(self): + """Test is cmorized false bacause ther is no member folder""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -253,6 +279,7 @@ class TestPrimaveraConvention(TestCase): self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) def test_is_cmorized_false_not_table_folder(self): + """Test is cmorized false bacause ther is no table folder""" cmor_var = Mock() omon = Mock() omon.name = 'Omon' @@ -265,6 +292,7 @@ class TestPrimaveraConvention(TestCase): @mock.patch('os.path.isfile') def test_is_cmorized_not_enough_vars(self, mock_is_file): + """Test is cmorized false because thera are not eniouch variables""" mock_is_file.return_value = True cmor_var = Mock() omon = Mock() @@ -276,18 +304,9 @@ class TestPrimaveraConvention(TestCase): 'mon/ocean/var')) self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - def test_is_cmorized_not_domain_folder(self): - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 2 - os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon')) - self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') def test_create_links_primavera(self, mock_create_link): + """Test create links""" member_path = os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/var/gn') os.makedirs(member_path) @@ -298,6 +317,7 @@ class TestPrimaveraConvention(TestCase): @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') def test_create_links_with_version_primavera(self, mock_create_link): + """Test create links with version""" member_path = os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/' 'var/gn/version') @@ -309,6 +329,7 @@ class TestPrimaveraConvention(TestCase): @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') def test_create_links_with_version_primavera_no_member(self, mock_create_link): + """Test create links with version full startdate""" member_path = os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/' 'var/gn/version') @@ -320,6 +341,7 @@ class TestPrimaveraConvention(TestCase): @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') def test_create_links_member_not_found_primavera(self, mock_create_link): + """Test create links when the member can not be found""" member_path = os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/r1i1p1f1/Omon/var/gn') os.makedirs(member_path) diff --git a/test/unit/general/test_attribute.py b/test/unit/general/test_attribute.py index 3e2cce9b1265305094206e8db41fd72bb1330c65..8473f8c5a8c3a592380e47134e581e72762e5fe8 100644 --- a/test/unit/general/test_attribute.py +++ b/test/unit/general/test_attribute.py @@ -1,5 +1,9 @@ # coding=utf-8 from unittest import TestCase +import os +from tempfile import mktemp + +import dummydata from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.box import Box @@ -21,6 +25,11 @@ class TestAttribute(TestCase): self.box = Box() self.box.min_depth = 0 self.box.max_depth = 100 + self.var_file = mktemp('.nc') + + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -53,3 +62,15 @@ class TestAttribute(TestCase): self.assertEqual(str(mixed), 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Attributte: att:value Grid: grid') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, var='ta', start_year=2000, stop_year=2000, method='constant', + constant=1) + + diag = Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'ta', 'grid', 'att', 'value') + + diag.variable_file = Mock() + diag.variable_file.local_file = self.var_file + diag.corrected = Mock() + diag.compute() + diag.corrected.set_local_file.assert_called_once() diff --git a/test/unit/general/test_module.py b/test/unit/general/test_module.py index 39549556dbb7de169ac2344cef559516dfb40963..fe8fc4a08ee5c0c52244d4a6e9b32f7a9490328a 100644 --- a/test/unit/general/test_module.py +++ b/test/unit/general/test_module.py @@ -1,5 +1,9 @@ # coding=utf-8 from unittest import TestCase +import os +from tempfile import mktemp + +import dummydata from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.box import Box @@ -21,6 +25,14 @@ class TestModule(TestCase): self.box = Box() self.box.min_depth = 0 self.box.max_depth = 100 + self.varu_file = mktemp('.nc') + self.varv_file = mktemp('.nc') + + def tearDown(self): + if os.path.exists(self.varu_file): + os.remove(self.varu_file) + if os.path.exists(self.varv_file): + os.remove(self.varv_file) def fake_parse(self, value): return value @@ -53,3 +65,21 @@ class TestModule(TestCase): self.assertEqual(str(mixed), 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' 'Variables: atmos:varu,varv,varmodule Grid: grid') + + def test_compute(self): + dummydata.model2.Model2(oname=self.varu_file, var='ua', start_year=2000, stop_year=2000, method='constant', + constant=1) + + dummydata.model2.Model2(oname=self.varv_file, var='va', start_year=2000, stop_year=2000, method='constant', + constant=1) + + diag = Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'ua', 'va', 'varmodule', 'grid') + + diag.component_u_file = Mock() + diag.component_u_file.local_file = self.varu_file + + diag.component_v_file = Mock() + diag.component_v_file.local_file = self.varv_file + diag.module_file = Mock() + diag.compute() + diag.module_file.set_local_file.assert_called_once() diff --git a/test/unit/general/test_rewrite.py b/test/unit/general/test_rewrite.py index ed0cb80f9eff75167726c689f5ffc67bfc7b612e..143eb1742d95cdc29f6c60975f5ed6b6b513cbf4 100644 --- a/test/unit/general/test_rewrite.py +++ b/test/unit/general/test_rewrite.py @@ -1,5 +1,9 @@ # coding=utf-8 from unittest import TestCase +import os +from tempfile import mktemp + +import dummydata from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.box import Box @@ -22,7 +26,12 @@ class TestRewrite(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.mixed = Rewrite(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', 'grid') + self.diag = Rewrite(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'ta', 'grid') + self.var_file = mktemp('.nc') + + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value @@ -47,5 +56,15 @@ class TestRewrite(TestCase): Rewrite.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), - 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var Grid: grid') + self.assertEqual(str(self.diag), + 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:ta Grid: grid') + + def test_compute(self): + dummydata.model2.Model2(oname=self.var_file, var='ta', start_year=2000, stop_year=2000, method='constant', + constant=1) + + self.diag.variable_file = Mock() + self.diag.variable_file.local_file = self.var_file + self.diag.corrected = Mock() + self.diag.compute() + self.diag.corrected.set_local_file.assert_called_once() diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py index 92321f3959c1e10dc37ca515adbb954ef7eca9fd..0e64ab2e5ea80c2999806e3c65d1da46fc975ddd 100644 --- a/test/unit/general/test_scale.py +++ b/test/unit/general/test_scale.py @@ -2,6 +2,11 @@ from unittest import TestCase from mock import Mock, patch +import os +from tempfile import mktemp + +import dummydata +import iris from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError @@ -24,6 +29,12 @@ class TestScale(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 + self.var_file = mktemp('.nc') + + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) + def fake_parse(self, value): return value @@ -73,8 +84,45 @@ class TestScale(TestCase): 'extra']) def test_str(self): - mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, + scale = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, Frequencies.three_hourly, False) - self.assertEqual(str(mixed), + self.assertEqual(str(scale), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' 'Variable: atmos:var Frequency: 3hr Apply mask: False') + + def test_compute_factor(self): + + scale = Scale(self.data_manager, '20010101', 0, 0, 10, 0, ModelingRealms.atmos, 'ta', 'grid', 1, 100, + Frequencies.three_hourly, False) + cube = self._get_data_and_test(scale) + self.assertEqual(cube.data.max(), 10) + + def test_compute_offset(self): + scale = Scale(self.data_manager, '20010101', 0, 0, 1, 10, ModelingRealms.atmos, 'ta', 'grid', 1, 100, + Frequencies.three_hourly, False) + cube = self._get_data_and_test(scale) + self.assertEqual(cube.data.max(), 11) + + def test_compute_too_low(self): + scale = Scale(self.data_manager, '20010101', 0, 0, 0, 10, ModelingRealms.atmos, 'ta', 'grid', 10, 100, + Frequencies.three_hourly, False) + cube = self._get_data_and_test(scale) + self.assertEqual(cube.data.max(), 1) + + def test_compute_too_high(self): + scale = Scale(self.data_manager, '20010101', 0, 0, 0, 10, ModelingRealms.atmos, 'ta', 'grid', 0, 0.5, + Frequencies.three_hourly, False) + cube = self._get_data_and_test(scale) + self.assertEqual(cube.data.max(), 1) + + def _get_data_and_test(self, scale): + dummydata.model2.Model2(oname=self.var_file, var='ta', start_year=2000, stop_year=2000, method='constant', + constant=1) + scale.variable_file = Mock() + scale.variable_file.local_file = self.var_file + scale.corrected = Mock() + scale.compute() + scale.corrected.set_local_file.assert_called_once() + cube = iris.load_cube(scale.corrected.set_local_file.call_args[0][0]) + self.assertEqual(cube.data.max(), cube.data.min()) + return cube diff --git a/test/unit/general/test_select_levels.py b/test/unit/general/test_select_levels.py index 146f0744345326290028499e94f5c62f6535e6fa..a9dac99ae4aaa7d70ad4fa69cfabe34d7640ffb7 100644 --- a/test/unit/general/test_select_levels.py +++ b/test/unit/general/test_select_levels.py @@ -1,5 +1,11 @@ # coding=utf-8 from unittest import TestCase +import os +from tempfile import mktemp + +import numpy as np +import dummydata +import iris from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError from earthdiagnostics.box import Box @@ -23,6 +29,11 @@ class TestSelectLevels(TestCase): self.box = Box() self.box.min_depth = 0 self.box.max_depth = 100 + self.var_file = mktemp('.nc') + + def tearDown(self): + if os.path.exists(self.var_file): + os.remove(self.var_file) def fake_parse(self, value): return value.split('-') @@ -61,7 +72,21 @@ class TestSelectLevels(TestCase): SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '20', 'grid', 'extra']) def test_str(self): - mixed = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 20) - self.assertEqual(str(mixed), + select = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 20) + self.assertEqual(str(select), 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' 'Levels: 0-20 Grid: grid') + + def test_compute(self): + dummydata.model3.Model3(oname=self.var_file, var='ta', start_year=2000, stop_year=2000, method='constant', + constant=1) + select = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'ta', 'grid', 0, 3) + select.variable_file = Mock() + select.variable_file.local_file = self.var_file + + select.result = Mock() + select.compute() + select.result.set_local_file.assert_called_once() + cube = iris.load_cube(select.result.set_local_file.call_args[0][0]) + original_cube = iris.load_cube(select.result.set_local_file.call_args[0][0]) + self.assertTrue(np.all(cube.coord('air_pressure').points == original_cube.coord('air_pressure').points[0:4])) diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py index 99dd57664405cdaedfe43c6aa113cea975281729..136539fc82f0de09bdc00edae70c6d45363262e5 100644 --- a/test/unit/general/test_verticalmeanmetersiris.py +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -16,7 +16,6 @@ from earthdiagnostics.modelingrealm import ModelingRealms class TestVerticalMeanMetersIris(TestCase): def setUp(self): - self.var_file = mktemp('.nc') self.data_manager = Mock() self.diags = Mock() @@ -27,6 +26,7 @@ class TestVerticalMeanMetersIris(TestCase): self.box = Box() self.box.min_depth = 0 self.box.max_depth = 100 + self.var_file = mktemp('.nc') def tearDown(self): if os.path.exists(self.var_file): @@ -97,3 +97,4 @@ class TestVerticalMeanMetersIris(TestCase): diag.variable_file.local_file = self.var_file diag.results = Mock() diag.compute() + diag.results.set_local_file.assert_called_once() diff --git a/test/unit/test_box.py b/test/unit/test_box.py index 8f69f2fa7068fdb6a4891319219515c3b4495da4..3134ef6fc45cd4a5982b0a918e5a84f3260fd15c 100644 --- a/test/unit/test_box.py +++ b/test/unit/test_box.py @@ -1,11 +1,14 @@ +"""Tests for box module""" # coding=utf-8 from unittest import TestCase from earthdiagnostics.box import Box class TestBox(TestCase): + """Test Box class""" def setUp(self): + """Prepare tests""" self.box1 = Box() self.box1.max_lat = 0 self.box1.min_lat = -20 @@ -33,6 +36,11 @@ class TestBox(TestCase): self.box4.max_depth = 20 def test_max_lat(self): + """ + Test max latitude setter + + Include tests for exceptions raising if value is out of range + """ with self.assertRaises(ValueError): Box().max_lat = 100 with self.assertRaises(ValueError): @@ -42,6 +50,11 @@ class TestBox(TestCase): Box().max_lat = 20 def test_min_lat(self): + """ + Test min latitude setter + + Include tests for exceptions raising if value is out of range + """ with self.assertRaises(ValueError): Box().min_lat = 100 with self.assertRaises(ValueError): @@ -51,6 +64,11 @@ class TestBox(TestCase): Box().min_lat = 90 def test_max_lon(self): + """ + Test max longitude setter + + Include tests for exceptions raising if value is out of range + """ with self.assertRaises(ValueError): Box().max_lon = 360 with self.assertRaises(ValueError): diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index e4c4463c498ed3aad3e4fba48ca2a2f8cbb103a7..c003a4015c387688f76ec133c5eb3e0bc54f1182 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -20,6 +20,8 @@ class TestCMORManager(TestCase): self.convention = create_autospec(DataConvention) self.convention.name = 'specs' + self.convention.lat_name = 'lat' + self.convention.lon_name = 'lon' self.convention.get_startdate_path.return_value = os.path.join(self.tmp_dir, 'expid', 'startdate') self.convention.get_cmor_folder_path.return_value = os.path.join(self.tmp_dir, 'expid', 'startdate', 'cmor') self.convention.get_file_name.return_value = 'filename' diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 4a0e51a7948459866a98651870e0595f2f390140..b18d7ebba7719153518ae32fe0475de1828dedf0 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -1,3 +1,4 @@ +"""Tests for config module""" # coding=utf-8 from unittest import TestCase import datetime @@ -12,6 +13,8 @@ from earthdiagnostics.data_convention import SPECSConvention, PrimaveraConventio class VariableMock(object): + """Mock for Variable""" + def __init__(self): self.domain = ModelingRealms.ocean self.short_name = 'tos' @@ -21,7 +24,10 @@ class VariableMock(object): class VariableManagerMock(object): + """Mock for Variable manager""" + def get_variable(self, alias, silent=False): + """Return a VariableMock given an alias""" if alias == 'bad': return None var = VariableMock() @@ -30,66 +36,82 @@ class VariableManagerMock(object): class ParserMock(mock.Mock): + """ConfigParser Mock""" def __init__(self, **kwargs): super(mock.Mock, self).__init__(**kwargs) self._values = {} def add_value(self, section, var, value): + """Add value to the parser""" self._values[self.get_var_string(section, var)] = value def get_var_string(self, section, var): + """Get var string""" return '{0}:{1}'.format(section, var) def get_value(self, section, var, default): + """Get value from mock parser""" try: return self._values[self.get_var_string(section, var)] except KeyError: return default def get_bool_option(self, section, var, default): + """Get bool option""" return self.get_value(section, var, default) def get_path_option(self, section, var, default=""): + """Get path option""" return self.get_value(section, var, default) def get_int_option(self, section, var, default=0): + """Get integer option""" return self.get_value(section, var, default) def get_choice_option(self, section, var, choices, default, ignore_case=True): + """Get choice option""" return self.get_value(section, var, default) def get_int_list_option(self, section, var, default=list(), separator=' '): + """Get integer list option""" try: return [int(val) for val in self._values[self.get_var_string(section, var)].split(separator)] except KeyError: return default def get_list_option(self, section, var, default=list(), separator=' '): + """Get list option""" try: return [val for val in self._values[self.get_var_string(section, var)].split(separator)] except KeyError: return default def get_option(self, section, var, default=None): + """get option""" return self.get_value(section, var, default) def has_section(self, section): + """Check if section exists""" start = '{0}:'.format(section) return any(x.startswith(start) for x in self._values) def options(self, section): + """Return all options for a given section""" start = '{0}:'.format(section) return [x[len(start):] for x in self._values if x.startswith(start)] class TestCMORConfig(TestCase): + """Tests for CMORConfig class""" def setUp(self): + """Prepare tests""" self.mock_parser = ParserMock() self.var_manager = VariableManagerMock() def test_basic_config(self): + """Test minimum configuration""" config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.ocean, True) self.assertEqual(config.atmosphere, True) @@ -113,11 +135,13 @@ class TestCMORConfig(TestCase): self.assertEqual(config.append_startdate, False) def test_cmorize(self): + """Test cmorize returns true always if list is not configured""" config = CMORConfig(self.mock_parser, self.var_manager) self.assertTrue(config.cmorize(VariableMock())) self.assertTrue(config.cmorize(None)) def test_cmorize_list(self): + """Test cmorize return true for variables in the given list""" self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:thetao ocean:tos') config = CMORConfig(self.mock_parser, self.var_manager) @@ -129,6 +153,7 @@ class TestCMORConfig(TestCase): self.assertTrue(config.cmorize(thetao_mock)) def test_bad_list(self): + """Test cmorize return false for malformed variables""" self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', '#ocean:tos') with self.assertRaises(ConfigException): CMORConfig(self.mock_parser, self.var_manager) @@ -142,6 +167,7 @@ class TestCMORConfig(TestCase): CMORConfig(self.mock_parser, self.var_manager) def test_not_cmorize(self): + """Test cmorize return false for variables not in the given list""" self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos') config = CMORConfig(self.mock_parser, self.var_manager) @@ -160,6 +186,7 @@ class TestCMORConfig(TestCase): self.assertFalse(config.cmorize(thetao_mock)) def test_comment(self): + """Test cmorize return false for variables after the comment mark""" self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos #ocean:thetao ') config = CMORConfig(self.mock_parser, self.var_manager) @@ -175,10 +202,12 @@ class TestCMORConfig(TestCase): CMORConfig(self.mock_parser, self.var_manager) def test_cmorization_chunk(self): + """Test chunk cmorization requested return always true if not given a list""" config = CMORConfig(self.mock_parser, self.var_manager) self.assertTrue(config.chunk_cmorization_requested(1)) def test_cmorize_only_some_chunks(self): + """Test chunk cmorization requested return true only if the value is in the list""" self.mock_parser.add_value('CMOR', 'CHUNKS', '3 5') config = CMORConfig(self.mock_parser, self.var_manager) self.assertTrue(config.chunk_cmorization_requested(3)) @@ -188,6 +217,7 @@ class TestCMORConfig(TestCase): self.assertFalse(config.chunk_cmorization_requested(6)) def test_any_required(self): + """Test any_required method""" config = CMORConfig(self.mock_parser, self.var_manager) self.assertTrue(config.any_required(['tos'])) @@ -200,6 +230,7 @@ class TestCMORConfig(TestCase): self.assertFalse(config.any_required(['tas'])) def test_hourly_vars(self): + """Test hourly vars specification""" config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.six_hourly), {}) @@ -218,6 +249,7 @@ class TestCMORConfig(TestCase): self.assertEqual(config.get_levels(Frequencies.six_hourly, 132), '0,5') def test_daily_vars(self): + """Test daily vars specification""" config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.daily), {}) @@ -236,6 +268,7 @@ class TestCMORConfig(TestCase): self.assertEqual(config.get_levels(Frequencies.daily, 132), '0,5') def test_monthly_vars(self): + """Test monthly vars specification""" config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.monthly), {}) @@ -254,11 +287,13 @@ class TestCMORConfig(TestCase): self.assertEqual(config.get_levels(Frequencies.monthly, 132), '0,5') def test_bad_frequency_vars(self): + """Test get variables fails if a bada frequency is specified""" config = CMORConfig(self.mock_parser, self.var_manager) with self.assertRaises(ValueError): config.get_variables(Frequencies.climatology) def test_requested_codes(self): + """Test requested codes returns a set of the codes requested at all frequencies""" self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') @@ -268,47 +303,59 @@ class TestCMORConfig(TestCase): class TestTHREDDSConfig(TestCase): + """Test THREDDS config""" def setUp(self): + """Prepare tests""" self.mock_parser = ParserMock() def test_basic_config(self): + """Test basic configuration""" config = THREDDSConfig(self.mock_parser) self.assertEqual(config.server_url, '') def test_url(self): + """Test SERVER_URL parameter""" self.mock_parser.add_value('THREDDS', 'SERVER_URL', 'test_url') config = THREDDSConfig(self.mock_parser) self.assertEqual(config.server_url, 'test_url') class TestReportConfig(TestCase): + """Test report config""" def setUp(self): + """Prepare tests""" self.mock_parser = ParserMock() def test_basic_config(self): + """test default config""" config = ReportConfig(self.mock_parser) self.assertEqual(config.path, '') self.assertEqual(config.maximum_priority, 10) def test_path(self): + """Test path configuration""" self.mock_parser.add_value('REPORT', 'PATH', 'new_path') config = ReportConfig(self.mock_parser) self.assertEqual(config.path, 'new_path') def test_priority(self): + """Test maximum priority configuration""" self.mock_parser.add_value('REPORT', 'MAXIMUM_PRIORITY', 3) config = ReportConfig(self.mock_parser) self.assertEqual(config.maximum_priority, 3) class TestExperimentConfig(TestCase): + """Test experiment config""" def setUp(self): + """Set up tests""" self.mock_parser = ParserMock() def test_basic_config(self): + """Test default values""" config = ExperimentConfig() config.parse_ini(self.mock_parser) @@ -322,6 +369,14 @@ class TestExperimentConfig(TestCase): self.assertEqual(config.ocean_timestep, 6) def test_members(self): + """ + Test all ways of specifing the members + + Including: + - Adding the prefix string or not + - Providing them as a space separated list + - Providing them as a range with start and end separated by - (both extremes included) + """ self.mock_parser.add_value('EXPERIMENT', 'MEMBERS', 'fc0 1') config = ExperimentConfig() config.parse_ini(self.mock_parser) @@ -343,6 +398,14 @@ class TestExperimentConfig(TestCase): self.assertEqual(config.members, [1, 2, 3]) def test_startdates(self): + """ + Test startadates parsing + + Two options: + - A simple list + - Using a regex + + """ self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '20001101 20011101') config = ExperimentConfig() config.parse_ini(self.mock_parser) @@ -361,6 +424,19 @@ class TestExperimentConfig(TestCase): u'20020801', u'20021101']) def test_auto_startdates(self): + """ + Test parsing startdates using the automatic generation + + + Reference syntax: {START,STOP,INTERVAL} where start and stop are dates and the interval is given with the code + NI where N is the number of intervals (default 1) and I is the base interval (Y,M,W,D) + + """ + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{2000,2001,1Y}') + config = ExperimentConfig() + config.parse_ini(self.mock_parser) + self.assertEqual(config.startdates, ['20000101', '20010101']) + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') config = ExperimentConfig() config.parse_ini(self.mock_parser) @@ -392,11 +468,13 @@ class TestExperimentConfig(TestCase): config.parse_ini(self.mock_parser) def test_get_member_str(self): + """Test get member str""" config = ExperimentConfig() config.parse_ini(self.mock_parser) self.assertEqual(config.get_member_str(1), 'fc1') def test_get_full_years(self): + """Test get full years method""" self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 3) self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 15) @@ -406,6 +484,7 @@ class TestExperimentConfig(TestCase): self.assertEqual(config.get_full_years('20000101'), [2000, 2001, 2002, 2003]) def test_get_year_chunks(self): + """Test get chunk years""" self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 3) self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 13) @@ -419,6 +498,7 @@ class TestExperimentConfig(TestCase): self.assertEqual(config.get_year_chunks('20000601', 1999), []) def test_get_chunk_list(self): + """Test get complete chunk list""" config = ExperimentConfig() config.startdates = ('20010101', ) config.members = (0, 1, 2) @@ -431,12 +511,14 @@ class TestExperimentConfig(TestCase): ('20010101', 1, 2), ('20010101', 2, 1), ('20010101', 2, 2)]) def test_get_member_list(self): + """Test get member list""" config = ExperimentConfig() config.startdates = ('20010101', ) config.members = (0, 1, 2) self.assertEqual(config.get_member_list(), [('20010101', 0), ('20010101', 1), ('20010101', 2)]) def test_get_chunk_start_str(self): + """Test get_chunk_start_str""" config = ExperimentConfig() self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 12) self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 3) @@ -444,6 +526,7 @@ class TestExperimentConfig(TestCase): self.assertEqual(config.get_chunk_start_str('20001101', 3), '20021101') def test_get_chunk_start_str_datetime(self): + """Test get_chunk_start_str when receiving a date object""" config = ExperimentConfig() self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 12) self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 3) @@ -452,6 +535,7 @@ class TestExperimentConfig(TestCase): self.assertEqual(config.get_chunk_start_str(date, 3), '20021101') def test_get_chunk_end_str(self): + """Test get_chunk_end_str""" config = ExperimentConfig() self.mock_parser.add_value('EXPERIMENT', 'CHUNK_SIZE', 12) self.mock_parser.add_value('EXPERIMENT', 'CHUNKS', 3) @@ -460,8 +544,10 @@ class TestExperimentConfig(TestCase): class TestConfig(TestCase): + """Tests for Config class""" def setUp(self): + """Prepare tests""" self.mock_parser = ParserMock() self.mock_parser.add_value('DIAGNOSTICS', 'FREQUENCY', 'mon') self.mock_parser.add_value('DIAGNOSTICS', 'DIAGS', 'diag1 diag2') @@ -469,6 +555,7 @@ class TestConfig(TestCase): self._environ = dict(os.environ) def tearDown(self): + """Cleanup""" os.environ.clear() os.environ.update(self._environ) @@ -489,17 +576,20 @@ class TestConfig(TestCase): config.parse('path') def test_diags(self): + """Test diags parsing""" config = Config() self.mock_parser.add_value('DIAGNOSTICS', 'DIAGS', 'diag1 diag2,opt1,opt2 # Commented diag') self._parse(config) self.assertEqual(config.get_commands(), (['diag1', 'diag2,opt1,opt2'])) def test_file_not_found(self): + """Test value error is raised if config file is not found""" config = Config() with self.assertRaises(ValueError): config.parse('path') def test_parse(self): + """Test parse minimal config""" config = Config() self._parse(config) self.assertEqual(config.frequency, Frequencies.monthly) @@ -515,12 +605,14 @@ class TestConfig(TestCase): self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) def test_alias(self): + """Test alias parsing""" config = Config() self.mock_parser.add_value('ALIAS', 'diag1', 'diag3') self._parse(config) self.assertEqual(config.get_commands(), ['diag3', 'diag2']) def test_auto_clean_ram_disk(self): + """Test that USE_RAMDISK forces AUTO_CLEAN to true""" config = Config() self.mock_parser.add_value('DIAGNOSTICS', 'AUTO_CLEAN', False) self.mock_parser.add_value('DIAGNOSTICS', 'USE_RAMDISK', True) @@ -529,6 +621,7 @@ class TestConfig(TestCase): self.assertEqual(config.use_ramdisk, True) def test_data_convention_primavera(self): + """Test parsing data convention PRIMAVERA""" config = Config() self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'primavera') self._parse(config) @@ -539,6 +632,7 @@ class TestConfig(TestCase): self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) def test_data_convention_cmip6(self): + """Test parsing data convention CMIP6""" config = Config() self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'cmip6') self._parse(config) @@ -549,6 +643,7 @@ class TestConfig(TestCase): self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) def test_data_convention_meteofrance(self): + """Test parsing data convention MeteoFrance""" config = Config() self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'meteofrance') self._parse(config) @@ -559,6 +654,7 @@ class TestConfig(TestCase): self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) def test_data_convention_preface(self): + """Test parsing data convention Preface""" config = Config() self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'preface') self._parse(config) diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index f425083b93010990a3dd1a510f2acf609988ebfc..f4baa2267ba2c2d236ce6083fb6e0ef8dbf13d32 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -1,5 +1,13 @@ +"""Test for diagnostic module""" # coding=utf-8 -from earthdiagnostics.diagnostic import * +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticOptionError, DiagnosticFloatOption, \ + DiagnosticDomainOption, DiagnosticIntOption, DiagnosticBoolOption, DiagnosticComplexStrOption, \ + DiagnosticListIntOption, DiagnosticChoiceOption, DiagnosticVariableOption, DiagnosticVariableListOption, \ + DiagnosticBasinOption, DiagnosticStatus +from earthdiagnostics.datafile import StorageStatus + +from earthdiagnostics.constants import Basins + from unittest import TestCase from earthdiagnostics.modelingrealm import ModelingRealms @@ -7,207 +15,265 @@ from mock import patch, Mock class TestDiagnosticOption(TestCase): + """Test diagnostic basic string option""" def test_good_default_value(self): + """Test default_value""" diag = DiagnosticOption('option', 'default') self.assertEqual('default', diag.parse('')) def test_no_default_value(self): + """Test raises if not value provided and no default is available""" diag = DiagnosticOption('option') with self.assertRaises(DiagnosticOptionError): self.assertEqual('default', diag.parse('')) def test_parse_value(self): + """Test parse simple value""" diag = DiagnosticOption('option') self.assertEqual('value', diag.parse('value')) class TestDiagnosticFloatOption(TestCase): + """Test diagnostic float option""" def test_float_default_value(self): + """Test float default_value""" diag = DiagnosticFloatOption('option', 3.0) self.assertEqual(3.0, diag.parse('')) def test_str_default_value(self): + """Test str default_value""" diag = DiagnosticFloatOption('option', '3') self.assertEqual(3.0, diag.parse('')) def test_bad_default_value(self): + """Test fails if default_value can not be converted to float""" diag = DiagnosticFloatOption('option', 'default') with self.assertRaises(ValueError): self.assertEqual('default', diag.parse('')) def test_no_default_value(self): + """Test raises if not value provided and no default is available""" diag = DiagnosticFloatOption('option') with self.assertRaises(DiagnosticOptionError): self.assertEqual('default', diag.parse('')) def test_parse_value(self): + """Test parse simple value""" diag = DiagnosticFloatOption('option') self.assertEqual(3.25, diag.parse('3.25')) class TestDiagnosticDomainOption(TestCase): + """Test diagnostic domain option""" def test_domain_default_value(self): + """Test domain default_value""" diag = DiagnosticDomainOption('option', ModelingRealms.ocean) self.assertEqual(ModelingRealms.ocean, diag.parse('')) def test_str_default_value(self): + """Test str default_value""" diag = DiagnosticDomainOption('option', 'atmos') self.assertEqual(ModelingRealms.atmos, diag.parse('')) def test_bad_default_value(self): + """Test raises if default_value can not be converted to ModellingRealm""" diag = DiagnosticDomainOption('option', 'default') with self.assertRaises(ValueError): diag.parse('') def test_no_default_value(self): + """Test no value and default_value is provided""" diag = DiagnosticDomainOption('option') with self.assertRaises(DiagnosticOptionError): diag.parse('') def test_parse_value(self): + """Test parse option""" diag = DiagnosticDomainOption('option') self.assertEqual(ModelingRealms.seaIce, diag.parse('seaice')) class TestDiagnosticIntOption(TestCase): + """Test diagnostic int option""" + def test_int_default_value(self): + """Test int default_value""" diag = DiagnosticIntOption('option', 3) self.assertEqual(3, diag.parse('')) def test_str_default_value(self): + """Test str default_value""" diag = DiagnosticIntOption('option', '3') self.assertEqual(3, diag.parse('')) def test_bad_default_value(self): + """Test raises if default_value can not be parsed to int""" diag = DiagnosticIntOption('option', 'default') with self.assertRaises(ValueError): diag.parse('') def test_no_default_value(self): + """Test raises if no value and default_value are provided""" diag = DiagnosticIntOption('option') with self.assertRaises(DiagnosticOptionError): diag.parse('') def test_parse_value(self): + """Test parse int value""" diag = DiagnosticIntOption('option') self.assertEqual(3, diag.parse('3')) def test_parse_bad_value(self): + """Test raises if value can not be parsed to int""" diag = DiagnosticIntOption('option') with self.assertRaises(ValueError): diag.parse('3.5') def test_good_low_limit(self): + """Test parse if low limit is configured""" diag = DiagnosticIntOption('option', None, 0) self.assertEqual(1, diag.parse('1')) def test_bad_low_limit(self): + """"Test raises if low limit is configured and value does not conform to it""" diag = DiagnosticIntOption('option', None, 0) with self.assertRaises(DiagnosticOptionError): diag.parse('-1') def test_good_high_limit(self): + """Test parse if high limit is configured""" diag = DiagnosticIntOption('option', None, None, 0) self.assertEqual(-1, diag.parse('-1')) def test_bad_high_limit(self): + """Test raises if high limit is configured and value does not conform to it""" diag = DiagnosticIntOption('option', None, None, 0) with self.assertRaises(DiagnosticOptionError): diag.parse('1') class TestDiagnosticBoolOption(TestCase): + """Test diagnostic bool option""" + def test_bool_default_value(self): + """Test parse bool default_value""" diag = DiagnosticBoolOption('option', True) self.assertEqual(True, diag.parse('')) def test_str_default_value(self): + """Test parse str default_value""" diag = DiagnosticBoolOption('option', 'False') self.assertEqual(False, diag.parse('')) def test_no_default_value(self): + """Test raises if neither value or default_value are provided""" diag = DiagnosticBoolOption('option') with self.assertRaises(DiagnosticOptionError): diag.parse('') def test_parse_True(self): + """Test parses 'True' as boolean True""" diag = DiagnosticBoolOption('option') - self.assertTrue(diag.parse('true')) + self.assertTrue(diag.parse('True')) def test_parse_true(self): + """Test parses 'true' as boolean True""" diag = DiagnosticBoolOption('option') self.assertTrue(diag.parse('true')) def test_parse_t(self): + """Test parses 't' as boolean True""" diag = DiagnosticBoolOption('option') self.assertTrue(diag.parse('t')) def test_parse_yes(self): + """Test parses 'YES' as boolean True""" diag = DiagnosticBoolOption('option') self.assertTrue(diag.parse('YES')) def test_parse_bad_value(self): + """Test parses random str as boolean False""" diag = DiagnosticBoolOption('option') self.assertFalse(diag.parse('3.5')) class TestDiagnosticComplexStrOption(TestCase): + """ + Test diagnostic complex string option + + It replaces '&.' por ' ' and '&;' por ',' + """ def test_complex_default_value(self): + """Test default value""" diag = DiagnosticComplexStrOption('option', 'default&.str&;&.working') self.assertEqual('default str, working', diag.parse('')) def test_simple_default_value(self): + """Test simple default value""" diag = DiagnosticComplexStrOption('default str, working', 'default str, working') self.assertEqual('default str, working', diag.parse('')) def test_no_default_value(self): + """Test raises if neither value or default_value are provided""" diag = DiagnosticComplexStrOption('option') with self.assertRaises(DiagnosticOptionError): diag.parse('') def test_parse_value(self): + """Test parse value""" diag = DiagnosticComplexStrOption('option') self.assertEqual('complex string, for testing', diag.parse('complex&.string&;&.for&.testing')) class TestDiagnosticListIntOption(TestCase): + """Test DiagnosticListIntOption Class""" + def test_tuple_default_value(self): + """Test use tuple default_value if no option is provided""" diag = DiagnosticListIntOption('option', (3,)) self.assertEqual((3,), diag.parse('')) def test_list_default_value(self): + """Test use list default_value if no option is provided""" diag = DiagnosticListIntOption('option', [3]) self.assertEqual([3], diag.parse('')) def test_str_default_value(self): + """Test use str default_value if no option is provided""" diag = DiagnosticListIntOption('option', '3-4') self.assertEqual([3, 4], diag.parse('')) def test_bad_default_value(self): + """Test raises if default value can not be converted to int""" diag = DiagnosticListIntOption('option', 'default') with self.assertRaises(ValueError): diag.parse('') def test_no_default_value(self): + """Test raises if neither value or default_value are provided""" diag = DiagnosticListIntOption('option') with self.assertRaises(DiagnosticOptionError): diag.parse('') def test_parse_value(self): + """Test parse a list of integers""" diag = DiagnosticListIntOption('option') self.assertEqual([3, 2], diag.parse('3-2')) def test_parse_single_value(self): + """Test parse a single integer""" diag = DiagnosticListIntOption('option') self.assertEqual([3], diag.parse('3')) def test_too_low(self): + """Test raises if low limit is configured and value does not conform to it""" diag = DiagnosticListIntOption('option', min_limit=5) with self.assertRaises(DiagnosticOptionError): diag.parse('3') def test_too_high(self): + """Test raises if high limit is configured and value does not conform to it""" diag = DiagnosticListIntOption('option', max_limit=5) with self.assertRaises(DiagnosticOptionError): diag.parse('8') @@ -219,20 +285,25 @@ class TestDiagnosticListIntOption(TestCase): class TestDiagnosticChoiceOption(TestCase): + """Test DiagnosticChoiceOption class""" def test_choice_value(self): + """Test parsing a choice""" diag = DiagnosticChoiceOption('option', ('a', 'b')) self.assertEqual('a', diag.parse('a')) def test_choice_default_value(self): + """Test use default value""" diag = DiagnosticChoiceOption('option', ('a', 'b'), default_value='a') self.assertEqual('a', diag.parse('')) def test_bad_default_value(self): + """Test error is raised if default option is not a valid choice""" with self.assertRaises(DiagnosticOptionError): DiagnosticChoiceOption('option', ('a', 'b'), default_value='c') def test_ignore_case_value(self): + """Test ignore_case option""" diag = DiagnosticChoiceOption('option', ('a', 'b')) self.assertEqual('b', diag.parse('b')) self.assertEqual('b', diag.parse('B')) @@ -244,20 +315,23 @@ class TestDiagnosticChoiceOption(TestCase): class TestDiagnosticVariableOption(TestCase): + """Test DiagnosticVariableOption class""" - def get_var_mock(self, name): + def _get_var_mock(self, name): mock = Mock() mock.short_name = name return mock def test_parse(self): + """Test parse""" var_manager_mock = Mock() - var_manager_mock.get_variable.return_value = self.get_var_mock('var1') + var_manager_mock.get_variable.return_value = self._get_var_mock('var1') diag = DiagnosticVariableOption(var_manager_mock) self.assertEqual('var1', diag.parse('var1')) def test_not_recognized(self): + """Test parsing a not recognized variable""" var_manager_mock = Mock() var_manager_mock.get_variable.return_value = None @@ -266,34 +340,40 @@ class TestDiagnosticVariableOption(TestCase): class TestDiagnosticVariableListOption(TestCase): + """Test DiagnosticVariableListOption class""" def test_parse_multiple(self): + """Test parsing multiple vars""" var_manager_mock = Mock() - var_manager_mock.get_variable.side_effect = (self.get_var_mock('var1'), self.get_var_mock('var2')) + var_manager_mock.get_variable.side_effect = (self._get_var_mock('var1'), self._get_var_mock('var2')) diag = DiagnosticVariableListOption(var_manager_mock, 'variables') self.assertEqual(['var1', 'var2'], diag.parse('var1:var2')) def test_parse_one(self): + """Test parsing only one var""" var_manager_mock = Mock() - var_manager_mock.get_variable.return_value = self.get_var_mock('var1') + var_manager_mock.get_variable.return_value = self._get_var_mock('var1') diag = DiagnosticVariableListOption(var_manager_mock, 'variables') self.assertEqual(['var1'], diag.parse('var1')) def test_not_recognized(self): + """Test parsing one var that can not be recognized""" var_manager_mock = Mock() var_manager_mock.get_variable.return_value = None diag = DiagnosticVariableListOption(var_manager_mock, 'variables') self.assertEqual(['var1'], diag.parse('var1')) - def get_var_mock(self, name): + def _get_var_mock(self, name): mock = Mock() mock.short_name = name return mock class TestDiagnostic(TestCase): + """Test base Diagnostic class""" def setUp(self): + """Prepare tests""" class MockDiag(Diagnostic): @classmethod def generate_jobs(cls, diags, options): @@ -311,33 +391,45 @@ class TestDiagnostic(TestCase): self.MockDiag = MockDiag def test_str(self): + """Test base __str__ implementation""" self.assertEqual(str(Diagnostic(None)), 'Developer must override base class __str__ method') + def test_repr(self): + """Test base __repr__ implementation""" + self.assertEqual(Diagnostic(None).__repr__(), str(Diagnostic(None))) + def test_compute_is_virtual(self): + """Test compute raises NotImplementedError""" with self.assertRaises(NotImplementedError): Diagnostic(None).compute() def test_declare_data_generated_is_virtual(self): + """Test declare_data_generated raises NotImplementedError""" with self.assertRaises(NotImplementedError): Diagnostic(None).declare_data_generated() def test_request_data_is_virtual(self): + """Test request_data raises NotImplementedError""" with self.assertRaises(NotImplementedError): Diagnostic(None).request_data() @patch.object(Diagnostic, 'dispatch') def test_set_status_call_dispatch(self, dispatch_mock): + """Test dispatch is called when setting a different statuts""" diag = Diagnostic(None) + old_status = diag.status diag.status = DiagnosticStatus.FAILED - dispatch_mock.assert_called_once_with(diag) + dispatch_mock.assert_called_once_with(diag, old_status) @patch.object(Diagnostic, 'dispatch') - def test_set_status_call_dispatch(self, dispatch_mock): + def test_set_status_call_dispatch_not_called(self, dispatch_mock): + """Test dispatch is not called when setting again the same status""" diag = Diagnostic(None) diag.status = diag.status assert not dispatch_mock.called, 'Dispatch should not have been called' def test_register(self): + """Test register diagnostic""" with self.assertRaises(ValueError): Diagnostic.register(TestDiagnostic) @@ -348,26 +440,23 @@ class TestDiagnostic(TestCase): Diagnostic.register(self.MockDiag) def test_get_diagnostic(self): + """Test get diagnostic""" self.assertIsNone(Diagnostic.get_diagnostic('none')) self.MockDiag.alias = 'mock' Diagnostic.register(self.MockDiag) self.assertIs(self.MockDiag, Diagnostic.get_diagnostic('mock')) def test_generate_jobs(self): + """Test generate_jobs raises NotImplementedError""" with self.assertRaises(NotImplementedError): Diagnostic.generate_jobs(None, ['']) - def test_compute(self): - with self.assertRaises(NotImplementedError): - Diagnostic(None).compute() - - def test_repr(self): - self.assertEqual(Diagnostic(None).__repr__(), str(Diagnostic(None))) - def test_empty_process_options(self): + """Test process options works even without options""" self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) def test_diagnostic_can_be_skipped(self): + """Test diagnostic can be skipped""" diag = self._get_diag_for_skipping() self.assertTrue(diag.can_skip_run()) @@ -382,28 +471,34 @@ class TestDiagnostic(TestCase): return diag def test_diagnostic_can_not_be_skipped_do_not_generate_files(self): + """Test diagnostic can not be skipped because it does not generate files""" diag = Diagnostic(None) self.assertFalse(diag.can_skip_run()) def test_diagnostic_can_not_be_skipped_modified(self): + """Test diagnostic can not be skipped because it modifies files""" diag = self._get_diag_for_skipping(modified=True) self.assertFalse(diag.can_skip_run()) def test_diagnostic_can_not_be_skipped_data_not_ready(self): + """Test diagnostic can not be skipped because results do not yet exist""" diag = self._get_diag_for_skipping(status=StorageStatus.PENDING) self.assertFalse(diag.can_skip_run()) class TestDiagnosticBasinOption(TestCase): + """Test for DiagnosticBasinOPtion""" @patch.object(Basins, 'parse') - def test_not_recognized(self, parse_mock): + def test_parse(self, parse_mock): + """Test parse good basin""" parse_mock.return_value = 'var1' diag = DiagnosticBasinOption('basin') self.assertEqual('var1', diag.parse('var1')) @patch.object(Basins, 'parse') def test_not_recognized(self, parse_mock): + """Test parsing a bad basin""" parse_mock.return_value = None diag = DiagnosticBasinOption('basin') with self.assertRaises(DiagnosticOptionError): diff --git a/test/unit/test_lint.py b/test/unit/test_lint.py index 287de6a6b2be5374bc27613cb4e51d556e259e83..4c1d1d91b1bc211b83e97af804d88e1135ae3dd2 100644 --- a/test/unit/test_lint.py +++ b/test/unit/test_lint.py @@ -1,6 +1,5 @@ """ Lint tests """ import os -import textwrap import unittest import pycodestyle # formerly known as pep8 @@ -10,6 +9,7 @@ class TestLint(unittest.TestCase): def test_pep8_conformance(self): """Test that we conform to PEP-8.""" + check_paths = [ 'earthdiagnostics', 'test', diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index f6c86e2a70fdf20a3626f9e977737b3562303b4d..874ad65895ce0ea9863206743a0a1a52e42371d4 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -1,3 +1,4 @@ +"""Test variable module""" # coding=utf-8 from mock import Mock @@ -9,18 +10,22 @@ from earthdiagnostics.frequency import Frequencies class TestCMORTable(TestCase): + """Test CMORTable class""" def setUp(self): + """Set up tests""" self.frequency = Mock() def test_str(self): - self.assertEqual(str(CMORTable('name', 'm', 'Month YEAR')), 'name') + """Test string representation""" + self.assertEqual(str(CMORTable('name', 'm', 'Month YEAR', 'realm')), 'name') def test_repr(self): - self.assertEqual(repr(CMORTable('name', 'm', 'Month YEAR')), 'name (m, Month YEAR)') + """Test string Representation""" + self.assertEqual(repr(CMORTable('name', 'm', 'Month YEAR', 'realm')), 'name (realm m, Month YEAR)') def test_lt(self): - self.assertLess(CMORTable('a', 'm', 'Month YEAR'), CMORTable('b', 'm', 'Month YEAR')) + self.assertLess(CMORTable('a', 'm', 'Month YEAR', 'realm'), CMORTable('b', 'm', 'Month YEAR', 'realm')) class TestVariableAlias(TestCase): @@ -188,7 +193,7 @@ class TestVariable(TestCase): convention.name = 'specs' var = Variable() var.domain = ModelingRealms.atmos - var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) + var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013', ModelingRealms.atmos)) table = var.get_table(Frequencies.monthly, convention) self.assertEqual(table.frequency, Frequencies.monthly) self.assertEqual(table.name, 'Amon') @@ -199,7 +204,7 @@ class TestVariable(TestCase): convention.name = 'specs' var = Variable() var.domain = ModelingRealms.atmos - var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) + var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013', ModelingRealms.atmos)) table = var.get_table(Frequencies.daily, convention) self.assertEqual(table.frequency, Frequencies.daily) self.assertEqual(table.name, 'day') @@ -214,34 +219,43 @@ class TestVariable(TestCase): class TestVariableManager(TestCase): + """Tests for VariableManager""" def setUp(self): + """Prepare tests""" self.var_manager = VariableManager() def tearDown(self): + """Cleanup""" self.var_manager.clean() def test_load_primavera(self): + """Test loading SPECS tables""" self.var_manager.load_variables('primavera') self.assertTrue(self.var_manager.get_all_variables()) def test_load_cmip6(self): + """Test loading SPECS tables""" self.var_manager.load_variables('cmip6') self.assertTrue(self.var_manager.get_all_variables()) def test_load_specs(self): + """Test loading SPECS tables""" self.var_manager.load_variables('specs') self.assertTrue(self.var_manager.get_all_variables()) def test_load_preface(self): + """Test loading SPECS tables""" self.var_manager.load_variables('preface') self.assertTrue(self.var_manager.get_all_variables()) def test_bad_load(self): + """Test loading a bad table raises an exception""" with self.assertRaises(Exception): self.var_manager.load_variables('badconvention') def test_get_variable(self): + """Test get variable""" var1 = self._get_var_mock('var1', ['var1_alias']) self.var_manager.register_variable(var1) self.var_manager.create_aliases_dict() @@ -253,6 +267,7 @@ class TestVariableManager(TestCase): self.assertIsNone(self.var_manager.get_variable('var2', True)) def test_get_variable_and_alias(self): + """Test get variable and alias""" var1 = self._get_var_mock('var1', ['var1_alias']) self.var_manager.register_variable(var1) self.var_manager.create_aliases_dict() @@ -278,6 +293,7 @@ class TestVariableManager(TestCase): return var1 def test_get_all_variables(self): + """Test get all variables""" var1 = self._get_var_mock('var1', ['var1_alias']) var2 = self._get_var_mock('var2', ['var2_alias']) diff --git a/test/unit/test_variable_type.py b/test/unit/test_variable_type.py index cb3dfa0549a7f4aedd0774b9f158577094d2914f..52cf76df8a78154cf86f4ed6800634565846979a 100644 --- a/test/unit/test_variable_type.py +++ b/test/unit/test_variable_type.py @@ -1,3 +1,4 @@ +"""Tests for VariableType class""" # coding=utf-8 from unittest import TestCase @@ -5,13 +6,17 @@ from earthdiagnostics.variable import VariableType class TestVariableType(TestCase): + """Tests for VariableType class""" def test_mean(self): + """Test to_str of MEAN""" self.assertEqual(VariableType.to_str(VariableType.MEAN), 'mean') def test_statistics(self): + """Test to_str of STATISTIC""" self.assertEqual(VariableType.to_str(VariableType.STATISTIC), 'statistics') def test_bad_one(self): + """Test to_str raises if type not recognized""" with self.assertRaises(ValueError): VariableType.to_str('bad type') diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 119d3c6fa6908eadda707f7085a30431785b78eb..58c0463d0054b7e84db34702cd10ccb63c3762e7 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -1,3 +1,4 @@ +"""Test work_manager file""" # coding=utf-8 from unittest import TestCase from mock import Mock @@ -9,21 +10,26 @@ from bscearth.utils import log class TestDownloader(TestCase): + """Test downloader""" def setUp(self): + """Set up tests""" self.downloader = Downloader() def test_start_and_stop(self): + """Basic test with no data""" self.downloader.start() self.assertTrue(self.downloader._thread.is_alive()) self.downloader.shutdown() self.assertFalse(self.downloader._thread.is_alive()) def test_submit(self): + """Test submit method""" datafile = Mock() self.downloader.submit(datafile) def test_download(self): + """Test download call for method""" datafile = Mock() self.downloader.submit(datafile) self.downloader.start() @@ -31,6 +37,7 @@ class TestDownloader(TestCase): datafile.download.assert_called_once() def test_download_fails(self): + """Test bahaviour when download fail""" datafile = Mock() def _download(): @@ -46,6 +53,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() def test_download_smaller_first(self): + """Test smaller downloads going first""" small_file = self._create_datafile_mock(size=1) self.downloader.submit(small_file) large_file = self._create_datafile_mock(size=10) @@ -62,6 +70,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() def test_download_not_none_first(self): + """Test downloads with known size go first""" small_file = self._create_datafile_mock(size=None) self.downloader.submit(small_file) large_file = self._create_datafile_mock(size=10) @@ -78,6 +87,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() def test_download_not_none_second(self): + """Test downloads with unknown size go last""" small_file = self._create_datafile_mock(size=1) self.downloader.submit(small_file) large_file = self._create_datafile_mock(size=None) @@ -94,6 +104,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() def test_download_both_none(self): + """Test downloads work when all have unknown size""" small_file = self._create_datafile_mock(size=None) self.downloader.submit(small_file) large_file = self._create_datafile_mock(size=None) @@ -110,6 +121,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() def test_download_can_not_order(self): + """Test downloads work when order can not be asigned""" small_file = self._create_datafile_mock(size=1) self.downloader.submit(small_file) large_file = self._create_datafile_mock(size=1) @@ -126,6 +138,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() def test_download_more_suscribers_first(self): + """Test downloads requested by more diagnostics go first""" no_suscribers = self._create_datafile_mock() self.downloader.submit(no_suscribers) suscriber = Mock() @@ -143,7 +156,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() def test_download_more_suscribers_waiting_first(self): - + """Test downloads with more diagnostics waiting go first""" class StatusDiag(Diagnostic): def __init__(self, data_manager, pending): super(StatusDiag, self).__init__(data_manager) @@ -187,30 +200,37 @@ class TestDownloader(TestCase): class MockFile(DataFile): + """Mock DataFile class for testing""" def __init__(self): super(MockFile, self).__init__() self.upload_fails = False def download(self): + """Simulate download step""" self.local_status = LocalStatus.READY def upload(self): + """Upload method""" if self.upload_fails: self.storage_status = StorageStatus.FAILED else: self.storage_status = StorageStatus.READY def clean_local(self): + """Simulate clean local call""" pass def check_is_in_storage(self): + """Simulate check storage""" return False class TestWorkManager(TestCase): + """Tests for workmanager class""" def setUp(self): + """Set up tests""" self.config = Mock() self.data_manager = Mock() self.config.max_cores = 1 @@ -221,7 +241,7 @@ class TestWorkManager(TestCase): self.work_manager = WorkManager(self.config, self.data_manager) def test_prepare_job_list(self): - + """Test job list preparation""" class Diag1(Diagnostic): alias = 'diag1' @@ -237,6 +257,7 @@ class TestWorkManager(TestCase): self.work_manager.prepare_job_list() def test_run(self): + """Test run""" self.config.max_cores = -1 class EmptyDiag(Diagnostic): @@ -262,6 +283,7 @@ class TestWorkManager(TestCase): self.assertFalse(self.work_manager.had_errors) def test_diag_can_be_skipped(self): + """Test run skipping diagnostics already done""" self.config.skip_diags_done = True class SkippedDiag(Diagnostic): @@ -290,6 +312,11 @@ class TestWorkManager(TestCase): self.assertFalse(self.work_manager.had_errors) def test_diag_can_be_skipped_but_no(self): + """ + Test run skipping diagnostics already done + + Diagnostic can be skipped in this case + """ self.config.skip_diags_done = False class SkippedDiag(Diagnostic): @@ -318,6 +345,7 @@ class TestWorkManager(TestCase): self.assertTrue(self.work_manager.had_errors) def test_failed_run(self): + """Test run when a diagnostic fails""" class FailDiag(Diagnostic): alias = 'diag1' @@ -360,6 +388,7 @@ class TestWorkManager(TestCase): return req def test_run_data(self): + """Test run with data diagnostic""" self.data_manager.config.max_cores = -1 class DataDiag(Diagnostic): @@ -385,7 +414,7 @@ class TestWorkManager(TestCase): self.assertFalse(self.work_manager.had_errors) def test_failed_run_upload_failed(self): - + """Test run failing at upload step""" self.data_manager.config.max_cores = -1 class DataDiag(Diagnostic): @@ -412,6 +441,7 @@ class TestWorkManager(TestCase): self.assertTrue(self.work_manager.had_errors) def test_run_empty(self): + """Test run with empty list""" if six.PY3: with self.assertLogs(log.Log.log) as cmd: self.work_manager.run()