From 7cf62ed165516a4acf6bca73930948d8d1f6f673 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 14 Feb 2018 16:46:42 +0100 Subject: [PATCH 01/70] Fixed bug in cmorization console output --- earthdiagnostics/cmormanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 931f4ec7..b88e7dca 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -564,7 +564,7 @@ class CMORManager(DataManager): def _cmorize_member(self, startdate, member): start_time = datetime.now() member_str = self.experiment.get_member_str(member) - Log.info('CMORizing startdate {0} member {1}. Starting at {0}', startdate, member_str, start_time) + Log.info('CMORizing startdate {0} member {1}. Starting at {2}', startdate, member_str, start_time) cmorizer = Cmorizer(self, startdate, member) cmorizer.cmorize_ocean() cmorizer.cmorize_atmos() -- GitLab From f2e72d341d1bff4c798c8a5f99c36c908c4288d7 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 14 Feb 2018 16:50:00 +0100 Subject: [PATCH 02/70] Fix some codacy issues --- earthdiagnostics/cmorizer.py | 4 ++-- earthdiagnostics/threddsmanager.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 1e2d718b..efa2da3f 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -665,8 +665,8 @@ class Cmorizer(object): startdate = parse_date(self.startdate) leadtime = [datetime(time.year, time.month, time.day, time.hour, time.minute, time.second) - startdate for time in leadtime] - for lt in range(0, len(leadtime)): - var[lt] = leadtime[lt].days + for lt, lead in enumerate(leadtime): + var[lt] = lead.days def _add_common_attributes(self, handler, frequency): cmor = self.config.cmor diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index b6606524..47463461 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -66,6 +66,7 @@ class THREDDSManager(DataManager): ------- bool """ + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', -- GitLab From 03d11129f45ff48070f94f1dbd91a233068b6624 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Feb 2018 10:57:55 +0100 Subject: [PATCH 03/70] Fixed bugs in region mean --- earthdiagnostics/ocean/regionmean.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index fa41fcf1..d3b7e36d 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -156,7 +156,7 @@ class RegionMean(Diagnostic): return cell.point - 1 in j_indexes def selected_level(cell): - return lev_limits[0] <= cell.point <= lev_limits[1] + return lev_limits[0] <= cell.point - 1 <= lev_limits[1] data = data.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) if has_levels: @@ -201,7 +201,7 @@ class RegionMean(Diagnostic): def _load_data(self): def add_i_j(cube, field, filename): if cube.var_name != self.variable: - return + raise iris.exceptions.IgnoreCubeException() if not cube.coords('i'): index = field.dimensions.index('i') i = np.arange(1, field.shape[index] + 1) @@ -219,7 +219,6 @@ class RegionMean(Diagnostic): cube.add_aux_coord(lev, index) data = iris.load_cube(self.variable_file.local_file, - iris.AttributeConstraint(short_name=self.variable), callback=add_i_j) return data @@ -346,12 +345,12 @@ class ComputeWeights(Diagnostic): return cell.point - 1 in j_indexes def selected_level(cell): - return min_level <= cell.point <= max_level + return min_level <= cell.point - 1 <= max_level e1_small = e1.extract(iris.Constraint(i=selected_i, j=selected_j)) e2_small = e2.extract(iris.Constraint(i=selected_i, j=selected_j)) e3_small = e3.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) - mask_small = mask_small[min_level:max_level, ...] + mask_small = mask_small[min_level:max_level+1, ...] mask_small = e3_small * mask_small e_small = e1_small * e2_small -- GitLab From 5aad12653650259825d67799152d71bcb33186df Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Feb 2018 11:37:50 +0100 Subject: [PATCH 04/70] Added fix to solve an issue when depth is model_level_number --- earthdiagnostics/ocean/regionmean.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index d3b7e36d..3dcbf1fc 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -218,8 +218,23 @@ class RegionMean(Diagnostic): lev = iris.coords.AuxCoord(i, var_name='lev') cube.add_aux_coord(lev, index) + coords = [] + handler = Utils.open_cdf(self.variable_file.local_file) + for variable in handler.variables: + if variable in ('time', 'lev', 'lat', 'lon', 'latitude', 'longitude'): + coords.append(variable) + + handler.variables[self.variable].coordinates = ' '.join(coords) + handler.close() + data = iris.load_cube(self.variable_file.local_file, callback=add_i_j) + + if data.coords('model_level_number'): + coord = data.coord('model_level_number') + coord.standard_name = 'depth' + coord.long_name = 'depth' + return data def _fix_file_metadata(self): -- GitLab From 51b328eef7ad87b40068638e7c8c14fc450ad018 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Feb 2018 13:15:54 +0100 Subject: [PATCH 05/70] Fix format error --- earthdiagnostics/ocean/regionmean.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 3dcbf1fc..78bc05b9 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -365,7 +365,7 @@ class ComputeWeights(Diagnostic): e1_small = e1.extract(iris.Constraint(i=selected_i, j=selected_j)) e2_small = e2.extract(iris.Constraint(i=selected_i, j=selected_j)) e3_small = e3.extract(iris.Constraint(i=selected_i, j=selected_j, lev=selected_level)) - mask_small = mask_small[min_level:max_level+1, ...] + mask_small = mask_small[min_level:max_level + 1, ...] mask_small = e3_small * mask_small e_small = e1_small * e2_small -- GitLab From 9e124cbb52458530806ad0c77dbafbf129ea3978 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Feb 2018 13:21:56 +0100 Subject: [PATCH 06/70] Fixed codacy warnings --- earthdiagnostics/diagnostic.py | 8 ++++++++ earthdiagnostics/earthdiags.py | 2 +- earthdiagnostics/ocean/interpolatecdo.py | 20 ++++++++++++++++++-- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 4090dcd6..0af27101 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -608,6 +608,14 @@ class DiagnosticListFrequenciesOption(DiagnosticOption): super(DiagnosticListFrequenciesOption, self).__init__(name, default_value) def parse(self, option_value): + """ + Parse option value + + Returns + ------- + List of Frequency + + """ option_value = self._check_default(option_value) if isinstance(option_value, (tuple, list)): return option_value diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 23107feb..75b89b82 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -385,7 +385,7 @@ class EarthDiags(object): # Small size differences can be due to the renaming of variables reference_size = os.stat(source).st_size delta_size = abs(reference_size - os.stat(destiny).st_size) - if delta_size < 2048 or delta_size / reference_size < 1/1000: + if delta_size < 2048 or delta_size / reference_size < 1 / 1000: Log.info('File {0} already exists', destiny) return True diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 5a09eaf3..e1cdbb79 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -105,7 +105,7 @@ class InterpolateCDO(Diagnostic): weights = TempFile.get() method = options['method'].lower() if options['weights_from_mask']: - temp = cls.get_sample_grid_file() + temp = cls._get_sample_grid_file() cls.compute_weights(method, target_grid, temp, weights) os.remove(temp) weights_job = None @@ -127,6 +127,21 @@ class InterpolateCDO(Diagnostic): @classmethod def compute_weights(cls, method, target_grid, sample_file, weights): + """ + Compute weights for interpolation from sample file + + Parameters + ---------- + method: int + Interpòlation method + target_grid: str + Grid to intepolate to. Can be anything understand by CDO + sample_file: str + Path to a file containing original mesh information + weights: + Path to the file to store the weights + """ + if method == InterpolateCDO.BILINEAR: Utils.cdo.genbil(target_grid, input=sample_file, output=weights) elif method == InterpolateCDO.BICUBIC: @@ -137,7 +152,7 @@ class InterpolateCDO(Diagnostic): Utils.cdo.gencon2(target_grid, input=sample_file, output=weights) @classmethod - def get_sample_grid_file(cls): + def _get_sample_grid_file(cls): temp = TempFile.get() handler = Utils.open_cdf('mask.nc') @@ -316,6 +331,7 @@ class ComputeWeights(Diagnostic): return 'Computing weights for CDO interpolation: Method {0.method} Target grid: {0.grid}'.format(self) def compute(self): + """Compute weights""" InterpolateCDO.compute_weights(self.method, self.grid, self.sample_data.local_file, self.weights_file) def request_data(self): -- GitLab From 68172b366315a4756754de6d6e2cdb7552b893c8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Feb 2018 13:26:58 +0100 Subject: [PATCH 07/70] Fixed codacy warnings --- earthdiagnostics/threddsmanager.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 47463461..f0f45be5 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -5,12 +5,13 @@ from datetime import datetime from time import strptime import iris +from iris.coords import DimCoord + import netCDF4 import numpy as np from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date from bscearth.utils.log import Log from cf_units import Unit -from iris.coords import DimCoord from datafile import DataFile, StorageStatus, LocalStatus from earthdiagnostics.datamanager import DataManager @@ -25,6 +26,7 @@ class THREDDSManager(DataManager): Parameters ---------- config: Config + """ def __init__(self, config): @@ -47,9 +49,11 @@ class THREDDSManager(DataManager): # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN, possible_versions=None): - """" + """ Check if a file exists in the storage + Creates a THREDDSSubset and checks if it is accesible + Parameters ---------- domain: ModelingRealm @@ -64,9 +68,9 @@ class THREDDSManager(DataManager): Returns ------- - bool - """ + THREDDSSubset + """ aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', @@ -281,6 +285,7 @@ class THREDDSSubset(DataFile): var: str start_time: datetime end_time: datetime + """ def __init__(self, thredds_path, file_path, var, start_time, end_time): -- GitLab From 36090a590e94fa89c7d02887f40578fb690c9acf Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 15 Feb 2018 13:27:47 +0100 Subject: [PATCH 08/70] Fixed codacy warnings --- earthdiagnostics/ocean/interpolatecdo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index e1cdbb79..f1ade167 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -133,15 +133,15 @@ class InterpolateCDO(Diagnostic): Parameters ---------- method: int - Interpòlation method + Interpolation method target_grid: str Grid to intepolate to. Can be anything understand by CDO sample_file: str Path to a file containing original mesh information weights: Path to the file to store the weights - """ + """ if method == InterpolateCDO.BILINEAR: Utils.cdo.genbil(target_grid, input=sample_file, output=weights) elif method == InterpolateCDO.BICUBIC: -- GitLab From e064797926fb6eeabe42dfe5f1096ade0be2e1ff Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:20:47 +0100 Subject: [PATCH 09/70] Changed way to run test --- .gitignore | 3 ++- .gitlab-ci.yml | 8 +++--- earthdiagnostics/earthdiags.py | 2 +- earthdiagnostics/ocean/interpolatecdo.py | 13 ++++++++-- earthdiagnostics/threddsmanager.py | 2 +- run_test.py | 29 ++++++++++++++++++++++ test/run_test.py | 31 ------------------------ 7 files changed, 48 insertions(+), 40 deletions(-) create mode 100644 run_test.py delete mode 100644 test/run_test.py diff --git a/.gitignore b/.gitignore index cafa800b..2ebdb8d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ *.pyc .idea/* doc/build/* +test/report/* *.err *.out *.nc .coverage -htmlcov \ No newline at end of file +htmlcov diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdc37552..a66d489f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,12 +8,12 @@ test_python2: script: - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - - coverage run -m unittest discover + - test/run_test.py test_python3: script: - conda env update -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics - - coverage run -m unittest discover - - coverage xml - - python-codacy-coverage -r coverage.xml + - test/run_test.py + - pip install codacy-coverage --upgrade + - python-codacy-coverage -r test/report/python3/coverage.xml diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 75b89b82..ab3a6e00 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -19,7 +19,7 @@ from earthdiagnostics.constants import Basins from earthdiagnostics.obsreconmanager import ObsReconManager from earthdiagnostics.threddsmanager import THREDDSManager from earthdiagnostics.utils import TempFile, Utils -from work_manager import WorkManager +from earthdiagnostics.work_manager import WorkManager class EarthDiags(object): diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index f1ade167..39314eee 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -105,7 +105,7 @@ class InterpolateCDO(Diagnostic): weights = TempFile.get() method = options['method'].lower() if options['weights_from_mask']: - temp = cls._get_sample_grid_file() + temp = cls.get_sample_grid_file() cls.compute_weights(method, target_grid, temp, weights) os.remove(temp) weights_job = None @@ -152,7 +152,16 @@ class InterpolateCDO(Diagnostic): Utils.cdo.gencon2(target_grid, input=sample_file, output=weights) @classmethod - def _get_sample_grid_file(cls): + def get_sample_grid_file(cls): + """ + Get a sample grid file + + Create a sample grid file from the definition in the masks file + + Returns + ------- + str + """ temp = TempFile.get() handler = Utils.open_cdf('mask.nc') diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index f0f45be5..be37da89 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -13,7 +13,7 @@ from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date from bscearth.utils.log import Log from cf_units import Unit -from datafile import DataFile, StorageStatus, LocalStatus +from earthdiagnostics.datafile import DataFile, StorageStatus, LocalStatus from earthdiagnostics.datamanager import DataManager from earthdiagnostics.utils import TempFile, Utils from earthdiagnostics.variable import VariableType diff --git a/run_test.py b/run_test.py new file mode 100644 index 00000000..d498085a --- /dev/null +++ b/run_test.py @@ -0,0 +1,29 @@ +# coding=utf-8 +""" +Script to run the tests for EarthDiagnostics and generate the code coverage report +""" + + +import os +import sys +work_path = os.path.abspath(os.path.join(os.path.dirname(__file__))) +os.chdir(work_path) +print(work_path) +import pytest + +version = sys.version_info[0] +report_dir = 'test/report/python{}'.format(version) +errno = pytest.main([ + 'test', + '--ignore=test/report', + '--cov=earthdiagnostics', + '--cov-report=term', + '--cov-report=html:{}/coverage_html'.format(report_dir), + '--cov-report=xml:{}/coverage.xml'.format(report_dir), +]) +sys.exit(errno) + + + + + diff --git a/test/run_test.py b/test/run_test.py deleted file mode 100644 index 12614344..00000000 --- a/test/run_test.py +++ /dev/null @@ -1,31 +0,0 @@ -# coding=utf-8 -""" -Script to run the tests for EarthDiagnostics and generate the code coverage report -""" - - -import os -import sys -work_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -os.chdir(work_path) -print(work_path) -import coverage -import unittest - -cov = coverage.Coverage() -cov.start() -if len(sys.argv) == 1: - suite = unittest.TestLoader().discover('.') -else: - suite = unittest.TestLoader().discover('.', pattern=sys.argv[1]) - - -unittest.TextTestRunner(verbosity=2).run(suite) -cov.stop() - -cov.save() -cov.report() -cov.html_report() - - - -- GitLab From 094c96b1316d2efb29ae182e1c5d5c9902fe735a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:24:09 +0100 Subject: [PATCH 10/70] Updated gitlab CI config --- .gitignore | 1 + .gitlab-ci.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2ebdb8d6..733c1474 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ test/report/* *.nc .coverage htmlcov +.pytest_cache diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a66d489f..da922f0f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,12 +8,12 @@ test_python2: script: - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - - test/run_test.py + - earthdiagnostics/run_test.py test_python3: script: - conda env update -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics - - test/run_test.py + - earthdiagnostics/run_test.py - pip install codacy-coverage --upgrade - python-codacy-coverage -r test/report/python3/coverage.xml -- GitLab From 3da46334dc5267aa289bb6e769c4886266468ec0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:25:55 +0100 Subject: [PATCH 11/70] Updated gitlab CI config --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da922f0f..7462ab6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,12 +8,12 @@ test_python2: script: - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - - earthdiagnostics/run_test.py + - ./run_test.py test_python3: script: - conda env update -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics - - earthdiagnostics/run_test.py + - ./run_test.py - pip install codacy-coverage --upgrade - python-codacy-coverage -r test/report/python3/coverage.xml -- GitLab From eaced8307623ae8785b00a7d3033be6dfe27867f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:30:03 +0100 Subject: [PATCH 12/70] Updated gitlab CI config --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7462ab6a..340e88a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,12 +8,12 @@ test_python2: script: - conda env update -f environment.yml -n earthdiagnostics2 python=2 - source activate earthdiagnostics - - ./run_test.py + - python run_test.py test_python3: script: - conda env update -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics - - ./run_test.py + - python run_test.py - pip install codacy-coverage --upgrade - python-codacy-coverage -r test/report/python3/coverage.xml -- GitLab From d16ed90ea6a0aeeb79d5bf983cf3d6a04e467313 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:32:37 +0100 Subject: [PATCH 13/70] Added pytest to requirements --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 14d62005..417ec6ba 100644 --- a/environment.yml +++ b/environment.yml @@ -20,6 +20,8 @@ dependencies: - mock - cmake - coverage +- pytest +- pytest-cov - pip: - bscearth.utils @@ -27,4 +29,3 @@ dependencies: - nco - exrex - xxhash - - codacy-coverage -- GitLab From c7d5f5459321d8c33e9874db9233e6e100d29835 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:34:40 +0100 Subject: [PATCH 14/70] Fixed codacy warnings --- run_test.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/run_test.py b/run_test.py index d498085a..a1689be6 100644 --- a/run_test.py +++ b/run_test.py @@ -1,15 +1,13 @@ # coding=utf-8 -""" -Script to run the tests for EarthDiagnostics and generate the code coverage report -""" - +"""Script to run the tests for EarthDiagnostics and generate the code coverage report""" import os import sys +import pytest work_path = os.path.abspath(os.path.join(os.path.dirname(__file__))) os.chdir(work_path) print(work_path) -import pytest + version = sys.version_info[0] report_dir = 'test/report/python{}'.format(version) @@ -22,8 +20,3 @@ errno = pytest.main([ '--cov-report=xml:{}/coverage.xml'.format(report_dir), ]) sys.exit(errno) - - - - - -- GitLab From 5a9a657b94e33b041816af1cfa2b9c092b9b6747 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:35:49 +0100 Subject: [PATCH 15/70] Updated CI config --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 340e88a9..0d6ce6a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,13 +7,13 @@ before_script: test_python2: script: - conda env update -f environment.yml -n earthdiagnostics2 python=2 - - source activate earthdiagnostics + - source activate earthdiagnostics2 - python run_test.py test_python3: script: - conda env update -f environment.yml -n earthdiagnostics3 python=3 - - source activate earthdiagnostics + - source activate earthdiagnostics3 - python run_test.py - pip install codacy-coverage --upgrade - python-codacy-coverage -r test/report/python3/coverage.xml -- GitLab From 100c402344247478dc2af513c50e03aa807fa280 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:39:44 +0100 Subject: [PATCH 16/70] Updated CI config --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d6ce6a2..92675184 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,4 +16,8 @@ test_python3: - source activate earthdiagnostics3 - python run_test.py - pip install codacy-coverage --upgrade + - ll + - ll test/ + - ll test/report + - ll test/report/python3 - python-codacy-coverage -r test/report/python3/coverage.xml -- GitLab From 256a1cc99201b0a81e3735fe520eecb215531acc Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:42:29 +0100 Subject: [PATCH 17/70] Updated CI config --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92675184..26d2b4a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,8 +16,8 @@ test_python3: - source activate earthdiagnostics3 - python run_test.py - pip install codacy-coverage --upgrade - - ll - - ll test/ - - ll test/report - - ll test/report/python3 + - ls -l + - ls -l test/ + - ls -l test/report + - ls -l test/report/python3 - python-codacy-coverage -r test/report/python3/coverage.xml -- GitLab From 208ddf64c57410e6297e402924ccd6a5015cf3ad Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:48:16 +0100 Subject: [PATCH 18/70] Updated CI config --- .gitlab-ci.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 26d2b4a0..101de622 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,12 +12,9 @@ test_python2: test_python3: script: - - conda env update -f environment.yml -n earthdiagnostics3 python=3 + - conda env remove -n earthdiagnostics3 + - conda env create -f environment.yml -n earthdiagnostics3 python=3 - source activate earthdiagnostics3 - python run_test.py - pip install codacy-coverage --upgrade - - ls -l - - ls -l test/ - - ls -l test/report - - ls -l test/report/python3 - python-codacy-coverage -r test/report/python3/coverage.xml -- GitLab From edcb81474be62b2d5b873ab282c7c8329ee2567e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 11:53:10 +0100 Subject: [PATCH 19/70] Updated CI config --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 101de622..b5715dea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,14 +6,14 @@ before_script: test_python2: script: - - conda env update -f environment.yml -n earthdiagnostics2 python=2 + - conda env update -f environment.yml -n earthdiagnostics2 python=2.7 - source activate earthdiagnostics2 - python run_test.py test_python3: script: - conda env remove -n earthdiagnostics3 - - conda env create -f environment.yml -n earthdiagnostics3 python=3 + - conda env create -f environment.yml -n earthdiagnostics3 python=3.6 - source activate earthdiagnostics3 - python run_test.py - pip install codacy-coverage --upgrade -- GitLab From 02971a72ab54a3570fadf335a196e61172d87d3c Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 12:02:27 +0100 Subject: [PATCH 20/70] Updated CI config --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5715dea..d1a9a2fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,9 @@ test_python2: test_python3: script: + - conda update conda - conda env remove -n earthdiagnostics3 + - conda clean --all --yes - conda env create -f environment.yml -n earthdiagnostics3 python=3.6 - source activate earthdiagnostics3 - python run_test.py -- GitLab From d1fabe54942b66b08312cd998a9864a38f63e965 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 13:06:05 +0100 Subject: [PATCH 21/70] Better CI config --- .gitlab-ci.yml | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1a9a2fa..615c8b18 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,19 +4,41 @@ before_script: - git submodule update --init --recursive - export PATH="$HOME/miniconda2/bin:$PATH" +stages: + - prepare + - test + - report + - clean + +prepare: + stage: prepare + script: + - conda update conda + test_python2: + stage: test script: - conda env update -f environment.yml -n earthdiagnostics2 python=2.7 - source activate earthdiagnostics2 - python run_test.py test_python3: + stage: test script: - - conda update conda - - conda env remove -n earthdiagnostics3 - - conda clean --all --yes - - conda env create -f environment.yml -n earthdiagnostics3 python=3.6 + - conda env update -f environment.yml -n earthdiagnostics3 python=3.6 - source activate earthdiagnostics3 - python run_test.py + +report_codacy: + stage: report + script: + - source activate earthdiagnostics3 - pip install codacy-coverage --upgrade - python-codacy-coverage -r test/report/python3/coverage.xml + +clean: + stage: clean + script: + - conda clean --all --yes + + -- GitLab From b38db1fa7e80cb3d4be412603d41dd9c95a5f613 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 13:27:39 +0100 Subject: [PATCH 22/70] Better CI config --- .gitlab-ci.yml | 9 ++++++-- diags.conf | 26 ++++++++++++------------ earthdiagnostics/ocean/interpolatecdo.py | 3 +++ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 615c8b18..0b7ac5dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,5 @@ before_script: - export GIT_SSL_NO_VERIFY=1 - - git submodule sync --recursive - - git submodule update --init --recursive - export PATH="$HOME/miniconda2/bin:$PATH" stages: @@ -10,6 +8,9 @@ stages: - report - clean +cache: + -test/report + prepare: stage: prepare script: @@ -18,6 +19,8 @@ prepare: test_python2: stage: test script: + - git submodule sync --recursive + - git submodule update --init --recursive - conda env update -f environment.yml -n earthdiagnostics2 python=2.7 - source activate earthdiagnostics2 - python run_test.py @@ -25,6 +28,8 @@ test_python2: test_python3: stage: test script: + - git submodule sync --recursive + - git submodule update --init --recursive - conda env update -f environment.yml -n earthdiagnostics3 python=3.6 - source activate earthdiagnostics3 - python run_test.py diff --git a/diags.conf b/diags.conf index 6ea8d8c7..34122c37 100644 --- a/diags.conf +++ b/diags.conf @@ -1,26 +1,26 @@ [DIAGNOSTICS] # Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) -DATA_ADAPTOR = CMOR +DATA_ADAPTOR = OBSRECON # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use -DATA_DIR = /esnas:/esarchive +DATA_DIR = /esarchive # Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) -DATA_TYPE = exp +DATA_TYPE = recon # CMORization type to use. Important also for THREDDS as it affects variable name conventions. # Options: SPECS (default), PRIMAVERA, CMIP6 -DATA_CONVENTION = PRIMAVERA +DATA_CONVENTION = SPECS # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = regmean,ocean,thetao +DIAGS = interpcdo,atmos,prlr,r240x121,bilinear,False,,False # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = mon +FREQUENCY = weekly # Path to CDFTOOLS binaries CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir @@ -73,11 +73,11 @@ SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = EC-Earth-Consortium -MODEL = EC-Earth3-HR -NAME = historical +INSTITUTE = gloh2o +MODEL = mswep +NAME = wekkly_means # Model version: Available versions -MODEL_VERSION =Ec3.2_O25L75 +MODEL_VERSION = # Atmospheric output timestep in hours ATMOS_TIMESTEP = 3 # Ocean output timestep in hours @@ -91,9 +91,9 @@ OCEAN_TIMESTEP = 3 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = a0n8 -STARTDATES = 19900101 -MEMBERS = fc0 +EXPID = mswep +STARTDATES = {19960101,20161201,1D} +MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 CHUNKS = 1 diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 39314eee..4549ca8c 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -65,6 +65,9 @@ class InterpolateCDO(Diagnostic): self.weights = weights def __eq__(self, other): + if type(self) != type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version and self.domain == other.domain and \ self.variable == other.variable and self.mask_oceans == other.mask_oceans and self.grid == other.grid and \ -- GitLab From ee4a696c66da01c0f4fc0c8a5b0eac8070b35d29 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 13:30:20 +0100 Subject: [PATCH 23/70] Better CI config --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b7ac5dd..009321c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,8 @@ stages: - clean cache: - -test/report + paths: + -test/report prepare: stage: prepare -- GitLab From b18db7a980df97aa2bc3cc54b279e16fdc198a75 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 13:31:10 +0100 Subject: [PATCH 24/70] Better CI config --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 009321c3..c5b843a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ stages: cache: paths: - -test/report + - test/report prepare: stage: prepare -- GitLab From 698dd74e7627477ffee6565358e02a9cbd188886 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 14:13:48 +0100 Subject: [PATCH 25/70] Fix _eq__ interpcdo --- earthdiagnostics/ocean/interpolatecdo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 4549ca8c..13e1de63 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -65,7 +65,7 @@ class InterpolateCDO(Diagnostic): self.weights = weights def __eq__(self, other): - if type(self) != type(other): + if type(self) is not type(other): return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ -- GitLab From a2bd1afe761a1542df2ecea46268617add295c59 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 14:41:52 +0100 Subject: [PATCH 26/70] Added tests for DiagnosticBasinOption --- test/unit/test_diagnostic.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index b1ff6c82..2ee05480 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -366,3 +366,19 @@ class TestDiagnostic(TestCase): def test_empty_process_options(self): self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) + + +class TestDiagnosticBasinOption(TestCase): + + @patch.object(Basins, 'parse') + def test_not_recognized(self, parse_mock): + parse_mock.return_value = 'var1' + diag = DiagnosticBasinOption('basin') + self.assertEqual('var1', diag.parse('var1')) + + @patch.object(Basins, 'parse') + def test_not_recognized(self, parse_mock): + parse_mock.return_value = None + diag = DiagnosticBasinOption('basin') + with self.assertRaises(DiagnosticOptionError): + diag.parse('bad') -- GitLab From a877085864861185eab33bb6f767aa5b5ddfb964 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 15:09:15 +0100 Subject: [PATCH 27/70] Download priorities now only take into account the first 100 files to save time --- diags.conf | 2 +- earthdiagnostics/work_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/diags.conf b/diags.conf index 34122c37..f5db5eba 100644 --- a/diags.conf +++ b/diags.conf @@ -92,7 +92,7 @@ OCEAN_TIMESTEP = 3 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = mswep -STARTDATES = {19960101,20161201,1D} +STARTDATES = {19970101,20161231,1D} MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 385a5525..3e15dad5 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -362,7 +362,7 @@ class Downloader(object): return time.sleep(0.01) continue - self._downloads.sort(key=cmp_to_key(Downloader._prioritize)) + self._downloads[:100].sort(key=cmp_to_key(Downloader._prioritize)) datafile = self._downloads[0] self._downloads.remove(datafile) datafile.download() -- GitLab From e3a858cb76e9f84b47406a6052f8dbbf94da62ed Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 15:59:52 +0100 Subject: [PATCH 28/70] Added base class for fix file diagnostics --- earthdiagnostics/general/attribute.py | 24 ++--- earthdiagnostics/general/fix_file.py | 88 +++++++++++++++++++ earthdiagnostics/general/rewrite.py | 53 +---------- earthdiagnostics/general/scale.py | 21 +---- .../general/simplify_dimensions.py | 22 +---- 5 files changed, 102 insertions(+), 106 deletions(-) create mode 100644 earthdiagnostics/general/fix_file.py diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index f80e9718..8615e58c 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -1,11 +1,12 @@ # coding=utf-8 """Set attributtes in netCDF files""" -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticComplexStrOption, \ - DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.general.fix_file import FixFile from earthdiagnostics.utils import Utils +from earthdiagnostics.diagnostic import DiagnosticDomainOption, DiagnosticVariableOption, DiagnosticOption, \ + DiagnosticComplexStrOption -class Attribute(Diagnostic): +class Attribute(FixFile): """ Set the value of an attribute @@ -34,13 +35,7 @@ class Attribute(Diagnostic): def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, attributte_name, attributte_value): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid + FixFile.__init__(self, data_manager, startdate, member, chunk, domain, variable, grid) self.attributte_name = attributte_name self.attributte_value = attributte_value @@ -78,15 +73,6 @@ class Attribute(Diagnostic): options['value'])) return job_list - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) def compute(self): """Run the diagnostic""" diff --git a/earthdiagnostics/general/fix_file.py b/earthdiagnostics/general/fix_file.py new file mode 100644 index 00000000..5c7d7378 --- /dev/null +++ b/earthdiagnostics/general/fix_file.py @@ -0,0 +1,88 @@ +# coding=utf-8 +"""Base diagnostic for fixing files""" +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption + + +class FixFile(Diagnostic): + """ + Base diagnostic class for diagnostic that read and reqrite the same files + + Must be derived + + :original author: Javier Vegas-Regidor + + :created: July 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable's name + :type variable: str + :param domain: variable's domain + :type domain: ModelingRealm + """ + + + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.variable = variable + self.domain = domain + self.grid = grid + + _STR_PREFIX = None + + def __str__(self): + return '{0._STR_PREFIX} Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) + + def __eq__(self, other): + if type(self) is not type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.domain == other.domain and self.variable == other.variable and self.grid == self.grid + + @classmethod + def generate_jobs(cls, diags, options): + """ + Create a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, domain, grid + :type options: list[str] + :return: + """ + options_available = cls._get_options(diags) + options = cls.process_options(options, options_available) + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(cls(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], options['grid'])) + return job_list + + @classmethod + def _get_options(cls, diags): + return [DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticOption('grid', '')] + + def request_data(self): + """Request data required by the diagnostic""" + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + """Declare data to be generated by the diagnostic""" + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index db19ee02..26cab98c 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -1,9 +1,9 @@ # coding=utf-8 """Rewrite netCDF file""" -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption +from earthdiagnostics.general.fix_file import FixFile -class Rewrite(Diagnostic): +class Rewrite(FixFile): """ Rewrites files without doing any calculations. @@ -30,54 +30,7 @@ class Rewrite(Diagnostic): alias = 'rewrite' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid - - def __str__(self): - return 'Rewrite output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, - self.grid) - - def __eq__(self, other): - return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.grid == self.grid - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, grid - :type options: list[str] - :return: - """ - options_available = (DiagnosticDomainOption(), - DiagnosticVariableOption(diags.data_manager.config.var_manager), - DiagnosticOption('grid', '')) - options = cls.process_options(options, options_available) - job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Rewrite(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], options['grid'])) - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + _STR_PREFIX = 'Rewrite output' def compute(self): """Run the diagnostic""" diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index b7d27de2..62944a3e 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -5,12 +5,13 @@ import math import numpy as np from earthdiagnostics.constants import Basins +from earthdiagnostics.general.fix_file import FixFile from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, DiagnosticVariableOption, \ DiagnosticFloatOption, DiagnosticBoolOption, DiagnosticListFrequenciesOption, DiagnosticOption from earthdiagnostics.utils import Utils -class Scale(Diagnostic): +class Scale(FixFile): """ Scales a variable by the given value also adding at offset @@ -40,13 +41,7 @@ class Scale(Diagnostic): def __init__(self, data_manager, startdate, member, chunk, value, offset, domain, variable, grid, min_limit, max_limit, frequency, apply_mask): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid + FixFile.__init__(self, data_manager, startdate, member, chunk, domain, variable, grid) self.value = value self.offset = offset self.min_limit = min_limit @@ -96,16 +91,6 @@ class Scale(Diagnostic): options['apply_mask'])) return job_list - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, frequency=self.frequency, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, frequency=self.frequency) - def compute(self): """Run the diagnostic""" variable_file = self.variable_file.local_file diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 6f73d379..fa5272d9 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -1,13 +1,13 @@ # coding=utf-8 """Convert i j files to lon lat when there is no interpolation required""" import numpy as np - +from earthdiagnostics.general.fix_file import FixFile from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption from earthdiagnostics.utils import Utils, TempFile -class SimplifyDimensions(Diagnostic): +class SimplifyDimensions(FixFile): """ Convert i j files to lon lat when there is no interpolation required @@ -29,13 +29,7 @@ class SimplifyDimensions(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid, data_convention): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid + FixFile.__init__(self, data_manager, startdate, member, chunk, domain, variable, grid) if data_convention in ('cmip6', 'primavera'): self.lon_name = 'longitude' self.lat_name = 'latitude' @@ -77,16 +71,6 @@ class SimplifyDimensions(Diagnostic): diags.config.data_convention)) return job_list - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, to_modify=True) - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - self.simplified = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) - def compute(self): """Run the diagnostic""" handler = Utils.open_cdf(self.variable_file.local_file) -- GitLab From 3b17a6d1d2bf3ff1f2c8c95bb5f21c9980697e9f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 16:16:46 +0100 Subject: [PATCH 29/70] Fix codacy warnings --- earthdiagnostics/general/attribute.py | 1 - earthdiagnostics/general/fix_file.py | 4 ---- earthdiagnostics/general/scale.py | 2 +- earthdiagnostics/general/simplify_dimensions.py | 2 +- 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 8615e58c..f1398569 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -73,7 +73,6 @@ class Attribute(FixFile): options['value'])) return job_list - def compute(self): """Run the diagnostic""" variable_file = self.variable_file.local_file diff --git a/earthdiagnostics/general/fix_file.py b/earthdiagnostics/general/fix_file.py index 5c7d7378..35ce2f3b 100644 --- a/earthdiagnostics/general/fix_file.py +++ b/earthdiagnostics/general/fix_file.py @@ -27,8 +27,6 @@ class FixFile(Diagnostic): :type domain: ModelingRealm """ - - def __init__(self, data_manager, startdate, member, chunk, domain, variable, grid): Diagnostic.__init__(self, data_manager) self.startdate = startdate @@ -84,5 +82,3 @@ class FixFile(Diagnostic): """Declare data to be generated by the diagnostic""" self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) - - diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 62944a3e..b1ceaca0 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -6,7 +6,7 @@ import numpy as np from earthdiagnostics.constants import Basins from earthdiagnostics.general.fix_file import FixFile -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticDomainOption, DiagnosticVariableOption, \ +from earthdiagnostics.diagnostic import DiagnosticDomainOption, DiagnosticVariableOption, \ DiagnosticFloatOption, DiagnosticBoolOption, DiagnosticListFrequenciesOption, DiagnosticOption from earthdiagnostics.utils import Utils diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index fa5272d9..1c613714 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -2,7 +2,7 @@ """Convert i j files to lon lat when there is no interpolation required""" import numpy as np from earthdiagnostics.general.fix_file import FixFile -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ +from earthdiagnostics.diagnostic import DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption from earthdiagnostics.utils import Utils, TempFile -- GitLab From bd0faeb9ef63a19a3e8d9d49b70d9357b5573850 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 16:22:22 +0100 Subject: [PATCH 30/70] Removed duplication --- earthdiagnostics/diagnostic.py | 3 +++ earthdiagnostics/ocean/regionmean.py | 2 +- earthdiagnostics/ocean/regionsum.py | 8 ++++++-- earthdiagnostics/ocean/rotation.py | 2 ++ earthdiagnostics/ocean/verticalgradient.py | 2 -- earthdiagnostics/ocean/verticalmean.py | 2 -- earthdiagnostics/ocean/verticalmeanmeters.py | 2 ++ 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 0af27101..252c1255 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -428,6 +428,9 @@ class Diagnostic(Publisher): return len([request.storage_status != StorageStatus.READY or request.local_status != LocalStatus.READY for request in self._requests]) + def _different_type(self, other): + return type(self) is not type(other) + class DiagnosticOption(object): """Class to manage string options for the diagnostic""" diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 78bc05b9..bb2431a3 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -64,7 +64,7 @@ class RegionMean(Diagnostic): self.lon_name = 'lon' def __eq__(self, other): - if type(self) is not type(other): + if self._different_type(other): return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index 6ba7a4e3..c334b37c 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -46,7 +46,7 @@ class RegionSum(Diagnostic): self.member = member self.chunk = chunk self.domain = domain - self.variable = variable + self.variable = variablex self.grid_point = grid_point.upper() self.box = box self.save3d = save3d @@ -58,8 +58,12 @@ class RegionSum(Diagnostic): self.lon_name = 'lon' def __eq__(self, other): + if self._different_type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.box == other.box and self.variable == other.variable + self.box == other.box and self.variable == other.variable and \ + self.grid_point == other.grid_point and self.grid == other.grid and self.basin == other.basin def __str__(self): return 'Region sum Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Variable: {0.variable} ' \ diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index b00a30b9..21e2e053 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -45,6 +45,8 @@ class Rotation(Diagnostic): self.tempTemplate = None def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variableu == other.variableu and \ self.variablev == other.variablev and self.executable == other.executable diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 64a933ed..9d14e91b 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -41,8 +41,6 @@ class VerticalGradient(Diagnostic): self.chunk = chunk self.variable = variable self.box = box - self.required_vars = [variable] - self.generated_vars = [variable + 'vmean'] def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index cf05d5eb..569598b5 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -42,8 +42,6 @@ class VerticalMean(Diagnostic): self.chunk = chunk self.variable = variable self.box = box - self.required_vars = [variable] - self.generated_vars = [variable + 'vmean'] def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index ee9d2084..18d9ce45 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -47,6 +47,8 @@ class VerticalMeanMeters(Diagnostic): self.grid_point = grid_point def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable -- GitLab From e103a899065acf382b8054b2b5108b4580b58c20 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 16:48:41 +0100 Subject: [PATCH 31/70] Fixed error --- earthdiagnostics/ocean/regionsum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/ocean/regionsum.py b/earthdiagnostics/ocean/regionsum.py index c334b37c..afb45224 100644 --- a/earthdiagnostics/ocean/regionsum.py +++ b/earthdiagnostics/ocean/regionsum.py @@ -46,7 +46,7 @@ class RegionSum(Diagnostic): self.member = member self.chunk = chunk self.domain = domain - self.variable = variablex + self.variable = variable self.grid_point = grid_point.upper() self.box = box self.save3d = save3d -- GitLab From a7a72b45fb92e887156a72e9efa497ba9d8ddc19 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 16:54:27 +0100 Subject: [PATCH 32/70] Added tyoe checker to Diagnostics _eq_ --- earthdiagnostics/general/attribute.py | 3 +++ earthdiagnostics/general/fix_file.py | 2 +- earthdiagnostics/general/module.py | 2 ++ earthdiagnostics/general/relink.py | 2 ++ earthdiagnostics/general/relinkall.py | 2 ++ earthdiagnostics/general/scale.py | 2 ++ earthdiagnostics/general/select_levels.py | 2 ++ earthdiagnostics/general/simplify_dimensions.py | 2 ++ earthdiagnostics/general/timemean.py | 2 ++ earthdiagnostics/general/verticalmeanmetersiris.py | 2 ++ earthdiagnostics/ocean/areamoc.py | 2 ++ earthdiagnostics/ocean/averagesection.py | 2 ++ earthdiagnostics/ocean/convectionsites.py | 2 ++ earthdiagnostics/ocean/cutsection.py | 2 ++ earthdiagnostics/ocean/gyres.py | 2 ++ earthdiagnostics/ocean/heatcontent.py | 2 +- earthdiagnostics/ocean/interpolate.py | 2 ++ earthdiagnostics/ocean/interpolatecdo.py | 2 +- earthdiagnostics/ocean/mask_land.py | 2 ++ earthdiagnostics/ocean/maxmoc.py | 2 ++ earthdiagnostics/ocean/mixedlayerheatcontent.py | 2 ++ earthdiagnostics/ocean/mixedlayersaltcontent.py | 2 ++ earthdiagnostics/ocean/moc.py | 2 ++ earthdiagnostics/ocean/mxl.py | 2 ++ earthdiagnostics/ocean/psi.py | 2 ++ earthdiagnostics/ocean/regionmean.py | 2 +- earthdiagnostics/ocean/verticalgradient.py | 2 ++ earthdiagnostics/ocean/verticalmean.py | 2 ++ earthdiagnostics/statistics/climatologicalpercentile.py | 2 ++ earthdiagnostics/statistics/daysoverpercentile.py | 2 ++ earthdiagnostics/statistics/discretize.py | 2 ++ earthdiagnostics/statistics/monthlypercentile.py | 2 ++ 32 files changed, 61 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index f1398569..1e8c8a50 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -45,6 +45,9 @@ class Attribute(FixFile): 'Grid: {0.grid}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.grid == other.grid and \ self.attributte_name == other.attributte_name and self.attributte_value == other.attributte_value diff --git a/earthdiagnostics/general/fix_file.py b/earthdiagnostics/general/fix_file.py index 35ce2f3b..83730292 100644 --- a/earthdiagnostics/general/fix_file.py +++ b/earthdiagnostics/general/fix_file.py @@ -43,7 +43,7 @@ class FixFile(Diagnostic): 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) def __eq__(self, other): - if type(self) is not type(other): + if self._different_type(other): return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.grid == self.grid diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index b8b31146..64904c38 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -49,6 +49,8 @@ class Module(Diagnostic): self.componentv, self.module, self.grid) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.componentu == other.componentu and \ self.componentv == other.componentv and self.module == other.module and self.grid == other.grid diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 1a9162d9..fcae8e51 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -47,6 +47,8 @@ class Relink(Diagnostic): 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.move_old == other.move_old and \ self.grid == other.grid diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 17d05df8..1b59e1b2 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -28,6 +28,8 @@ class RelinkAll(Diagnostic): return 'Relink all output Startdate: {0}'.format(self.startdate) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate @classmethod diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index b1ceaca0..de904de8 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -57,6 +57,8 @@ class Scale(FixFile): 'Frequency: {0.frequency} Apply mask: {0.apply_mask}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ self.apply_mask == other.apply_mask and self.value == other.value and self.offset == other.offset diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 239c2df1..3a6c38f1 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -45,6 +45,8 @@ class SelectLevels(Diagnostic): self.grid, self.box.min_depth, self.box.max_depth) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.grid == self.grid diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index 1c613714..90af9df6 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -43,6 +43,8 @@ class SimplifyDimensions(FixFile): self.grid) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.grid == self.grid diff --git a/earthdiagnostics/general/timemean.py b/earthdiagnostics/general/timemean.py index 0f0b5322..2addbe5e 100644 --- a/earthdiagnostics/general/timemean.py +++ b/earthdiagnostics/general/timemean.py @@ -46,6 +46,8 @@ class TimeMean(Diagnostic): 'Variable: {0.domain}:{0.variable} Original frequency: {0.frequency} Grid: {0.grid}'.format(self) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ self.grid == other.grid and self._target_frequency == other._target_frequency diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py index 4cc5f394..602fc5cd 100644 --- a/earthdiagnostics/general/verticalmeanmetersiris.py +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -49,6 +49,8 @@ class VerticalMeanMetersIris(Diagnostic): self.box = box def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 0f20d28d..d94a724b 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -56,6 +56,8 @@ class AreaMoc(Diagnostic): self.box = box def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.basin == other.basin and self.box == other.box diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 27eac1aa..aa879381 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -50,6 +50,8 @@ class AverageSection(Diagnostic): self.grid = grid def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.box == other.box diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index 08523f34..8a82d654 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -46,6 +46,8 @@ class ConvectionSites(Diagnostic): return 'Convection sites Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index 2690e6b5..1059e443 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -60,6 +60,8 @@ class CutSection(Diagnostic): self.box.min_lat = self.value def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.zonal == other.zonal and \ self.value == other.value diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index 942a2467..38d22f9f 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -43,6 +43,8 @@ class Gyres(Diagnostic): self.var_vsftbarot = None def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index 476c639f..a9332915 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -52,7 +52,7 @@ class HeatContent(Diagnostic): self.max_level = max_level def __eq__(self, other): - if not isinstance(other, HeatContent): + if self._different_type(other): return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.basin == other.basin and self.mxloption == other.mxloption diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 56f47844..76e9d78a 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -62,6 +62,8 @@ class Interpolate(Diagnostic): self.original_grid = original_grid def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version and self.domain == other.domain and \ self.variable == other.variable and self.grid == other.grid and \ diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 13e1de63..eccdf7ea 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -65,7 +65,7 @@ class InterpolateCDO(Diagnostic): self.weights = weights def __eq__(self, other): - if type(self) is not type(other): + if self._different_type(other): return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index 01062c11..451d9d2a 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -37,6 +37,8 @@ class MaskLand(Diagnostic): self.grid = grid def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index eba04a78..0e13d71f 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -56,6 +56,8 @@ class MaxMoc(Diagnostic): 'Basin: {4}'.format(self.startdate, self.member, self.year, self.box, self.basin.name) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.year == other.year and \ self.box == other.box and self.basin == other.basin diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index fa03a558..529d7108 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -40,6 +40,8 @@ class MixedLayerHeatContent(Diagnostic): self.generated_vars = ['scvertsum'] def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk def __str__(self): diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index a6f7b455..170815fe 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -44,6 +44,8 @@ class MixedLayerSaltContent(Diagnostic): self.chunk) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk @classmethod diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index a300e8b1..644b4f38 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -47,6 +47,8 @@ class Moc(Diagnostic): return 'MOC Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk @classmethod diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index 1c733c83..2022daaa 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -32,6 +32,8 @@ class Mxl(Diagnostic): self.chunk = chunk def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk def __str__(self): diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index 03107eda..b9b32f02 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -38,6 +38,8 @@ class Psi(Diagnostic): self.chunk = chunk def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk def __str__(self): diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index bb2431a3..12121f52 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -314,7 +314,7 @@ class ComputeWeights(Diagnostic): self.box = box def __eq__(self, other): - if type(self) is not type(other): + if self._different_type(other): return False return self.weights_file == other.weights_file and self.basin == other.basin and \ self.grid_point == other.grid_point and self.box != other.box diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 9d14e91b..32ee7460 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -43,6 +43,8 @@ class VerticalGradient(Diagnostic): self.box = box def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 569598b5..795dc456 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -44,6 +44,8 @@ class VerticalMean(Diagnostic): self.box = box def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 8050f7ef..b4585b23 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -47,6 +47,8 @@ class ClimatologicalPercentile(Diagnostic): self.leadtime_files = {} def __eq__(self, other): + if self._different_type(other): + return False return self.domain == other.domain and self.variable == other.variable and \ self.start_year == other.start_year and self.end_year == other.end_year and \ self.forecast_month == other.forecast_month diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py index ad1b25f3..ca1c77df 100644 --- a/earthdiagnostics/statistics/daysoverpercentile.py +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -48,6 +48,8 @@ class DaysOverPercentile(Diagnostic): self.lon_coord = None def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.domain == other.domain and \ self.variable == other.variable and self.start_year == other.start_year and \ self.end_year == other.end_year diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py index e577d90b..e96a95f7 100644 --- a/earthdiagnostics/statistics/discretize.py +++ b/earthdiagnostics/statistics/discretize.py @@ -92,6 +92,8 @@ class Discretize(Diagnostic): self._bins = value def __eq__(self, other): + if self._different_type(other): + return False return self.domain == other.domain and self.variable == other.variable and self.num_bins == other.num_bins and \ self.min_value == other.min_value and self.max_value == other.max_value and \ self.startdate == other.startdate diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 9e3575bd..bbd33dd6 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -39,6 +39,8 @@ class MonthlyPercentile(Diagnostic): self.percentiles = percentiles def __eq__(self, other): + if self._different_type(other): + return False return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.percentiles == other.percentiles -- GitLab From c08425b7e22ef72d97c8a4c793fa929a53abfc88 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 17:20:21 +0100 Subject: [PATCH 33/70] Fixed cmorization bug --- earthdiagnostics/cmorizer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index efa2da3f..b01d7901 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -381,15 +381,14 @@ class Cmorizer(object): os.remove(filename) @staticmethod - def _remove_valid_limits(filename): - handler = Utils.open_cdf(filename) + def _remove_valid_limits(handler): for variable in handler.variables.keys(): var = handler.variables[variable] if 'valid_min' in var.ncattrs(): del var.valid_min if 'valid_max' in var.ncattrs(): del var.valid_max - handler.close() + handler.sync() def _get_nc_file_frequency(self, filename): file_parts = os.path.basename(filename).split('_') -- GitLab From 1b3ae4c3a2a2e0975a19fe6b71cce00567f69436 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 16 Feb 2018 17:21:57 +0100 Subject: [PATCH 34/70] Added leadtime to coordinates attribute if present --- earthdiagnostics/datafile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 8bd25b4d..23a3a006 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -304,7 +304,8 @@ class DataFile(Publisher): def _correct_metadata(self): handler = Utils.open_cdf(self.local_file) var_handler = handler.variables[self.final_name] - coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name}, set(handler.variables.keys())) + coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name, 'leadtime'}, + set(handler.variables.keys())) var_handler.coordinates = ' '.join(coords) if not self.cmor_var: handler.close() -- GitLab From 685ee5bc64f5f4fbc327351e8916235827b10597 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 19 Feb 2018 10:57:04 +0100 Subject: [PATCH 35/70] Refactored compute weights for heatcontentlayer --- earthdiagnostics/ocean/heatcontentlayer.py | 60 +++++++++++++--------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index 88a74a7c..135f9f4b 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -81,29 +81,7 @@ class HeatContentLayer(Diagnostic): @classmethod def _compute_weights(cls, box): - handler = Utils.open_cdf('mesh_zgr.nc') - # mask = Utils.get_mask(options['basin']) - mask = handler.variables['tmask'][:] - if 'e3t' in handler.variables: - e3t = handler.variables['e3t'][:] - elif 'e3t_0' in handler.variables: - e3t = handler.variables['e3t_0'][:] - else: - raise Exception('e3t variable can not be found') - if 'gdepw' in handler.variables: - depth = handler.variables['gdepw'][:] - elif 'gdepw_0' in handler.variables: - depth = handler.variables['gdepw_0'][:] - else: - raise Exception('gdepw variable can not be found') - e3t_3d = e3t.shape != depth.shape - if e3t_3d: - mask = e3t_3d * mask - else: - e3t = e3t[0, :] - while len(depth.shape) < 4: - depth = np.expand_dims(depth, -1) - handler.close() + depth, e3t, mask = cls._get_mesh_info() def calculate_weight(e3t_point, depth_point, mask_point): """Calculate the weight for each cell""" @@ -123,6 +101,12 @@ class HeatContentLayer(Diagnostic): calc = np.vectorize(calculate_weight, otypes='f') weight = calc(e3t, depth, mask) + max_level, min_level = cls._get_used_levels(weight) + weight = weight[:, min_level:max_level, :] + return max_level, min_level, weight + + @classmethod + def _get_used_levels(cls, weight): # Now we will reduce to the levels with any weight != 0 to avoid loading too much data on memory levels = weight.shape[1] min_level = 0 @@ -131,8 +115,34 @@ class HeatContentLayer(Diagnostic): max_level = min_level while max_level < (levels - 1) and weight[:, max_level + 1, :].any(): max_level += 1 - weight = weight[:, min_level:max_level, :] - return max_level, min_level, weight + return max_level, min_level + + @classmethod + def _get_mesh_info(cls): + handler = Utils.open_cdf('mesh_zgr.nc') + # mask = Utils.get_mask(options['basin']) + mask = handler.variables['tmask'][:] + if 'e3t' in handler.variables: + e3t = handler.variables['e3t'][:] + elif 'e3t_0' in handler.variables: + e3t = handler.variables['e3t_0'][:] + else: + raise Exception('e3t variable can not be found') + if 'gdepw' in handler.variables: + depth = handler.variables['gdepw'][:] + elif 'gdepw_0' in handler.variables: + depth = handler.variables['gdepw_0'][:] + else: + raise Exception('gdepw variable can not be found') + e3t_3d = e3t.shape != depth.shape + if e3t_3d: + mask = e3t_3d * mask + else: + e3t = e3t[0, :] + while len(depth.shape) < 4: + depth = np.expand_dims(depth, -1) + handler.close() + return depth, e3t, mask def request_data(self): """Request data required by the diagnostic""" -- GitLab From 2b012236a4575d9bb08194131fe4ade05ec39d52 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 19 Feb 2018 16:07:29 +0100 Subject: [PATCH 36/70] Added cmorizer tests --- earthdiagnostics/cmorizer.py | 18 +++-- test/integration/__init__.py | 0 test/integration/test_cmorizer.py | 126 ++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 test/integration/__init__.py create mode 100644 test/integration/test_cmorizer.py diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index b01d7901..471bcf08 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -366,18 +366,19 @@ class Cmorizer(object): Cmorizer._remove_valid_limits(handler) self._add_common_attributes(handler, frequency) self._update_time_variables(handler) - handler.sync() + + variables = handler.variables.keys() + handler.close() Log.info('Splitting file {0}', filename) - for variable in handler.variables.keys(): + for variable in variables: if variable in Cmorizer.NON_DATA_VARIABLES: continue try: - self.extract_variable(filename, handler, frequency, variable) + self.extract_variable(filename, frequency, variable) except Exception as ex: Log.error('Variable {0} can not be cmorized: {1}', variable, ex) Log.result('File {0} cmorized!', filename) - handler.close() os.remove(filename) @staticmethod @@ -408,14 +409,13 @@ class Cmorizer(object): variables = Utils.get_file_variables(filename) return self.cmor.any_required(variables) - def extract_variable(self, file_path, handler, frequency, variable): + def extract_variable(self, file_path, frequency, variable): """ Extract a variable from a file and creates the CMOR file Parameters ---------- file_path:str - handler: netCDF4.Dataset frequency: Frequency variable: str @@ -436,7 +436,7 @@ class Cmorizer(object): Utils.nco.ncks(input=file_path, output=temp, options=('-v {0}'.format(variable),)) self._rename_level_variables(temp, var_cmor) - self._add_coordinate_variables(handler, temp) + self._add_coordinate_variables(file_path, temp) if alias.basin is None: region = None @@ -503,13 +503,15 @@ class Cmorizer(object): raise Exception('File {0} start date is not a valid chunk start date'.format(file_path)) return chunk - def _add_coordinate_variables(self, handler, temp): + def _add_coordinate_variables(self, file_path, temp): handler_cmor = Utils.open_cdf(temp) + handler = Utils.open_cdf(file_path, 'r') Utils.copy_variable(handler, handler_cmor, self.lon_name, False) Utils.copy_variable(handler, handler_cmor, self.lat_name, False) if 'time' in handler_cmor.dimensions.keys(): Utils.copy_variable(handler, handler_cmor, 'leadtime', False) handler_cmor.close() + handler.close() @staticmethod def _rename_level_variables(temp, var_cmor): diff --git a/test/integration/__init__.py b/test/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py new file mode 100644 index 00000000..4b4b8e37 --- /dev/null +++ b/test/integration/test_cmorizer.py @@ -0,0 +1,126 @@ +from earthdiagnostics.cmorizer import Cmorizer +from earthdiagnostics.utils import TempFile +from bscearth.utils import log + +from unittest import TestCase +from mock import Mock +import os +import tempfile +import shutil +import iris +import iris.cube +from iris.coords import DimCoord, AuxCoord +import tarfile +import numpy as np +import six + + +class TestCmorizer(TestCase): + + def _get_variable_and_alias(self, variable): + mock_alias = Mock() + mock_alias.basin = None + mock_alias.grid = None + + mock_variable = Mock() + mock_variable.short_name = variable + mock_variable.domain = 'domain' + + return mock_alias, mock_variable + + def _get_file_path(self, *args, **kwargs): + return os.path.join(self.tmp_dir, args[3], '{0[3]}.nc'.format(args)) + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + + self.data_manager = Mock() + self.data_manager.get_file_path = self._get_file_path + self.data_manager.config.data_dir = os.path.join(self.tmp_dir, 'data') + self.data_manager.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') + TempFile.scratch_folder = self.data_manager.config.scratch_dir + self.data_manager.config.data_convention = 'data_convention' + + self.data_manager.config.var_manager.get_variable_and_alias = self._get_variable_and_alias + + self.data_manager.config.experiment.expid = 'expid' + self.data_manager.config.experiment.model = 'model' + self.data_manager.config.experiment.experiment_name = 'experiment_name' + self.data_manager.config.experiment.num_chunks = 1 + self.data_manager.config.experiment.chunk_size = 1 + self.data_manager.config.experiment.institute = 'institute' + self.data_manager.config.experiment.get_member_str.return_value = 'member' + + self.data_manager.config.cmor.ocean = True + self.data_manager.config.cmor.atmosphere = True + self.data_manager.config.cmor.filter_files = '' + self.data_manager.config.cmor.associated_experiment = 'associated_experiment' + self.data_manager.config.cmor.initialization_method = 'initialization_method' + self.data_manager.config.cmor.initialization_description = 'initialization_description' + self.data_manager.config.cmor.physics_version = 'physics_version' + self.data_manager.config.cmor.physics_description = 'physics_description' + self.data_manager.config.cmor.initialization_description = 'initialization_description' + self.data_manager.config.cmor.associated_model = 'initialization_description' + self.data_manager.config.cmor.source = 'source' + + os.makedirs(self.data_manager.config.data_dir) + os.makedirs(self.data_manager.config.scratch_dir) + + def create_ocean_files(self, filename, tar_name): + iris.FUTURE.netcdf_no_unlimited = True + coord_data = np.array([0, 1], np.float) + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', units='degrees_east') + time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', units='days since 1950-01-01') + depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') + + var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') + var1.add_dim_coord(time, 0) + var1.add_dim_coord(lat, 1) + var1.add_dim_coord(lon, 2) + + var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') + var2.add_dim_coord(time, 0) + var2.add_dim_coord(lat, 1) + var2.add_dim_coord(lon, 2) + var2.add_dim_coord(depth, 3) + + folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', + 'outputs') + os.makedirs(folder_path) + file_path = os.path.join(folder_path, filename) + iris.save((var1, var2), file_path, zlib=True) + + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') + tar.add(file_path, arcname=filename, recursive=False) + tar.close() + os.remove(file_path) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_skip_ocean_cmorization(self): + self.data_manager.config.cmor.ocean = False + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_skip_atmos_cmorization(self): + self.data_manager.config.cmor.atmosphere = False + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + + def test_ocean_cmorization(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) -- GitLab From 6d6b1ded648f8f7076df1538b154b275fd3fd51d Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 19 Feb 2018 16:51:53 +0100 Subject: [PATCH 37/70] Added gzip test --- test/integration/test_cmorizer.py | 59 +++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 4b4b8e37..b69a04fa 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -11,6 +11,7 @@ import iris import iris.cube from iris.coords import DimCoord, AuxCoord import tarfile +import gzip as gz import numpy as np import six @@ -66,12 +67,15 @@ class TestCmorizer(TestCase): os.makedirs(self.data_manager.config.data_dir) os.makedirs(self.data_manager.config.scratch_dir) - def create_ocean_files(self, filename, tar_name): + def create_ocean_files(self, filename, tar_name, gzip=False): iris.FUTURE.netcdf_no_unlimited = True coord_data = np.array([0, 1], np.float) - lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', units='degrees_north') - lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', units='degrees_east') - time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', units='days since 1950-01-01') + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', + units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', + units='degrees_east') + time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', + units='days since 1950-01-01') depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') @@ -91,6 +95,15 @@ class TestCmorizer(TestCase): file_path = os.path.join(folder_path, filename) iris.save((var1, var2), file_path, zlib=True) + if gzip: + import subprocess + process = subprocess.Popen(('gzip', file_path), stdout=subprocess.PIPE) + comunicate = process.communicate() + file_path = "{0}.gz".format(file_path) + filename = "{0}.gz".format(filename) + if process.returncode != 0: + raise Exception('Can not compress: {0}'.format(comunicate)) + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') tar.add(file_path, arcname=filename, recursive=False) tar.close() @@ -109,8 +122,20 @@ class TestCmorizer(TestCase): cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_atmos() - def test_ocean_cmorization(self): - self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + def test_ocean_cmorization_no_files(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def _test_ocean_cmor(self): if six.PY3: with self.assertLogs(log.Log.log) as cmd: cmorizer = Cmorizer(self.data_manager, '19900101', 0) @@ -122,5 +147,27 @@ class TestCmorizer(TestCase): else: cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_ocean() + + def test_ocean_cmorization(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_gzip(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', gzip=True) + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_PPO(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'PPO_expid_1D_xx_19900101_19900131.tar') + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_diags(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'diags_expid_1D_xx_19900101_19900131.tar') + self._test_ocean_cmor() self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) -- GitLab From 26ecaf3fbfc6786501fc3f4b689df39e94ee6ead Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 19 Feb 2018 17:07:37 +0100 Subject: [PATCH 38/70] Added test for cmor config --- test/integration/test_cmorizer.py | 54 ++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index b69a04fa..71daa20e 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -37,6 +37,7 @@ class TestCmorizer(TestCase): self.data_manager = Mock() self.data_manager.get_file_path = self._get_file_path + self.data_manager.is_cmorized.return_value = False self.data_manager.config.data_dir = os.path.join(self.tmp_dir, 'data') self.data_manager.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') TempFile.scratch_folder = self.data_manager.config.scratch_dir @@ -52,6 +53,7 @@ class TestCmorizer(TestCase): self.data_manager.config.experiment.institute = 'institute' self.data_manager.config.experiment.get_member_str.return_value = 'member' + self.data_manager.config.cmor.force = False self.data_manager.config.cmor.ocean = True self.data_manager.config.cmor.atmosphere = True self.data_manager.config.cmor.filter_files = '' @@ -67,7 +69,7 @@ class TestCmorizer(TestCase): os.makedirs(self.data_manager.config.data_dir) os.makedirs(self.data_manager.config.scratch_dir) - def create_ocean_files(self, filename, tar_name, gzip=False): + def create_ocean_files(self, filename, tar_name, gzip=False, backup=False): iris.FUTURE.netcdf_no_unlimited = True coord_data = np.array([0, 1], np.float) lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', @@ -104,6 +106,9 @@ class TestCmorizer(TestCase): if process.returncode != 0: raise Exception('Can not compress: {0}'.format(comunicate)) + if backup: + filename = os.path.join('backup', filename) + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') tar.add(file_path, arcname=filename, recursive=False) tar.close() @@ -114,13 +119,46 @@ class TestCmorizer(TestCase): def test_skip_ocean_cmorization(self): self.data_manager.config.cmor.ocean = False - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_ocean() + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertLogs([record for record in cmd.records if + record.message == 'Skipping ocean cmorization due to configuration']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() def test_skip_atmos_cmorization(self): self.data_manager.config.cmor.atmosphere = False - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_atmos() + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + self.assertLogs([record for record in cmd.records if + record.message == 'Skipping atmosphere cmorization due to configuration']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_skip_when_cmorized(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.is_cmorized.return_value = True + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if + record.message == 'No need to unpack file 1/1']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_force(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.is_cmorized.return_value = True + self.data_manager.config.cmor.force = True + self._test_ocean_cmor() def test_ocean_cmorization_no_files(self): if six.PY3: @@ -160,6 +198,12 @@ class TestCmorizer(TestCase): self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + def test_ocean_cmorization_backup(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', backup=True) + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + def test_ocean_cmorization_PPO(self): self.create_ocean_files('expid_1d_19900101_19900131.nc', 'PPO_expid_1D_xx_19900101_19900131.tar') self._test_ocean_cmor() -- GitLab From 9fed0b38da8712bd79e8c63b4ec4f33a6ba7b936 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 19 Feb 2018 18:08:45 +0100 Subject: [PATCH 39/70] Added more cmorizer tests --- earthdiagnostics/cmorizer.py | 13 +++-- test/integration/test_cmorizer.py | 95 +++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 471bcf08..1aeb6859 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -119,7 +119,7 @@ class Cmorizer(object): def _cmorize_nc_files(self): nc_files = glob.glob(os.path.join(self.cmor_scratch, '*.nc')) - for filename in nc_files: + for filename in self._filter_files(nc_files): self._cmorize_nc_file(filename) self._clean_cmor_scratch() @@ -156,11 +156,12 @@ class Cmorizer(object): Log.debug('Moving file {0}', filepath) shutil.move(filepath, filepath.replace('/backup/', '/')) zip_files = glob.glob(os.path.join(self.cmor_scratch, '*.gz')) - for zip_file in self._filter_files(zip_files): - try: - Utils.unzip(zip_file) - except Utils.UnzipException as ex: - Log.error('File {0} could not be unzipped: {1}', tarfile, ex) + if zip_files: + for zip_file in self._filter_files(zip_files): + try: + Utils.unzip(zip_file) + except Utils.UnzipException as ex: + Log.error('File {0} could not be unzipped: {1}', tarfile, ex) def _clean_cmor_scratch(self): if os.path.exists(self.cmor_scratch): diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 71daa20e..f664db2c 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -154,6 +154,19 @@ class TestCmorizer(TestCase): cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_ocean() + def test_skip_when_not_requested(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.chunk_cmorization_requested.return_value = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if + record.message == 'No need to unpack file 1/1']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + def test_force(self): self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') self.data_manager.is_cmorized.return_value = True @@ -173,6 +186,64 @@ class TestCmorizer(TestCase): cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_ocean() + def test_ocean_cmorization_not_vars_requested(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.any_required.return_value = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_no_vars_recognized(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + + def not_recognized(*args): + return None, None + self.data_manager.config.var_manager.get_variable_and_alias= not_recognized + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_var2_not_requested(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + + def _reject_var2(cmor_var): + return cmor_var.short_name != 'var2' + + self.data_manager.config.cmor.cmorize = _reject_var2 + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + def _test_ocean_cmor(self): if six.PY3: with self.assertLogs(log.Log.log) as cmd: @@ -192,6 +263,30 @@ class TestCmorizer(TestCase): self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + def test_ocean_cmorization_with_filter(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.filter_files = 'expid' + self._test_ocean_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_ocean_cmorization_with_bad_filter(self): + self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar') + self.data_manager.config.cmor.filter_files = 'badfilter' + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertFalse(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + def test_ocean_cmorization_gzip(self): self.create_ocean_files('expid_1d_19900101_19900131.nc', 'MMO_19900101-19900131.tar', gzip=True) self._test_ocean_cmor() -- GitLab From 18a582627c6485bddbd02a36871fdc22c2636c41 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 11:10:39 +0100 Subject: [PATCH 40/70] Added test for cmorization of MMA files --- earthdiagnostics/utils.py | 2 +- test/integration/test_cmorizer.py | 78 +++++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index bf44e4b4..ee51684c 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -42,7 +42,7 @@ class Utils(object): nco = Nco() """An instance of Nco class ready to be used""" - cdo = Cdo() + cdo = Cdo(env=os.environ) """An instance of Cdo class ready to be used""" @staticmethod diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index f664db2c..8be5b180 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -9,9 +9,8 @@ import tempfile import shutil import iris import iris.cube -from iris.coords import DimCoord, AuxCoord +from iris.coords import DimCoord import tarfile -import gzip as gz import numpy as np import six @@ -23,11 +22,16 @@ class TestCmorizer(TestCase): mock_alias.basin = None mock_alias.grid = None + mock_variable = self._get_variable(variable) + + return mock_alias, mock_variable + + def _get_variable(self, variable, silent=False): mock_variable = Mock() mock_variable.short_name = variable mock_variable.domain = 'domain' - return mock_alias, mock_variable + return mock_variable def _get_file_path(self, *args, **kwargs): return os.path.join(self.tmp_dir, args[3], '{0[3]}.nc'.format(args)) @@ -44,6 +48,8 @@ class TestCmorizer(TestCase): self.data_manager.config.data_convention = 'data_convention' self.data_manager.config.var_manager.get_variable_and_alias = self._get_variable_and_alias + self.data_manager.config.var_manager.get_variable = self._get_variable + self.data_manager.variable_list = self.data_manager.config.var_manager self.data_manager.config.experiment.expid = 'expid' self.data_manager.config.experiment.model = 'model' @@ -52,6 +58,7 @@ class TestCmorizer(TestCase): self.data_manager.config.experiment.chunk_size = 1 self.data_manager.config.experiment.institute = 'institute' self.data_manager.config.experiment.get_member_str.return_value = 'member' + self.data_manager.config.experiment.atmos_timestep = 6 self.data_manager.config.cmor.force = False self.data_manager.config.cmor.ocean = True @@ -310,3 +317,68 @@ class TestCmorizer(TestCase): self._test_ocean_cmor() self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def test_atmos_cmorization(self): + self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + self._test_atmos_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var1', 'var1.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) + + def _test_atmos_cmor(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + + def create_mma_files(self, filename, tar_name, gzip=False): + iris.FUTURE.netcdf_no_unlimited = True + coord_data = np.array([0, 1], np.float) + folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', + 'outputs') + filepath_gg, filename_gg = self._create_file(coord_data, folder_path, filename.replace('??', 'GG'), gzip) + filepath_sh, filename_sh = self._create_file(coord_data, folder_path, filename.replace('??', 'SH'), gzip) + + tar = tarfile.TarFile(os.path.join(folder_path, tar_name), mode='w') + tar.add(filepath_gg, arcname=filename_gg, recursive=False) + tar.add(filepath_sh, arcname=filename_sh, recursive=False) + tar.close() + os.remove(filepath_gg) + os.remove(filepath_sh) + + def _create_file(self, coord_data, folder_path, filename, gzip): + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', + units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', + units='degrees_east') + time = DimCoord(coord_data, standard_name='time', long_name='time', var_name='time', + units='days since 1950-01-01') + depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') + var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') + var1.add_dim_coord(time, 0) + var1.add_dim_coord(lat, 1) + var1.add_dim_coord(lon, 2) + var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') + var2.add_dim_coord(time, 0) + var2.add_dim_coord(lat, 1) + var2.add_dim_coord(lon, 2) + var2.add_dim_coord(depth, 3) + if not os.path.isdir(folder_path): + os.makedirs(folder_path) + file_path = os.path.join(folder_path, filename) + iris.save((var1, var2), file_path, zlib=True) + if gzip: + import subprocess + process = subprocess.Popen(('gzip', file_path), stdout=subprocess.PIPE) + comunicate = process.communicate() + file_path = "{0}.gz".format(file_path) + filename = "{0}.gz".format(filename) + if process.returncode != 0: + raise Exception('Can not compress: {0}'.format(comunicate)) + return file_path, filename -- GitLab From 51c8fe4807986588836754e2c01cb47d889834b9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 11:14:23 +0100 Subject: [PATCH 41/70] Added test for cmorization of MMA files --- test/integration/test_cmorizer.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 8be5b180..eb861bff 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -382,3 +382,35 @@ class TestCmorizer(TestCase): if process.returncode != 0: raise Exception('Can not compress: {0}'.format(comunicate)) return file_path, filename + + def test_skip_when_not_requested_mma(self): + self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + self.data_manager.config.cmor.chunk_cmorization_requested.return_value = False + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + self.assertTrue([record for record in cmd.records if + record.message == 'No need to unpack file 1/1']) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_ocean() + + def test_force_mma(self): + self.create_mma_files('MMA_1d_??_19900101_19900131.nc', 'MMA_expid_19901101_fc0_19900101-19900131.tar') + self.data_manager.is_cmorized.return_value = True + self.data_manager.config.cmor.force = True + self._test_atmos_cmor() + + def test_ocean_cmorization_no_mma_files(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + else: + cmorizer = Cmorizer(self.data_manager, '19900101', 0) + cmorizer.cmorize_atmos() -- GitLab From 06c92d695eab52edd51e90af79247b959b047a38 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 15:20:34 +0100 Subject: [PATCH 42/70] Added test for cmorization of grib files --- earthdiagnostics/cmorizer.py | 8 +--- test/integration/test_cmorizer.py | 65 +++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 1aeb6859..b7350ca9 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -339,12 +339,8 @@ class Cmorizer(object): import pygrib grib_handler = pygrib.open(gribfile) dates = set() - try: - while True: - mes = grib_handler.next() - dates.add(mes.analDate) - except StopIteration: - pass + for mes in grib_handler: + dates.add(mes.validDate) dates = list(dates) dates.sort() atmos_timestep = dates[1] - dates[0] diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index eb861bff..6e832dc7 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -1,5 +1,5 @@ from earthdiagnostics.cmorizer import Cmorizer -from earthdiagnostics.utils import TempFile +from earthdiagnostics.utils import TempFile, Utils from bscearth.utils import log from unittest import TestCase @@ -63,6 +63,7 @@ class TestCmorizer(TestCase): self.data_manager.config.cmor.force = False self.data_manager.config.cmor.ocean = True self.data_manager.config.cmor.atmosphere = True + self.data_manager.config.cmor.use_grib = True self.data_manager.config.cmor.filter_files = '' self.data_manager.config.cmor.associated_experiment = 'associated_experiment' self.data_manager.config.cmor.initialization_method = 'initialization_method' @@ -72,13 +73,16 @@ class TestCmorizer(TestCase): self.data_manager.config.cmor.initialization_description = 'initialization_description' self.data_manager.config.cmor.associated_model = 'initialization_description' self.data_manager.config.cmor.source = 'source' + self.data_manager.config.cmor.get_requested_codes.return_value = [142, 143] + self.data_manager.config.cmor.get_variables.return_value = [142, 143] + self.data_manager.config.cmor.get_levels.return_value = None os.makedirs(self.data_manager.config.data_dir) os.makedirs(self.data_manager.config.scratch_dir) def create_ocean_files(self, filename, tar_name, gzip=False, backup=False): iris.FUTURE.netcdf_no_unlimited = True - coord_data = np.array([0, 1], np.float) + coord_data = np.array([1, 2], np.float) lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', units='degrees_north') lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', @@ -93,6 +97,7 @@ class TestCmorizer(TestCase): var1.add_dim_coord(lon, 2) var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') + time.bounds = np.array([[0.5, 1.5], [1.5, 2.5]], np.float) var2.add_dim_coord(time, 0) var2.add_dim_coord(lat, 1) var2.add_dim_coord(lon, 2) @@ -402,7 +407,7 @@ class TestCmorizer(TestCase): self.data_manager.config.cmor.force = True self._test_atmos_cmor() - def test_ocean_cmorization_no_mma_files(self): + def test_atmos_cmorization_no_mma_files(self): if six.PY3: with self.assertLogs(log.Log.log) as cmd: cmorizer = Cmorizer(self.data_manager, '19900101', 0) @@ -414,3 +419,57 @@ class TestCmorizer(TestCase): else: cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_atmos() + + def create_grib_files(self, filename): + + iris.FUTURE.netcdf_no_unlimited = True + coord_data = np.array([0, 1], np.float) + folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', + 'outputs') + filepath_gg = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), [142, 254]) + filepath_sh = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [143, 255]) + Utils.cdo.copy(input=filepath_gg, output=filepath_gg.replace('.nc', '.grb'), + options='-f grb2') + Utils.cdo.copy(input=filepath_sh, output=filepath_sh.replace('.nc', '.grb'), + options='-f grb2') + os.remove(filepath_gg) + os.remove(filepath_sh) + + def _create_file_for_grib(self, coord_data, folder_path, filename, codes): + lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', + units='degrees_north') + lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', + units='degrees_east') + time_data = np.array([0, 0.25], np.float) + time = DimCoord(time_data, standard_name='time', long_name='time', var_name='time', + units='days since 1950-01-01') + depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') + var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') + var1.add_dim_coord(time, 0) + var1.add_dim_coord(lat, 1) + var1.add_dim_coord(lon, 2) + var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') + var2.add_dim_coord(time, 0) + var2.add_dim_coord(lat, 1) + var2.add_dim_coord(lon, 2) + var2.add_dim_coord(depth, 3) + if not os.path.isdir(folder_path): + os.makedirs(folder_path) + file_path = os.path.join(folder_path, filename) + iris.save((var1, var2), file_path, zlib=True) + + handler = Utils.open_cdf(file_path) + handler.variables['var1'].code = np.int32(codes[0]) + handler.variables['var1'].table = np.int32(128) + + handler.variables['var2'].code = np.int32(codes[1]) + handler.variables['var2'].table = np.int32(128) + handler.close() + + return file_path + + def test_grib_cmorization(self): + self.create_grib_files('ICM??expid+199001.nc') + self._test_atmos_cmor() + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'CP', 'CP.nc'))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'LSP', 'LSP.nc'))) -- GitLab From 27f0bdc41b896ce2249f598170324d03c79f6692 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 15:28:56 +0100 Subject: [PATCH 43/70] Added test to quality control --- .codacy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.codacy.yml b/.codacy.yml index 445a627b..01be9bca 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -5,9 +5,6 @@ engines: coverage: enabled: true - exclude_paths: [ - 'tests', - ] metrics: enabled: true duplication: -- GitLab From 0be19f5ddbeb7fdc4f94737ce4fb0b5fc41e3aac Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 15:34:26 +0100 Subject: [PATCH 44/70] Removed test from quality control --- .codacy.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.codacy.yml b/.codacy.yml index 01be9bca..445a627b 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -5,6 +5,9 @@ engines: coverage: enabled: true + exclude_paths: [ + 'tests', + ] metrics: enabled: true duplication: -- GitLab From fe37733ef00d57bc15451bbbc37197792be1111e Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 15:59:52 +0100 Subject: [PATCH 45/70] Added PEP8 checker as test --- test/integration/test_cmorizer.py | 2 +- test/unit/general/test_attribute.py | 5 +-- test/unit/general/test_dailymean.py | 5 +-- test/unit/general/test_module.py | 5 +-- test/unit/general/test_monthlymean.py | 5 +-- test/unit/general/test_relink.py | 5 +-- test/unit/general/test_rewrite.py | 4 +-- test/unit/general/test_scale.py | 5 +-- test/unit/general/test_select_levels.py | 5 +-- test/unit/general/test_simplify_dimensions.py | 8 +++-- .../general/test_verticalmeanmetersiris.py | 5 +-- test/unit/general/test_yearlymean.py | 5 +-- test/unit/ocean/test_areamoc.py | 6 ++-- test/unit/ocean/test_averagesection.py | 5 +-- test/unit/ocean/test_cutsection.py | 5 +-- test/unit/ocean/test_heatcontent.py | 4 +-- test/unit/ocean/test_interpolate.py | 7 ++-- test/unit/ocean/test_interpolatecdo.py | 7 ++-- test/unit/ocean/test_maxmoc.py | 4 +-- test/unit/ocean/test_region_mean.py | 5 +-- test/unit/ocean/test_rotation.py | 3 +- test/unit/ocean/test_siasiesiv.py | 2 +- test/unit/ocean/test_verticalmean.py | 4 +-- test/unit/ocean/test_verticalmeanmeters.py | 4 +-- .../test_climatologicalpercentile.py | 4 +-- .../statistics/test_daysoverpercentile.py | 4 +-- test/unit/statistics/test_discretize.py | 8 ++--- .../unit/statistics/test_monthlypercentile.py | 2 +- test/unit/test_cmormanager.py | 19 +++++------ test/unit/test_config.py | 33 ++++++++----------- test/unit/test_data_manager.py | 2 -- test/unit/test_frequency.py | 1 - test/unit/test_lint.py | 32 ++++++++++++++++++ test/unit/test_modelling_realm.py | 4 --- test/unit/test_obsreconmanager.py | 4 --- test/unit/test_publisher.py | 1 - test/unit/test_utils.py | 1 - test/unit/test_variable_type.py | 3 -- test/unit/test_workmanager.py | 1 - 39 files changed, 129 insertions(+), 105 deletions(-) create mode 100644 test/unit/test_lint.py diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 6e832dc7..e59b9c5c 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -220,7 +220,7 @@ class TestCmorizer(TestCase): def not_recognized(*args): return None, None - self.data_manager.config.var_manager.get_variable_and_alias= not_recognized + self.data_manager.config.var_manager.get_variable_and_alias = not_recognized if six.PY3: with self.assertLogs(log.Log.log) as cmd: cmorizer = Cmorizer(self.data_manager, '19900101', 0) diff --git a/test/unit/general/test_attribute.py b/test/unit/general/test_attribute.py index 5616b347..3e2cce9b 100644 --- a/test/unit/general/test_attribute.py +++ b/test/unit/general/test_attribute.py @@ -50,5 +50,6 @@ class TestAttribute(TestCase): def test_str(self): mixed = Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'att', 'value') - self.assertEqual(str(mixed), 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 ' - 'Variable: atmos:var Attributte: att:value Grid: grid') + self.assertEqual(str(mixed), + 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Attributte: att:value Grid: grid') diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py index 8fa092c6..426e8931 100644 --- a/test/unit/general/test_dailymean.py +++ b/test/unit/general/test_dailymean.py @@ -51,5 +51,6 @@ class TestDailyMean(TestCase): def test_str(self): mixed = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') - self.assertEqual(str(mixed), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + self.assertEqual(str(mixed), + 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_module.py b/test/unit/general/test_module.py index 218223b9..39549556 100644 --- a/test/unit/general/test_module.py +++ b/test/unit/general/test_module.py @@ -50,5 +50,6 @@ class TestModule(TestCase): def test_str(self): mixed = Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'varu', 'varv', 'varmodule', 'grid') - self.assertEqual(str(mixed), 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' - 'Variables: atmos:varu,varv,varmodule Grid: grid') + self.assertEqual(str(mixed), + 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' + 'Variables: atmos:varu,varv,varmodule Grid: grid') diff --git a/test/unit/general/test_monthlymean.py b/test/unit/general/test_monthlymean.py index adc75cab..5672c462 100644 --- a/test/unit/general/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -51,5 +51,6 @@ class TestMonthlyMean(TestCase): MonthlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + self.assertEqual(str(self.mixed), + 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_relink.py b/test/unit/general/test_relink.py index 970bb0ea..d2810bd1 100644 --- a/test/unit/general/test_relink.py +++ b/test/unit/general/test_relink.py @@ -57,5 +57,6 @@ class TestRelink(TestCase): def test_str(self): mixed = Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', True, 'grid') - self.assertEqual(str(mixed), 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' - 'Variable: ocean:var Grid: grid') + self.assertEqual(str(mixed), + 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/general/test_rewrite.py b/test/unit/general/test_rewrite.py index 4a6fd038..ed0cb80f 100644 --- a/test/unit/general/test_rewrite.py +++ b/test/unit/general/test_rewrite.py @@ -47,5 +47,5 @@ class TestRewrite(TestCase): Rewrite.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: atmos:var Grid: grid') + self.assertEqual(str(self.mixed), + 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var Grid: grid') diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py index 23e1f7b7..92321f39 100644 --- a/test/unit/general/test_scale.py +++ b/test/unit/general/test_scale.py @@ -75,5 +75,6 @@ class TestScale(TestCase): def test_str(self): mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, Frequencies.three_hourly, False) - self.assertEqual(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' - 'Variable: atmos:var Frequency: 3hr Apply mask: False') + self.assertEqual(str(mixed), + 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' + 'Variable: atmos:var Frequency: 3hr Apply mask: False') diff --git a/test/unit/general/test_select_levels.py b/test/unit/general/test_select_levels.py index bca61711..146f0744 100644 --- a/test/unit/general/test_select_levels.py +++ b/test/unit/general/test_select_levels.py @@ -62,5 +62,6 @@ class TestSelectLevels(TestCase): def test_str(self): mixed = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 20) - self.assertEqual(str(mixed), 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Levels: 0-20 Grid: grid') + self.assertEqual(str(mixed), + 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Levels: 0-20 Grid: grid') diff --git a/test/unit/general/test_simplify_dimensions.py b/test/unit/general/test_simplify_dimensions.py index 5a36288e..348f7053 100644 --- a/test/unit/general/test_simplify_dimensions.py +++ b/test/unit/general/test_simplify_dimensions.py @@ -51,6 +51,8 @@ class TestSimplifyDimensions(TestCase): SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid', 'extra']) def test_str(self): - mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'convention') - self.assertEqual(str(mixed), 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Grid: grid') + mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + 'grid', 'convention') + self.assertEqual(str(mixed), + 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Grid: grid') diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py index ca972cac..49f6a1b5 100644 --- a/test/unit/general/test_verticalmeanmetersiris.py +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -71,5 +71,6 @@ class TestVerticalMeanMetersIris(TestCase): box.min_depth = 0 box.max_depth = 100 mixed = VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', box) - self.assertEqual(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Box: 0-100m') + self.assertEqual(str(mixed), + 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Box: 0-100m') diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py index 36b48eaf..5c0d7c03 100644 --- a/test/unit/general/test_yearlymean.py +++ b/test/unit/general/test_yearlymean.py @@ -51,5 +51,6 @@ class TestYearlyMean(TestCase): YearlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Original frequency: freq Grid: ') + self.assertEqual(str(self.mixed), + 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/ocean/test_areamoc.py b/test/unit/ocean/test_areamoc.py index 648ce675..cb7e7e7e 100644 --- a/test/unit/ocean/test_areamoc.py +++ b/test/unit/ocean/test_areamoc.py @@ -8,7 +8,7 @@ from mock import Mock, patch class TestAreaMoc(TestCase): - + def setUp(self): self.data_manager = Mock() self.diags = Mock() @@ -53,5 +53,5 @@ class TestAreaMoc(TestCase): AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.psi), 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 ' - 'Basin: Atlantic') + self.assertEqual(str(self.psi), + 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 Basin: Atlantic') diff --git a/test/unit/ocean/test_averagesection.py b/test/unit/ocean/test_averagesection.py index a5905589..5b00d64b 100644 --- a/test/unit/ocean/test_averagesection.py +++ b/test/unit/ocean/test_averagesection.py @@ -51,5 +51,6 @@ class TestAverageSection(TestCase): def test_str(self): diag = AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', self.box, 'grid') - self.assertEqual(str(diag), 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' - 'Variable: ocean:var Grid: grid') + self.assertEqual(str(diag), + 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/ocean/test_cutsection.py b/test/unit/ocean/test_cutsection.py index e27102a0..c88bc664 100644 --- a/test/unit/ocean/test_cutsection.py +++ b/test/unit/ocean/test_cutsection.py @@ -51,5 +51,6 @@ class TestCutSection(TestCase): CutSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' - 'Zonal: True Value: 0') + self.assertEqual(str(self.psi), + 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' + 'Zonal: True Value: 0') diff --git a/test/unit/ocean/test_heatcontent.py b/test/unit/ocean/test_heatcontent.py index a2387016..517f1d30 100644 --- a/test/unit/ocean/test_heatcontent.py +++ b/test/unit/ocean/test_heatcontent.py @@ -44,5 +44,5 @@ class TestHeatContent(TestCase): def test_str(self): diag = HeatContent(self.data_manager, '20010101', 0, 0, Basins().Global, -1, self.box, 1, 20) - self.assertEqual(str(diag), 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100 ' - 'Basin: Global') + self.assertEqual(str(diag), + 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100 Basin: Global') diff --git a/test/unit/ocean/test_interpolate.py b/test/unit/ocean/test_interpolate.py index 8bd284dc..7a348c25 100644 --- a/test/unit/ocean/test_interpolate.py +++ b/test/unit/ocean/test_interpolate.py @@ -71,6 +71,7 @@ class TestInterpolate(TestCase): def test_str(self): diag = Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'model_version', True, 'original_grid') - self.assertEqual(str(diag), 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' - 'Target grid: grid Invert lat: True Model: model_version ' - 'Original grid: original_grid') + self.assertEqual(str(diag), + 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Target grid: grid Invert lat: True Model: model_version ' + 'Original grid: original_grid') diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index fccf47de..73c6d39e 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -88,6 +88,7 @@ class TestInterpolate(TestCase): def test_str(self): diag = InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'atmos_grid', 'model_version', False, 'orig', None) - self.assertEqual(str(diag), 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' - 'Target grid: atmos_grid Original grid: orig Mask ocean: False ' - 'Model: model_version') + self.assertEqual(str(diag), + 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' + 'Target grid: atmos_grid Original grid: orig Mask ocean: False ' + 'Model: model_version') diff --git a/test/unit/ocean/test_maxmoc.py b/test/unit/ocean/test_maxmoc.py index 3224050e..2605e0cc 100644 --- a/test/unit/ocean/test_maxmoc.py +++ b/test/unit/ocean/test_maxmoc.py @@ -63,5 +63,5 @@ class TestMaxMoc(TestCase): MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.maxmoc), 'Max moc Startdate: 20000101 Member: 1 Year: 2000 ' - 'Box: 0.0N0m Basin: Global') + self.assertEqual(str(self.maxmoc), + 'Max moc Startdate: 20000101 Member: 1 Year: 2000 Box: 0.0N0m Basin: Global') diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py index 2ee865fa..313456fe 100644 --- a/test/unit/ocean/test_region_mean.py +++ b/test/unit/ocean/test_region_mean.py @@ -98,5 +98,6 @@ class TestRegionMean(TestCase): diag = RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', box, False, 'file', True, Basins().Global) - self.assertEqual(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var ' - 'Box: 1-10 Save 3D: False Save variance: True') + self.assertEqual(str(diag), + 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Box: 1-10 ' + 'Save 3D: False Save variance: True') diff --git a/test/unit/ocean/test_rotation.py b/test/unit/ocean/test_rotation.py index 08c61fb1..4cc90e47 100644 --- a/test/unit/ocean/test_rotation.py +++ b/test/unit/ocean/test_rotation.py @@ -43,4 +43,5 @@ class TestMixedLayerHeatContent(TestCase): Rotation.generate_jobs(self.diags, ['diagnostic', 'ocean', 'varu', 'varv', 'exe', 'extra']) def test_str(self): - self.assertEqual(str(self.mixed), 'Rotate variables Startdate: 20000101 Member: 1 Chunk: 1 Variables: ocean:varu , ocean:varv') + self.assertEqual(str(self.mixed), + 'Rotate variables Startdate: 20000101 Member: 1 Chunk: 1 Variables: ocean:varu , ocean:varv') diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py index 59ac3043..c2d7cbab 100644 --- a/test/unit/ocean/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -21,4 +21,4 @@ class TestSiasiesiv(TestCase): def test_str(self): self.assertEqual(str(self.psi), - 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global Omit volume: False') + 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global Omit volume: False') diff --git a/test/unit/ocean/test_verticalmean.py b/test/unit/ocean/test_verticalmean.py index f6ae18c2..3b25f50f 100644 --- a/test/unit/ocean/test_verticalmean.py +++ b/test/unit/ocean/test_verticalmean.py @@ -53,5 +53,5 @@ class TestVerticalMean(TestCase): VerticalMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' - 'Box: 0-100') + self.assertEqual(str(self.mixed), + 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var Box: 0-100') diff --git a/test/unit/ocean/test_verticalmeanmeters.py b/test/unit/ocean/test_verticalmeanmeters.py index ff6580a2..315e6a36 100644 --- a/test/unit/ocean/test_verticalmeanmeters.py +++ b/test/unit/ocean/test_verticalmeanmeters.py @@ -60,5 +60,5 @@ class TestVerticalMeanMeters(TestCase): VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEqual(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Box: 0-100m') + self.assertEqual(str(self.mixed), + 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 Variable: ocean:var Box: 0-100m') diff --git a/test/unit/statistics/test_climatologicalpercentile.py b/test/unit/statistics/test_climatologicalpercentile.py index 3a905cbb..385174b6 100644 --- a/test/unit/statistics/test_climatologicalpercentile.py +++ b/test/unit/statistics/test_climatologicalpercentile.py @@ -38,5 +38,5 @@ class TestClimatologicalPercentile(TestCase): diagnostic = ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, 11, self.diags.config.experiment) - self.assertEqual(str(diagnostic), 'Climatological percentile Variable: ocean:var Period: 2000-2001 ' - 'Forecast month: 11') + self.assertEqual(str(diagnostic), + 'Climatological percentile Variable: ocean:var Period: 2000-2001 Forecast month: 11') diff --git a/test/unit/statistics/test_daysoverpercentile.py b/test/unit/statistics/test_daysoverpercentile.py index afc1a03e..caf59553 100644 --- a/test/unit/statistics/test_daysoverpercentile.py +++ b/test/unit/statistics/test_daysoverpercentile.py @@ -30,5 +30,5 @@ class TestDaysOverPercentile(TestCase): def test_str(self): diagnostic = DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, '20001101', 11) - self.assertEqual(str(diagnostic), 'Days over percentile Startdate: 20001101 Variable: ocean:var ' - 'Climatology: 2000-2001') + self.assertEqual(str(diagnostic), + 'Days over percentile Startdate: 20001101 Variable: ocean:var Climatology: 2000-2001') diff --git a/test/unit/statistics/test_discretize.py b/test/unit/statistics/test_discretize.py index 872c5109..78a71479 100644 --- a/test/unit/statistics/test_discretize.py +++ b/test/unit/statistics/test_discretize.py @@ -50,11 +50,11 @@ class TestClimatologicalPercentile(TestCase): def test_str(self): diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, 10, 40) - self.assertEqual(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' - 'Range: [10, 40]') + self.assertEqual(str(diagnostic), + 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 Range: [10, 40]') diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, float('nan'), float('nan')) - self.assertEqual(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' - 'Range: [None, None]') + self.assertEqual(str(diagnostic), + 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 Range: [None, None]') diff --git a/test/unit/statistics/test_monthlypercentile.py b/test/unit/statistics/test_monthlypercentile.py index 8127d5a4..d0e4da48 100644 --- a/test/unit/statistics/test_monthlypercentile.py +++ b/test/unit/statistics/test_monthlypercentile.py @@ -38,4 +38,4 @@ class TestMonthlyPercentile(TestCase): def test_str(self): self.assertEqual(str(self.diagnostic), 'Monthly percentile Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Percentiles: 10, 90') + 'Variable: ocean:var Percentiles: 10, 90') diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 6a32c367..0117df3f 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -105,6 +105,7 @@ class TestCMORManager(TestCase): os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + def test_get_file_path_specs_empty_time_info(self): cmor_manager = CMORManager(self.config) with self.assertRaises(ValueError): @@ -165,8 +166,8 @@ class TestCMORManager(TestCase): 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_1998.nc')) - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, - 1, 'mon', year='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', + cmor_var, 1, 'mon', year='1998') def test_get_file_path_raise_incompatible_date_info(self): cmor_manager = CMORManager(self.config) @@ -232,7 +233,8 @@ class TestCMORManager(TestCase): self.assertEqual(file_path, os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expnameS19900101/' 'r2i1p1f1/Omon/var/ocean_grid/version/' - 'var_Omon_model_expnameS19900101_r2i1p1f1_ocean_grid_198901-198912.nc')) + 'var_Omon_model_expnameS19900101_r2i1p1f1_ocean_grid_' + '198901-198912.nc')) def test_get_file_path_primavera_no_cmor(self): self._configure_primavera() @@ -314,8 +316,8 @@ class TestCMORManager(TestCase): 'var/ocean_grid/version/' 'var_Omon_model_expname_r2i1p1f1_ocean_grid_1998.nc')) - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, - 1, 'mon', year='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', + cmor_var, 1, 'mon', year='1998') def test_get_file_path_primavera_date_str(self): self._configure_primavera() @@ -346,7 +348,7 @@ class TestCMORManager(TestCase): self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1, possible_versions=('version1', 'version2'))) isfile.return_value = False - self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1, possible_versions=('version1', 'version2'))) def test_get_file_path_meteofrance(self): @@ -797,8 +799,6 @@ class TestCMORManager(TestCase): datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) self.assertEqual(datafile.remote_file, '/path/to/file') - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') def test_request_year(self, mock_get_file_path): self.config.experiment.get_year_chunks.return_value = (1, 2) @@ -833,6 +833,3 @@ class TestMergeYear(TestCase): MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998)) self.assertEqual(MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998, 'grid'), MergeYear(self.data_manager, ModelingRealms.ocean, 'var', 'startdate', 1, 1998, 'grid')) - - - diff --git a/test/unit/test_config.py b/test/unit/test_config.py index b1ee9671..eb9dfd6a 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -204,10 +204,10 @@ class TestCMORConfig(TestCase): self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.six_hourly), {128: None, - 129: '1', - 130: '1,2', - 131: '1,2,3,4,5,6,7,8,9', - 132: '0,5'}) + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) self.assertEqual(config.get_levels(Frequencies.six_hourly, 128), None) self.assertEqual(config.get_levels(Frequencies.six_hourly, 129), '1') @@ -222,10 +222,10 @@ class TestCMORConfig(TestCase): self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.daily), {128: None, - 129: '1', - 130: '1,2', - 131: '1,2,3,4,5,6,7,8,9', - 132: '0,5'}) + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) self.assertEqual(config.get_levels(Frequencies.daily, 128), None) self.assertEqual(config.get_levels(Frequencies.daily, 129), '1') @@ -240,10 +240,10 @@ class TestCMORConfig(TestCase): self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') config = CMORConfig(self.mock_parser, self.var_manager) self.assertEqual(config.get_variables(Frequencies.monthly), {128: None, - 129: '1', - 130: '1,2', - 131: '1,2,3,4,5,6,7,8,9', - 132: '0,5'}) + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) self.assertEqual(config.get_levels(Frequencies.monthly, 128), None) self.assertEqual(config.get_levels(Frequencies.monthly, 129), '1') @@ -355,8 +355,8 @@ class TestExperimentConfig(TestCase): config = ExperimentConfig() config.parse_ini(self.mock_parser) self.assertEqual(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', - u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', - u'20020801', u'20021101']) + u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', + u'20020801', u'20021101']) def test_auto_startdates(self): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') @@ -524,8 +524,3 @@ class TestConfig(TestCase): namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'earthdiagnostics/CDFTOOLS_primavera.namlist')) self.assertEqual(os.environ['NAM_CDF_NAMES'], namelist) - - - - - diff --git a/test/unit/test_data_manager.py b/test/unit/test_data_manager.py index 00d03b8c..cb4c7af8 100644 --- a/test/unit/test_data_manager.py +++ b/test/unit/test_data_manager.py @@ -65,5 +65,3 @@ class TestDataManager(TestCase): box.get_lat_str.return_value = '_lat' box.get_depth_str.return_value = '_depth' self.assertEqual(self.data_manager._get_final_var_name(box, 'var'), 'var_lon_lat_depth') - - diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py index b92069e3..be424181 100644 --- a/test/unit/test_frequency.py +++ b/test/unit/test_frequency.py @@ -40,4 +40,3 @@ class TestFrequency(TestCase): def test_get_climatology(self): self.assertEqual(Frequency('clim').folder_name(VariableType.STATISTIC), 'clim') self.assertEqual(Frequency('clim').folder_name(VariableType.MEAN), 'clim') - diff --git a/test/unit/test_lint.py b/test/unit/test_lint.py new file mode 100644 index 00000000..03bdeb4b --- /dev/null +++ b/test/unit/test_lint.py @@ -0,0 +1,32 @@ +""" Lint tests """ +import os +import textwrap +import unittest + +import pycodestyle # formerly known as pep8 + + +class TestLint(unittest.TestCase): + + def test_pep8_conformance(self): + """Test that we conform to PEP-8.""" + check_paths = [ + 'earthdiagnostics', + 'test', + ] + exclude_paths = [ + ] + + print("PEP8 check of directories: {}\n".format(', '.join(check_paths))) + + # Get paths wrt package root + package_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + for paths in (check_paths, exclude_paths): + for i, path in enumerate(paths): + paths[i] = os.path.join(package_root, path) + + style = pycodestyle.StyleGuide() + style.options.exclude.extend(exclude_paths) + style.options.max_line_length = 120 + + self.assertEqual(style.check_files(check_paths).total_errors, 0) diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 3752346f..103bcdc3 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -49,7 +49,3 @@ class TestModellingRealm(TestCase): def test_get_6hrplev(self): self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev') - - - - diff --git a/test/unit/test_obsreconmanager.py b/test/unit/test_obsreconmanager.py index 05d98f35..b04cc568 100644 --- a/test/unit/test_obsreconmanager.py +++ b/test/unit/test_obsreconmanager.py @@ -136,7 +136,3 @@ class TestObsReconManager(TestCase): cmor_manager = ObsReconManager(self.config) datafile = cmor_manager.request_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) self.assertEqual(datafile.remote_file, '/path/to/file') - - - - diff --git a/test/unit/test_publisher.py b/test/unit/test_publisher.py index a9124832..f884f46a 100644 --- a/test/unit/test_publisher.py +++ b/test/unit/test_publisher.py @@ -45,4 +45,3 @@ class TestPublisher(TestCase): pub.subscribe(suscriber2, callback=suscriber.callback) self.assertFalse(pub.only_suscriber(suscriber)) self.assertFalse(pub.only_suscriber(suscriber2)) - diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 2162cab9..3b274e3f 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -57,4 +57,3 @@ class TestUtils(TestCase): Utils.rename_variable('file', 'old', 'new', False, True) rename_mock.assert_has_calls((mock.call('file', {'old': 'new'}, True, False), mock.call('file', {'old': 'new'}, False, True))) - diff --git a/test/unit/test_variable_type.py b/test/unit/test_variable_type.py index 5b98e6ae..cb3dfa05 100644 --- a/test/unit/test_variable_type.py +++ b/test/unit/test_variable_type.py @@ -15,6 +15,3 @@ class TestVariableType(TestCase): def test_bad_one(self): with self.assertRaises(ValueError): VariableType.to_str('bad type') - - - diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index ad5c88b6..6052f226 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -31,7 +31,6 @@ class TestDownloader(TestCase): class TestWorkManager(TestCase): - def setUp(self): self.config = Mock() self.data_manager = Mock() -- GitLab From bca275788c0f5e1b3dfb0b712b06424a77e68ac3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 16:06:10 +0100 Subject: [PATCH 46/70] Added pycodestyle to requirements --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 417ec6ba..e11e830e 100644 --- a/environment.yml +++ b/environment.yml @@ -22,6 +22,7 @@ dependencies: - coverage - pytest - pytest-cov +- pycodestyle - pip: - bscearth.utils -- GitLab From 1ebca29418d06f68848184b49a1cf073fe47654a Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 16:22:00 +0100 Subject: [PATCH 47/70] Changed deprecated call --- earthdiagnostics/variable.py | 2 +- test/integration/test_cmorizer.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index fee14f52..36efe266 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -280,7 +280,7 @@ class VariableManager(object): if row[1].value in excel.sheetnames: table_data[row[1].value] = (Frequency(row[2].value), 'Date missing') for sheet_name in excel.sheetnames: - sheet = excel.get_sheet_by_name(sheet_name) + sheet = excel[sheet_name] if sheet.title == 'Primday': pass if sheet['A1'].value != 'Priority': diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index e59b9c5c..7e04c77d 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -330,14 +330,21 @@ class TestCmorizer(TestCase): self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'var2', 'var2.nc'))) def _test_atmos_cmor(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_atmos() - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + + try: + self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + except AssertionError: + print(cmd) + raise else: cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_atmos() -- GitLab From 85eb95b1b99b7b18bf7883cb0f69710783ce05b9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 17:45:34 +0100 Subject: [PATCH 48/70] Fixed bug in MMA cmorization We were using the fullpath instead of the filename in a split, and this was provoking errors with the tests --- earthdiagnostics/cmorizer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index b7350ca9..2e1cfde4 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -4,6 +4,7 @@ import glob import os import shutil import uuid +import traceback from datetime import datetime from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months @@ -172,6 +173,7 @@ class Cmorizer(object): sh_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_SH_*.nc')) gg_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_GG_*.nc')) + merged_sh = TempFile.get() merged_gg = TempFile.get() @@ -182,7 +184,7 @@ class Cmorizer(object): Utils.cdo.mergetime(input=gg_files, output=merged_gg) for filename in sh_files + gg_files: os.remove(filename) - tar_startdate = tarfile[0:-4].split('_')[5].split('-') + tar_startdate = os.path.basename(tarfile[0:-4]).split('_')[4].split('-') shutil.move(merged_gg, os.path.join(self.cmor_scratch, 'MMAGG_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) shutil.move(merged_sh, os.path.join(self.cmor_scratch, 'MMASH_1m_{0[0]}_{0[1]}.nc'.format(tar_startdate))) @@ -217,7 +219,7 @@ class Cmorizer(object): self._cmorize_nc_files() Log.result('Atmospheric file {0}/{1} finished'.format(count, len(tar_files))) except Exception as ex: - Log.error('Could not cmorize atmospheric file {0}: {1}', count, ex) + Log.error('Could not cmorize atmospheric file {0}: {1}\n {2}', count, ex, traceback.format_exc()) count += 1 -- GitLab From 2df0d600020d76d05345d4f2b2004bf5a9febcba Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 20 Feb 2018 17:55:04 +0100 Subject: [PATCH 49/70] Fixed PEP8 error --- earthdiagnostics/cmorizer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 2e1cfde4..8bf4133f 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -173,7 +173,6 @@ class Cmorizer(object): sh_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_SH_*.nc')) gg_files = glob.glob(os.path.join(self.cmor_scratch, 'MMA_*_GG_*.nc')) - merged_sh = TempFile.get() merged_gg = TempFile.get() -- GitLab From 2e20407a3b9c3cf8a9648567b7606da5d08d38e8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Feb 2018 12:10:31 +0100 Subject: [PATCH 50/70] Improved grib tests --- earthdiagnostics/cmorizer.py | 7 +++---- test/integration/test_cmorizer.py | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 8bf4133f..248fa5bf 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -532,8 +532,7 @@ class Cmorizer(object): if os.path.exists('ICM'): os.remove('ICM') Utils.execute_shell_command('grib_filter -o ICM rules_files ' - '{0} {1}'.format(os.path.basename(prev_gribfile), - os.path.basename(gribfile))) + '{0} {1}'.format(prev_gribfile, gribfile)) os.remove('rules_files') Utils.remove_file(prev_gribfile) @@ -593,10 +592,10 @@ class Cmorizer(object): if var_code == 201: cdo_operator = "-monmean -daymax {0}".format(cdo_operator) elif var_code == 202: - cdo_operator = "-monmean -daymax {0}".format(cdo_operator) + cdo_operator = "-monmean -daymin {0}".format(cdo_operator) else: cdo_operator = "-monmean {0} ".format(cdo_operator) - if frequency == Frequencies.daily: + elif frequency == Frequencies.daily: if var_code == 201: cdo_operator = "-daymax {0} ".format(cdo_operator) elif var_code == 202: diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 7e04c77d..807e8c76 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -73,8 +73,8 @@ class TestCmorizer(TestCase): self.data_manager.config.cmor.initialization_description = 'initialization_description' self.data_manager.config.cmor.associated_model = 'initialization_description' self.data_manager.config.cmor.source = 'source' - self.data_manager.config.cmor.get_requested_codes.return_value = [142, 143] - self.data_manager.config.cmor.get_variables.return_value = [142, 143] + self.data_manager.config.cmor.get_requested_codes.return_value = [142, 143, 201, 202] + self.data_manager.config.cmor.get_variables.return_value = [142, 143, 201, 202] self.data_manager.config.cmor.get_levels.return_value = None os.makedirs(self.data_manager.config.data_dir) @@ -427,14 +427,14 @@ class TestCmorizer(TestCase): cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_atmos() - def create_grib_files(self, filename): - + def create_grib_files(self, filename, month): + filename = filename.format(month) iris.FUTURE.netcdf_no_unlimited = True coord_data = np.array([0, 1], np.float) folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', 'outputs') - filepath_gg = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), [142, 254]) - filepath_sh = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [143, 255]) + filepath_gg = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), [142, 201], month) + filepath_sh = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [143, 202], month) Utils.cdo.copy(input=filepath_gg, output=filepath_gg.replace('.nc', '.grb'), options='-f grb2') Utils.cdo.copy(input=filepath_sh, output=filepath_sh.replace('.nc', '.grb'), @@ -442,14 +442,15 @@ class TestCmorizer(TestCase): os.remove(filepath_gg) os.remove(filepath_sh) - def _create_file_for_grib(self, coord_data, folder_path, filename, codes): + def _create_file_for_grib(self, coord_data, folder_path, filename, codes, month): + month -= 1 lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', units='degrees_north') lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', units='degrees_east') - time_data = np.array([0, 0.25], np.float) + time_data = np.array([0, 0.25], np.float) + month * 31 time = DimCoord(time_data, standard_name='time', long_name='time', var_name='time', - units='days since 1950-01-01') + units='days since 1990-01-01') depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') var1.add_dim_coord(time, 0) @@ -476,7 +477,9 @@ class TestCmorizer(TestCase): return file_path def test_grib_cmorization(self): - self.create_grib_files('ICM??expid+199001.nc') + self.data_manager.config.experiment.chunk_size = 2 + self.create_grib_files('ICM??expid+19900{}.nc', 1) + self.create_grib_files('ICM??expid+19900{}.nc', 2) self._test_atmos_cmor() self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'CP', 'CP.nc'))) self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'LSP', 'LSP.nc'))) -- GitLab From 437b181d6a6c46be9e2eccc1d22cee914edb0053 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Feb 2018 12:43:00 +0100 Subject: [PATCH 51/70] Improved grib tests --- earthdiagnostics/cmorizer.py | 2 +- test/integration/test_cmorizer.py | 43 ++++++++++++++----------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 248fa5bf..3f3304bc 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -286,7 +286,7 @@ class Cmorizer(object): codes = self.cmor.get_requested_codes() if 228 in codes: - codes.update(142, 143) + codes.update((142, 143)) codes_str = ','.join([str(code) for code in codes]) try: if grid == 'SH': diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 807e8c76..e163135b 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -73,8 +73,8 @@ class TestCmorizer(TestCase): self.data_manager.config.cmor.initialization_description = 'initialization_description' self.data_manager.config.cmor.associated_model = 'initialization_description' self.data_manager.config.cmor.source = 'source' - self.data_manager.config.cmor.get_requested_codes.return_value = [142, 143, 201, 202] - self.data_manager.config.cmor.get_variables.return_value = [142, 143, 201, 202] + self.data_manager.config.cmor.get_requested_codes.return_value = {228, 142, 143, 201, 202, 129, 169, 180} + self.data_manager.config.cmor.get_variables.return_value = {228, 142, 143, 201, 202, 129, 169, 180} self.data_manager.config.cmor.get_levels.return_value = None os.makedirs(self.data_manager.config.data_dir) @@ -433,8 +433,11 @@ class TestCmorizer(TestCase): coord_data = np.array([0, 1], np.float) folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', 'outputs') - filepath_gg = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), [142, 201], month) - filepath_sh = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [143, 202], month) + filepath_gg = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), + [142, 143, 129, 169, 180], + month) + filepath_sh = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [201, 202], + month) Utils.cdo.copy(input=filepath_gg, output=filepath_gg.replace('.nc', '.grb'), options='-f grb2') Utils.cdo.copy(input=filepath_sh, output=filepath_sh.replace('.nc', '.grb'), @@ -451,28 +454,22 @@ class TestCmorizer(TestCase): time_data = np.array([0, 0.25], np.float) + month * 31 time = DimCoord(time_data, standard_name='time', long_name='time', var_name='time', units='days since 1990-01-01') - depth = DimCoord(coord_data, standard_name='depth', long_name='Depth', var_name='lev', units='m') - var1 = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), long_name='Variable 1', var_name='var1') - var1.add_dim_coord(time, 0) - var1.add_dim_coord(lat, 1) - var1.add_dim_coord(lon, 2) - var2 = iris.cube.Cube(np.random.rand(2, 2, 2, 2).astype(np.float), long_name='Variable 2', var_name='var2') - var2.add_dim_coord(time, 0) - var2.add_dim_coord(lat, 1) - var2.add_dim_coord(lon, 2) - var2.add_dim_coord(depth, 3) + vars = [] + for code in codes: + var = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), + long_name='Variable {}'.format(code), + var_name='var{}'.format(code)) + var.add_dim_coord(time, 0) + var.add_dim_coord(lat, 1) + var.add_dim_coord(lon, 2) + var.attributes['table'] = np.int32(128) + var.attributes['code'] = np.int32(code) + vars.append(var) + if not os.path.isdir(folder_path): os.makedirs(folder_path) file_path = os.path.join(folder_path, filename) - iris.save((var1, var2), file_path, zlib=True) - - handler = Utils.open_cdf(file_path) - handler.variables['var1'].code = np.int32(codes[0]) - handler.variables['var1'].table = np.int32(128) - - handler.variables['var2'].code = np.int32(codes[1]) - handler.variables['var2'].table = np.int32(128) - handler.close() + iris.save(vars, file_path, zlib=True, local_keys=('table', 'code')) return file_path -- GitLab From 08ec137b9fe87aa5289087dce28a4ee0abe0756b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Feb 2018 13:02:02 +0100 Subject: [PATCH 52/70] Improved grib tests --- test/integration/test_cmorizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index e163135b..2cb1af68 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -478,5 +478,5 @@ class TestCmorizer(TestCase): self.create_grib_files('ICM??expid+19900{}.nc', 1) self.create_grib_files('ICM??expid+19900{}.nc', 2) self._test_atmos_cmor() - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'CP', 'CP.nc'))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'LSP', 'LSP.nc'))) + for var in ('CP', 'EWSS', 'LSP', 'MN2T', 'MX2T', 'SSRD', 'TP', 'Z'): + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, var, '{}.nc'.format(var)))) -- GitLab From c8045eabc483ad6574f0cc5e47869bb0b2cb1ba8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Feb 2018 13:10:18 +0100 Subject: [PATCH 53/70] Improved grib tests and fixed CDO version --- earthdiagnostics/cmorizer.py | 10 +++++----- environment.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 3f3304bc..a9b3e7df 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -258,7 +258,7 @@ class Cmorizer(object): self._obtain_atmos_timestep(gribfile) full_file = self._get_monthly_grib(current_date, gribfile, grid) - if not self._unpack_grib(full_file, gribfile, grid): + if not self._unpack_grib(full_file, gribfile, grid, current_date.month): os.remove(gribfile) return @@ -280,7 +280,7 @@ class Cmorizer(object): self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, '{0}hr'.format(self.atmos_timestep)) - def _unpack_grib(self, full_file, gribfile, grid): + def _unpack_grib(self, full_file, gribfile, grid, month): Log.info('Unpacking... ') # remap on regular Gauss grid @@ -290,12 +290,12 @@ class Cmorizer(object): codes_str = ','.join([str(code) for code in codes]) try: if grid == 'SH': - Utils.cdo.splitparam(input='-sp2gpl -selcode,{0} {1} '.format(codes_str, full_file), + Utils.cdo.splitparam(input='-sp2gpl -selmon,{2} -selcode,{0} {1} '.format(codes_str, full_file, month), output=gribfile + '_', options='-f nc4') else: - Utils.cdo.splitparam(input='-selcode,{0} {1}'.format(codes_str, full_file), + Utils.cdo.splitparam(input='-selmon,{2} -selcode,{0} {1}'.format(codes_str, full_file, month), output=gribfile + '_', options='-R -f nc4') # total precipitation (remove negative values) @@ -546,7 +546,7 @@ class Cmorizer(object): continue new_units = None - cdo_operator = '-selmon,{0}'.format(month) + cdo_operator = '' cdo_operator = self._get_time_average(cdo_operator, frequency, var_code) cdo_operator = self._fix_time_shift(cdo_operator, var_code) diff --git a/environment.yml b/environment.yml index e11e830e..1cccf464 100644 --- a/environment.yml +++ b/environment.yml @@ -8,7 +8,7 @@ dependencies: - iris - netcdf4 - numpy -- cdo +- cdo=1.9.2 - nco - python-cdo - coverage -- GitLab From 2613c4152cdc8977d1f381f924136b82fd0e9b7b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Feb 2018 16:01:37 +0100 Subject: [PATCH 54/70] Added debug info for CDO --- earthdiagnostics/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index ee51684c..8bca05b9 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -42,7 +42,7 @@ class Utils(object): nco = Nco() """An instance of Nco class ready to be used""" - cdo = Cdo(env=os.environ) + cdo = Cdo(env=os.environ, debug=True) """An instance of Cdo class ready to be used""" @staticmethod -- GitLab From 629c07b7de6d677c7cab7b754cca675446891d81 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 21 Feb 2018 16:08:05 +0100 Subject: [PATCH 55/70] Splitted CDO call --- earthdiagnostics/cmorizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index a9b3e7df..ed4ca4c8 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -302,7 +302,7 @@ class Cmorizer(object): if 228 in codes: Utils.cdo.setcode(228, input='-setmisstoc,0 -setvrange,0,Inf ' - '-add {0}_{{142,143}}.128.nc'.format(gribfile), + '-add {0}_142.128.nc {0}_143.128.nc'.format(gribfile), output='{0}_228.128.nc'.format(gribfile), options='-f nc4') return True -- GitLab From 6115e938a122f1c6aa4752bc353f5f9ed5646ef6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 22 Feb 2018 15:16:05 +0100 Subject: [PATCH 56/70] Added value tests for grib testing --- earthdiagnostics/cmorizer.py | 25 +++++---- test/integration/test_cmorizer.py | 84 ++++++++++++++++++++++++------- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index ed4ca4c8..3f5c12b3 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -64,6 +64,10 @@ class Cmorizer(object): 'time_counter_bounds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': self.lat_name, 'nav_lon': self.lon_name, 'x': 'i', 'y': 'j'} + @property + def ICM_path(self): + return os.path.join(self.config.scratch_dir, 'ICM') + def cmorize_ocean(self): """Cmorize ocean files from MMO files""" if not self.cmor.ocean: @@ -310,13 +314,14 @@ class Cmorizer(object): Log.info('No requested codes found in {0} file'.format(grid)) return False finally: - Utils.remove_file('ICM') + Utils.remove_file(self.ICM_path) def _get_monthly_grib(self, current_date, gribfile, grid): prev_gribfile = self._get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) if os.path.exists(prev_gribfile): self._merge_grib_files(current_date, prev_gribfile, gribfile) - full_file = 'ICM' + + full_file = self.ICM_path else: full_file = gribfile return full_file @@ -522,18 +527,18 @@ class Cmorizer(object): if var_cmor.domain == ModelingRealms.atmos: Utils.rename_variables(temp, {'depth': 'plev'}, False, True) - @staticmethod - def _merge_grib_files(current_month, prev_gribfile, gribfile): + def _merge_grib_files(self, current_month, prev_gribfile, gribfile): Log.info('Merging data from different files...') - fd = open('rules_files', 'w') + rules_path = os.path.join(self.config.scratch_dir, 'rules_files') + fd = open(rules_path, 'w') fd.write('if (dataDate >= {0.year}{0.month:02}01) {{ write ; }}\n'.format(current_month)) fd.close() # get first timestep for each month from previous file (if possible) - if os.path.exists('ICM'): - os.remove('ICM') - Utils.execute_shell_command('grib_filter -o ICM rules_files ' - '{0} {1}'.format(prev_gribfile, gribfile)) - os.remove('rules_files') + if os.path.exists(self.ICM_path): + os.remove(self.ICM_path) + Utils.execute_shell_command('grib_filter -o {2} {3} ' + '{0} {1}'.format(prev_gribfile, gribfile, self.ICM_path, rules_path)) + os.remove(rules_path) Utils.remove_file(prev_gribfile) def _ungrib_vars(self, gribfile, month, frequency): diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 2cb1af68..30408d38 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -36,6 +36,9 @@ class TestCmorizer(TestCase): def _get_file_path(self, *args, **kwargs): return os.path.join(self.tmp_dir, args[3], '{0[3]}.nc'.format(args)) + def _get_file_path_grib(self, *args, **kwargs): + return os.path.join(self.tmp_dir, args[3], str(args[6]), '{0[3]}.nc'.format(args)) + def setUp(self): self.tmp_dir = tempfile.mkdtemp() @@ -433,17 +436,10 @@ class TestCmorizer(TestCase): coord_data = np.array([0, 1], np.float) folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', 'outputs') - filepath_gg = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), - [142, 143, 129, 169, 180], - month) - filepath_sh = self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [201, 202], - month) - Utils.cdo.copy(input=filepath_gg, output=filepath_gg.replace('.nc', '.grb'), - options='-f grb2') - Utils.cdo.copy(input=filepath_sh, output=filepath_sh.replace('.nc', '.grb'), - options='-f grb2') - os.remove(filepath_gg) - os.remove(filepath_sh) + self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'GG'), [142, 143, 129, 169, 180], + month) + self._create_file_for_grib(coord_data, folder_path, filename.replace('??', 'SH'), [201, 202], + month) def _create_file_for_grib(self, coord_data, folder_path, filename, codes, month): month -= 1 @@ -451,14 +447,16 @@ class TestCmorizer(TestCase): units='degrees_north') lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', units='degrees_east') - time_data = np.array([0, 0.25], np.float) + month * 31 + time_data = np.array([0.25, 0.50], np.float) + month * 31 time = DimCoord(time_data, standard_name='time', long_name='time', var_name='time', units='days since 1990-01-01') vars = [] for code in codes: - var = iris.cube.Cube(np.random.rand(2, 2, 2).astype(np.float), + var = iris.cube.Cube(np.ones((2, 2, 2), np.float) * code, long_name='Variable {}'.format(code), var_name='var{}'.format(code)) + var.data[0, ...] += time_data[0] + var.data[1, ...] += time_data[1] var.add_dim_coord(time, 0) var.add_dim_coord(lat, 1) var.add_dim_coord(lon, 2) @@ -470,13 +468,65 @@ class TestCmorizer(TestCase): os.makedirs(folder_path) file_path = os.path.join(folder_path, filename) iris.save(vars, file_path, zlib=True, local_keys=('table', 'code')) - - return file_path + Utils.cdo.settaxis('1990-0{}-01,06:00,6hour'.format(month + 1), + input=file_path, + output=file_path.replace('.nc', '.grb'), + options='-f grb2') + os.remove(file_path) def test_grib_cmorization(self): self.data_manager.config.experiment.chunk_size = 2 + self.data_manager.get_file_path = self._get_file_path_grib self.create_grib_files('ICM??expid+19900{}.nc', 1) self.create_grib_files('ICM??expid+19900{}.nc', 2) self._test_atmos_cmor() - for var in ('CP', 'EWSS', 'LSP', 'MN2T', 'MX2T', 'SSRD', 'TP', 'Z'): - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, var, '{}.nc'.format(var)))) + variables = { + 'CP': 143, + 'EWSS': 180, + 'LSP': 142, + 'MN2T': 202, + 'MX2T': 201, + 'SSRD': 169, + # 'TP': 228, + 'Z': 129 + } + for var, code in six.iteritems(variables): + self.assertTrue(os.path.isdir(os.path.join(self.tmp_dir, var))) + base_data = np.ones((2, 2, 2), np.float) * code + factor = 1 + month_offsets = np.array((0.375, 31.375)) + daily_offsets = np.array((0.375, 31.375)) + hourly_offsets = np.array((0.25, 0.5, 31.25, 31.5)) + if code == 129: + factor = 9.81 + elif code in (180, 169): + factor = 6 * 3600 + elif code == 228: + factor = 6 * 3600 / 1000 + if code == 201: + month_offsets = np.array((0.5, 31.5)) + daily_offsets = np.array((0.5, 31.5)) + elif code == 202: + month_offsets = np.array((0.25, 31.25)) + daily_offsets = np.array((0.25, 31.25)) + base_data /= factor + month_offsets /= factor + daily_offsets /= factor + hourly_offsets /= factor + + monthly = iris.load_cube(os.path.join(self.tmp_dir, var, 'mon', '{}.nc'.format(var))) + self._test_data(monthly, base_data, month_offsets, var, 'Month') + + daily = iris.load_cube(os.path.join(self.tmp_dir, var, 'day', '{}.nc'.format(var))) + self._test_data(daily, base_data, daily_offsets, var, 'Day') + + hourly = iris.load_cube(os.path.join(self.tmp_dir, var, '6hr', '{}.nc'.format(var))) + self._test_data(hourly, base_data, hourly_offsets, var, 'Hour') + + def _test_data(self, data, base_data, offsets, var, freq): + self.assertEqual(data.coord('time').shape, (len(offsets),)) + for x, offset in enumerate(offsets): + self.assertTrue(np.allclose(data.data[x, ...], base_data + offset), + '{} {} data wrong for {}'.format(freq,x, var)) + + -- GitLab From 6f1eaf98e57bb761f94e862291a9b55e6e62d1b6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 22 Feb 2018 15:46:23 +0100 Subject: [PATCH 57/70] Fixed tests and codacy issues --- earthdiagnostics/cmorizer.py | 13 +++++++------ test/integration/test_cmorizer.py | 4 +--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 3f5c12b3..c4d11378 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -65,7 +65,8 @@ class Cmorizer(object): 'tbnds': 'bnds', 'nav_lat': self.lat_name, 'nav_lon': self.lon_name, 'x': 'i', 'y': 'j'} @property - def ICM_path(self): + def path_icm(self): + """Path to the ICM file""" return os.path.join(self.config.scratch_dir, 'ICM') def cmorize_ocean(self): @@ -314,14 +315,14 @@ class Cmorizer(object): Log.info('No requested codes found in {0} file'.format(grid)) return False finally: - Utils.remove_file(self.ICM_path) + Utils.remove_file(self.path_icm) def _get_monthly_grib(self, current_date, gribfile, grid): prev_gribfile = self._get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) if os.path.exists(prev_gribfile): self._merge_grib_files(current_date, prev_gribfile, gribfile) - full_file = self.ICM_path + full_file = self.path_icm else: full_file = gribfile return full_file @@ -534,10 +535,10 @@ class Cmorizer(object): fd.write('if (dataDate >= {0.year}{0.month:02}01) {{ write ; }}\n'.format(current_month)) fd.close() # get first timestep for each month from previous file (if possible) - if os.path.exists(self.ICM_path): - os.remove(self.ICM_path) + if os.path.exists(self.path_icm): + os.remove(self.path_icm) Utils.execute_shell_command('grib_filter -o {2} {3} ' - '{0} {1}'.format(prev_gribfile, gribfile, self.ICM_path, rules_path)) + '{0} {1}'.format(prev_gribfile, gribfile, self.path_icm, rules_path)) os.remove(rules_path) Utils.remove_file(prev_gribfile) diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 30408d38..e838dd3c 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -527,6 +527,4 @@ class TestCmorizer(TestCase): self.assertEqual(data.coord('time').shape, (len(offsets),)) for x, offset in enumerate(offsets): self.assertTrue(np.allclose(data.data[x, ...], base_data + offset), - '{} {} data wrong for {}'.format(freq,x, var)) - - + '{} {} data wrong for {}'.format(freq, x, var)) -- GitLab From a3d1d7b8bb2b69839f6b7063dbf2956ff08f2c00 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 22 Feb 2018 16:11:27 +0100 Subject: [PATCH 58/70] All grib cmorized variables now have value test --- earthdiagnostics/cmorizer.py | 10 +++++----- test/integration/test_cmorizer.py | 7 ++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index c4d11378..c8aa6d19 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -563,10 +563,10 @@ class Cmorizer(object): if levels: cdo_operator = "{0} -sellevel,{1}".format(cdo_operator, levels) - Utils.execute_shell_command('cdo -t ecmwf setreftime,{0} ' - '{1} {2}_{3}.128.nc ' - '{2}_{3}_{4}.nc'.format(cdo_reftime, cdo_operator, - gribfile, var_code, frequency)) + Utils.cdo.setreftime(cdo_reftime, + input='{2} {0}_{1}.128.nc '.format(gribfile, var_code, cdo_operator), + output='{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency), + options='-t ecmwf') h_var_file = '{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency) handler = Utils.open_cdf(h_var_file) @@ -626,7 +626,7 @@ class Cmorizer(object): elif var_code in (144, 182, 205, 228): # precipitation/evaporation/runoff new_units = "kg m-2 s-1" - cdo_operator = "-mulc,1000 -divc,{0}".format(self.experiment.atmos_timestep * 3600) + cdo_operator = "-mulc,1000 -divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) return cdo_operator, new_units def _merge_and_cmorize_atmos(self, chunk_start, chunk_end, grid, frequency): diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index e838dd3c..51bb3f88 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -487,9 +487,10 @@ class TestCmorizer(TestCase): 'MN2T': 202, 'MX2T': 201, 'SSRD': 169, - # 'TP': 228, + 'TP': 228, 'Z': 129 } + iris.FUTURE.netcdf_promote = True for var, code in six.iteritems(variables): self.assertTrue(os.path.isdir(os.path.join(self.tmp_dir, var))) base_data = np.ones((2, 2, 2), np.float) * code @@ -502,6 +503,10 @@ class TestCmorizer(TestCase): elif code in (180, 169): factor = 6 * 3600 elif code == 228: + base_data = np.ones((2, 2, 2), np.float) * (142 + 143) + month_offsets *= 2 + daily_offsets *= 2 + hourly_offsets *= 2 factor = 6 * 3600 / 1000 if code == 201: month_offsets = np.array((0.5, 31.5)) -- GitLab From a6c1ac5b6187e90b26b0748e10ae05bc675e8ecc Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 22 Feb 2018 17:58:16 +0100 Subject: [PATCH 59/70] Fixed tests for python 2 --- test/integration/test_cmorizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 51bb3f88..13a11132 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -501,13 +501,13 @@ class TestCmorizer(TestCase): if code == 129: factor = 9.81 elif code in (180, 169): - factor = 6 * 3600 + factor = 6 * 3600.0 elif code == 228: base_data = np.ones((2, 2, 2), np.float) * (142 + 143) month_offsets *= 2 daily_offsets *= 2 hourly_offsets *= 2 - factor = 6 * 3600 / 1000 + factor = 6 * 3600.0 / 1000 if code == 201: month_offsets = np.array((0.5, 31.5)) daily_offsets = np.array((0.5, 31.5)) -- GitLab From 2cd1bc1e23d4930a714ed8665beb3f1bc5683be6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 23 Feb 2018 16:28:06 +0100 Subject: [PATCH 60/70] Small improvements --- .gitignore | 1 + earthdiagnostics/cmorizer.py | 4 +--- earthdiagnostics/earthdiags.py | 2 +- earthdiagnostics/utils.py | 26 +++++++------------------- environment.yml | 1 + run_test.py | 2 ++ 6 files changed, 13 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 733c1474..7352c474 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ test/report/* .coverage htmlcov .pytest_cache +prof diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index c8aa6d19..7770f86f 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -363,7 +363,7 @@ class Cmorizer(object): os.remove(filename) return - Utils.convert2netcdf4(filename) + # Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) Utils.rename_variables(filename, self.alt_coord_names, False, True) handler = Utils.open_cdf(filename) @@ -554,9 +554,7 @@ class Cmorizer(object): cdo_operator = '' cdo_operator = self._get_time_average(cdo_operator, frequency, var_code) - cdo_operator = self._fix_time_shift(cdo_operator, var_code) - cdo_operator, new_units = self._change_units(cdo_operator, new_units, var_code) levels = self.config.cmor.get_levels(frequency, var_code) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index ab3a6e00..a16c1ce1 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -104,7 +104,7 @@ class EarthDiags(object): if Log.console_handler.level <= Log.DEBUG: Utils.cdo.debug = True - Utils.nco.debug = False # This is due to a bug in nco. Must change when it's solved + Utils.nco.debug = True Utils.cdo.CDO = find_executable('cdo') diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 8bca05b9..e5325c35 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -42,7 +42,7 @@ class Utils(object): nco = Nco() """An instance of Nco class ready to be used""" - cdo = Cdo(env=os.environ, debug=True) + cdo = Cdo(env=os.environ) """An instance of Cdo class ready to be used""" @staticmethod @@ -554,24 +554,12 @@ class Utils(object): @classmethod def _is_compressed_netcdf4(cls, filetoconvert): - is_compressed = True - handler = Utils.open_cdf(filetoconvert) - if not handler.file_format == 'NETCDF4': - is_compressed = False - handler.close() - else: - handler.close() - ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) - ncdump_result = ncdump_result[0].replace('\t', '').split('\n') - for var in handler.variables: - if not '{0}:_DeflateLevel = 4 ;'.format(var) in ncdump_result: - is_compressed = False - break - if not '{0}:_Shuffle = "true" ;'.format(var) in ncdump_result: - is_compressed = False - break - return is_compressed - + ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) + ncdump_result = ncdump_result[0].replace('\t', '').split('\n') + if any(':_Shuffle = "true"' in line for line in ncdump_result) and \ + any(':_DeflateLevel = 4' in line for line in ncdump_result): + return True + return False # noinspection PyPep8Naming @staticmethod diff --git a/environment.yml b/environment.yml index 1cccf464..a642f28b 100644 --- a/environment.yml +++ b/environment.yml @@ -30,3 +30,4 @@ dependencies: - nco - exrex - xxhash + - pytest-profiling diff --git a/run_test.py b/run_test.py index a1689be6..cec7ec80 100644 --- a/run_test.py +++ b/run_test.py @@ -18,5 +18,7 @@ errno = pytest.main([ '--cov-report=term', '--cov-report=html:{}/coverage_html'.format(report_dir), '--cov-report=xml:{}/coverage.xml'.format(report_dir), + '--profile', + '--profile-svg', ]) sys.exit(errno) -- GitLab From 05c0d63c0c3a4a6fe65ad5712bc3df26cf150523 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 23 Feb 2018 18:30:23 +0100 Subject: [PATCH 61/70] Improved workmanager tests --- earthdiagnostics/work_manager.py | 7 +- test/integration/test_cmorizer.py | 6 +- test/unit/test_workmanager.py | 120 +++++++++++++++++++++++++++++- 3 files changed, 122 insertions(+), 11 deletions(-) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 3e15dad5..a1ef5584 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -83,6 +83,9 @@ class WorkManager(object): self.uploader = ThreadPoolExecutor(self.config.parallel_uploads) self.executor = ThreadPoolExecutor(self.threads) + self.lock = threading.Lock() + self.lock.acquire() + for job in self.jobs: job.request_data() job.declare_data_generated() @@ -101,10 +104,6 @@ class WorkManager(object): self.downloader.submit(file_object) self.downloader.start() - self.lock = threading.Lock() - self.lock.acquire() - - # self.check_completion() self.lock.acquire() self.downloader.shutdown() diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 13a11132..0f5fb8ea 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -84,7 +84,6 @@ class TestCmorizer(TestCase): os.makedirs(self.data_manager.config.scratch_dir) def create_ocean_files(self, filename, tar_name, gzip=False, backup=False): - iris.FUTURE.netcdf_no_unlimited = True coord_data = np.array([1, 2], np.float) lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', units='degrees_north') @@ -150,7 +149,7 @@ class TestCmorizer(TestCase): with self.assertLogs(log.Log.log) as cmd: cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_atmos() - self.assertLogs([record for record in cmd.records if + self.assertTrue([record for record in cmd.records if record.message == 'Skipping atmosphere cmorization due to configuration']) else: cmorizer = Cmorizer(self.data_manager, '19900101', 0) @@ -353,7 +352,6 @@ class TestCmorizer(TestCase): cmorizer.cmorize_atmos() def create_mma_files(self, filename, tar_name, gzip=False): - iris.FUTURE.netcdf_no_unlimited = True coord_data = np.array([0, 1], np.float) folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', 'outputs') @@ -432,7 +430,6 @@ class TestCmorizer(TestCase): def create_grib_files(self, filename, month): filename = filename.format(month) - iris.FUTURE.netcdf_no_unlimited = True coord_data = np.array([0, 1], np.float) folder_path = os.path.join(self.data_manager.config.data_dir, 'expid', 'original_files', '19900101', 'member', 'outputs') @@ -490,7 +487,6 @@ class TestCmorizer(TestCase): 'TP': 228, 'Z': 129 } - iris.FUTURE.netcdf_promote = True for var, code in six.iteritems(variables): self.assertTrue(os.path.isdir(os.path.join(self.tmp_dir, var))) base_data = np.ones((2, 2, 2), np.float) * code diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 6052f226..d1efd7dc 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -1,8 +1,11 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.work_manager import Downloader, WorkManager -from earthdiagnostics.diagnostic import Diagnostic from mock import Mock +import six +from earthdiagnostics.work_manager import Downloader, WorkManager +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticStatus +from earthdiagnostics.datafile import DataFile, StorageStatus, LocalStatus +from bscearth.utils import log class TestDownloader(TestCase): @@ -29,11 +32,28 @@ class TestDownloader(TestCase): datafile.download.assert_called_once() +class MockFile(DataFile): + + def download(self): + self.local_status = LocalStatus.READY + + def upload(self): + self.storage_status = StorageStatus.READY + + def clean_local(self): + pass + + class TestWorkManager(TestCase): def setUp(self): self.config = Mock() self.data_manager = Mock() + self.config.max_cores = 1 + self.config.parallel_uploads = 1 + self.data_manager.requested_files = {} + self.data_manager.request_chunk = self._request_chunk + self.data_manager.declare_chunk = self._declare_chunk self.work_manager = WorkManager(self.config, self.data_manager) def test_prepare_job_list(self): @@ -51,3 +71,99 @@ class TestWorkManager(TestCase): self.config.get_commands.return_value = ['diag1', 'baddiag', 'diag1,badoption'] Diagnostic.register(Diag1) self.work_manager.prepare_job_list() + + def test_run(self): + self.config.max_cores = -1 + + class EmptyDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Null diag' + + def compute(self): + pass + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + self.work_manager.jobs = [EmptyDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) + + def test_failed_run(self): + class FailDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Fail diag' + + def compute(self): + raise Exception('Must fail') + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + self.work_manager.jobs = [FailDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.FAILED) + + def _request_chunk(self, *args, **kwargs): + req = MockFile() + req.storage_status = StorageStatus.READY + req.local_status = LocalStatus.PENDING + req.remote_file = 'requested' + self.data_manager.requested_files[req.remote_file] = req + return req + + def _declare_chunk(self, *args, **kwargs): + req = MockFile() + req.remote_file = 'generated' + self.data_manager.requested_files[req.remote_file] = req + req.storage_status = StorageStatus.PENDING + req.local_status = LocalStatus.NOT_REQUESTED + return req + + def test_run_data(self): + self.data_manager.config.max_cores = -1 + + class DataDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Data diag' + + def compute(self): + self.declared.local_status = LocalStatus.READY + + def request_data(self): + self.request = self.request_chunk(None, None, None, None, None) + + def declare_data_generated(self): + self.declared = self.declare_chunk(None, None, None, None, None) + + self.work_manager.jobs = [DataDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) + + def test_run_empty(self): + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.work_manager.jobs = [] + self.work_manager.run() + self.assertTrue([record for record in cmd.records if + record.message == 'No diagnostics to run']) + else: + self.work_manager.run() -- GitLab From eacde3226747da209ef34b0cb911d4f96f64d4d2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 23 Feb 2018 18:38:00 +0100 Subject: [PATCH 62/70] Fixed tests in python 2 --- test/unit/test_workmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index d1efd7dc..8686e7bb 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -159,9 +159,9 @@ class TestWorkManager(TestCase): self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) def test_run_empty(self): + self.work_manager.jobs = [] if six.PY3: with self.assertLogs(log.Log.log) as cmd: - self.work_manager.jobs = [] self.work_manager.run() self.assertTrue([record for record in cmd.records if record.message == 'No diagnostics to run']) -- GitLab From 0d10ff9dda89f5a3be05f491b6dc82c559c75e23 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 26 Feb 2018 17:33:47 +0100 Subject: [PATCH 63/70] Finsihed uploader tests --- earthdiagnostics/work_manager.py | 13 +-- test/unit/test_workmanager.py | 156 ++++++++++++++++++++++++++++++- 2 files changed, 162 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index a1ef5584..dc969b42 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -361,8 +361,9 @@ class Downloader(object): return time.sleep(0.01) continue - self._downloads[:100].sort(key=cmp_to_key(Downloader._prioritize)) - datafile = self._downloads[0] + downloads = self._downloads[:100] + downloads.sort(key=cmp_to_key(Downloader._prioritize)) + datafile = downloads[0] self._downloads.remove(datafile) datafile.download() except Exception as ex: @@ -382,7 +383,7 @@ class Downloader(object): def _prioritize(datafile1, datafile2): waiting = Downloader._suscribers_waiting(datafile1) - Downloader._suscribers_waiting(datafile2) if waiting: - return -waiting + return waiting suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) if suscribers: @@ -392,12 +393,12 @@ class Downloader(object): if datafile2.size is None: return 0 else: - return -1 + return 1 elif datafile2.size is None: - return 1 + return -1 size = datafile1.size - datafile2.size if size: - return -size + return size return 0 def shutdown(self): diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 8686e7bb..57377aea 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -28,9 +28,163 @@ class TestDownloader(TestCase): self.downloader.submit(datafile) self.downloader.start() self.downloader.shutdown() - datafile.download.assert_called_once() + def test_download(self): + datafile = Mock() + + def _download(): + raise Exception('Error') + datafile.download = _download + self.downloader.submit(datafile) + if six.PY3: + with self.assertLogs(log.Log.log, log.Log.CRITICAL): + self.downloader.start() + self.downloader.shutdown() + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_smaller_first(self): + small_file = self._create_datafile_mock(size=1) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=10) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 1', + 'INFO:bscearth.utils:Suscribers: () Size: 10']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_not_none_first(self): + small_file = self._create_datafile_mock(size=None) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=10) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 10', + 'INFO:bscearth.utils:Suscribers: () Size: None']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_not_none_second(self): + small_file = self._create_datafile_mock(size=1) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=None) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 1', + 'INFO:bscearth.utils:Suscribers: () Size: None']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_both_none(self): + small_file = self._create_datafile_mock(size=None) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=None) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: None', + 'INFO:bscearth.utils:Suscribers: () Size: None']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_can_not_order(self): + small_file = self._create_datafile_mock(size=1) + self.downloader.submit(small_file) + large_file = self._create_datafile_mock(size=1) + self.downloader.submit(large_file) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: () Size: 1', + 'INFO:bscearth.utils:Suscribers: () Size: 1']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_more_suscribers_first(self): + no_suscribers = self._create_datafile_mock() + self.downloader.submit(no_suscribers) + suscriber = Mock() + one_suscriber = self._create_datafile_mock(suscribers=(suscriber, )) + self.downloader.submit(one_suscriber) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: ({0},) Size: 0'.format(str(suscriber)), + 'INFO:bscearth.utils:Suscribers: () Size: 0']) + else: + self.downloader.start() + self.downloader.shutdown() + + def test_download_more_suscribers_waiting_first(self): + + class StatusDiag(Diagnostic): + def __init__(self, data_manager, pending): + super(StatusDiag, self).__init__(data_manager) + self.pending = pending + + def __str__(self): + return str(self.pending) + + def pending_requests(self): + return self.pending + + suscriber = StatusDiag(Mock(), 1) + suscriber.status = DiagnosticStatus.WAITING + one_waiting_suscriber = self._create_datafile_mock(suscribers=(suscriber,)) + self.downloader.submit(one_waiting_suscriber) + suscriber = StatusDiag(Mock(), 2) + suscriber.status = DiagnosticStatus.FAILED + one_failed_suscriber = self._create_datafile_mock(suscribers=(suscriber, )) + self.downloader.submit(one_failed_suscriber) + if six.PY3: + with self.assertLogs(log.Log.log) as cmd: + self.downloader.start() + self.downloader.shutdown() + self.assertListEqual(cmd.output, + ['INFO:bscearth.utils:Suscribers: (2,) Size: 0', + 'INFO:bscearth.utils:Suscribers: (1,) Size: 0']) + else: + self.downloader.start() + self.downloader.shutdown() + + def _create_datafile_mock(self, size=0, suscribers=tuple()): + data_mock = Mock() + data_mock.size = size + data_mock.suscribers = suscribers + + def _download(): + log.Log.info('Suscribers: {0.suscribers} Size: {0.size}'.format(data_mock)) + data_mock.download = _download + + return data_mock + class MockFile(DataFile): -- GitLab From 51462177841f252e006d44ba927a1d90ff52e0c5 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 26 Feb 2018 18:10:52 +0100 Subject: [PATCH 64/70] added test for skipped jobs --- earthdiagnostics/work_manager.py | 7 ++--- test/unit/test_workmanager.py | 54 ++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index dc969b42..daec4061 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -90,13 +90,12 @@ class WorkManager(object): job.request_data() job.declare_data_generated() job.subscribe(self, self._job_status_changed) - job.check_is_ready() - - if self.config.skip_diags_done: - for job in self.jobs: + if self.config.skip_diags_done: if job.can_skip_run(): Log.info('Diagnostic {0} already done. Skipping !', job) job.status = DiagnosticStatus.COMPLETED + continue + job.check_is_ready() for file_object in self.data_manager.requested_files.values(): file_object.subscribe(self, self._file_object_status_changed) diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 57377aea..030fb3ea 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -250,6 +250,60 @@ class TestWorkManager(TestCase): for diag in self.work_manager.jobs: self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) + def test_diag_can_be_skipped(self): + self.config.skip_diags_done = True + + class SkippedDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Skipped diag' + + def compute(self): + raise Exception('Diagnostic should not be computed') + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + def can_skip_run(self): + return True + + self.work_manager.jobs = [SkippedDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.COMPLETED) + + def test_diag_can_be_skipped_but_no(self): + self.config.skip_diags_done = False + + class SkippedDiag(Diagnostic): + alias = 'diag1' + + def __str__(self): + return 'Skipped diag' + + def compute(self): + raise Exception('Diagnostic should not be computed') + + def request_data(self): + pass + + def declare_data_generated(self): + pass + + def can_skip_run(self): + return True + + self.work_manager.jobs = [SkippedDiag(self.data_manager)] + self.work_manager.run() + + for diag in self.work_manager.jobs: + self.assertEqual(diag.status, DiagnosticStatus.FAILED) + def test_failed_run(self): class FailDiag(Diagnostic): alias = 'diag1' -- GitLab From a394f703620ec1686cca0ec07156da5a04a4e92f Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 26 Feb 2018 18:39:03 +0100 Subject: [PATCH 65/70] Added test for clean method in earthdiags --- earthdiagnostics/earthdiags.py | 18 +++++++++------- test/unit/test_earthdiags.py | 38 +++++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index a16c1ce1..338fc346 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -45,19 +45,20 @@ class EarthDiags(object): else: version = pkg_resources.require("earthdiagnostics")[0].version - def __init__(self, config_file): - Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) + def __init__(self): + self.time = dict() + self.data_manager = None + self.threads = None + self.had_errors = False self.config = Config() + + def read_config(self, config_file): + Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) self.config.parse(config_file) os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' - TempFile.scratch_folder = self.config.scratch_dir cdftools.path = self.config.cdftools_path self._create_dic_variables() - self.time = dict() - self.data_manager = None - self.threads = None - self.had_errors = False Log.debug('Diags ready') Log.info('Running diags for experiment {0}, startdates {1}, members {2}', self.config.experiment.expid, self.config.experiment.startdates, self.config.experiment.members) @@ -116,7 +117,8 @@ class EarthDiags(object): Log.critical('Configuration file {0} can not be found', config_file_path) return False try: - diags = EarthDiags(config_file_path) + diags = EarthDiags() + diags.read_config(config_file_path) if args.clean: result = diags.clean() elif args.report: diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py index 95468b71..02242aa4 100644 --- a/test/unit/test_earthdiags.py +++ b/test/unit/test_earthdiags.py @@ -1,15 +1,33 @@ # coding=utf-8 from unittest import TestCase +import os +import tempfile from earthdiagnostics.earthdiags import EarthDiags -# -# class TestEarthDiags(TestCase): -# -# def setUp(self): -# self.earthdiags = EarthDiags('path/to/conf') -# -# self.earthdiags.config() -# -# def test_clean(self): -# self.earthdiags.parse_args() + +class TestEarthDiags(TestCase): + + def setUp(self): + self.earthdiags = EarthDiags() + + def test_clean(self): + self.earthdiags.config.scratch_dir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.earthdiags.config.scratch_dir, 'extra_folder')) + + self.assertTrue(os.path.isdir(self.earthdiags.config.scratch_dir)) + self.earthdiags.clean() + self.assertFalse(os.path.isdir(self.earthdiags.config.scratch_dir)) + + def test_clean_link(self): + temp_folder = tempfile.mkdtemp() + scratch_dir = tempfile.mktemp() + os.symlink(temp_folder, scratch_dir) + os.makedirs(os.path.join(temp_folder, 'extra_folder')) + self.earthdiags.config.scratch_dir = scratch_dir + + self.assertTrue(os.path.islink(self.earthdiags.config.scratch_dir)) + self.assertTrue(os.path.isdir(temp_folder)) + self.earthdiags.clean() + self.assertFalse(os.path.islink(self.earthdiags.config.scratch_dir)) + self.assertFalse(os.path.isdir(temp_folder)) -- GitLab From 9ef8294112b0b7dbb645f76356e5500df91ef9d9 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 27 Feb 2018 10:09:45 +0100 Subject: [PATCH 66/70] Added test for read config --- earthdiagnostics/earthdiags.py | 11 ++++++++ test/unit/test_earthdiags.py | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 338fc346..1652ac09 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -53,6 +53,17 @@ class EarthDiags(object): self.config = Config() def read_config(self, config_file): + """ + Read config file and initialize earthdiagnostics + + Parameters + ---------- + config_file + + Returns + ------- + + """ Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) self.config.parse(config_file) os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py index 02242aa4..5693ae9d 100644 --- a/test/unit/test_earthdiags.py +++ b/test/unit/test_earthdiags.py @@ -4,6 +4,9 @@ import os import tempfile from earthdiagnostics.earthdiags import EarthDiags +from earthdiagnostics.utils import TempFile +from earthdiagnostics import cdftools +from mock import Mock class TestEarthDiags(TestCase): @@ -31,3 +34,47 @@ class TestEarthDiags(TestCase): self.earthdiags.clean() self.assertFalse(os.path.islink(self.earthdiags.config.scratch_dir)) self.assertFalse(os.path.isdir(temp_folder)) + + def test_read_config(self): + self.earthdiags.config = Mock() + self.earthdiags.config.scratch_dir = 'scratch_path' + self.earthdiags.config.cdftools_path = 'cdftools_path' + self.earthdiags.config.data_convention = 'SPECS' + + self.earthdiags.read_config('config/path') + + self.earthdiags.config.parse.assert_called_once_with('config/path') + self.assertEqual(os.environ['HDF5_USE_FILE_LOCKING'], 'FALSE') + self.assertEqual(self.earthdiags.config.scratch_dir, TempFile.scratch_folder) + self.assertEqual(self.earthdiags.config.cdftools_path, cdftools.path) + self.assertDictEqual(self.earthdiags.dic_variables, + {'nav_lat': 'lat', + 'nav_lev': 'lev', + 'nav_lon': 'lon', + 't': 'time', + 'time_counter': 'time', + 'x': 'i', + 'y': 'j', + 'z': 'lev'}) + + def test_read_config_primavera(self): + self.earthdiags.config = Mock() + self.earthdiags.config.scratch_dir = 'scratch_path' + self.earthdiags.config.cdftools_path = 'cdftools_path' + self.earthdiags.config.data_convention = 'PRIMAVERA' + + self.earthdiags.read_config('config/path') + + self.earthdiags.config.parse.assert_called_once_with('config/path') + self.assertEqual(os.environ['HDF5_USE_FILE_LOCKING'], 'FALSE') + self.assertEqual(self.earthdiags.config.scratch_dir, TempFile.scratch_folder) + self.assertEqual(self.earthdiags.config.cdftools_path, cdftools.path) + self.assertDictEqual(self.earthdiags.dic_variables, + {'nav_lat': 'latitude', + 'nav_lev': 'lev', + 'nav_lon': 'longitude', + 't': 'time', + 'time_counter': 'time', + 'x': 'i', + 'y': 'j', + 'z': 'lev'}) -- GitLab From 5baa87feef53e5715b94252a3e8c4eaf92dcba20 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 27 Feb 2018 11:16:22 +0100 Subject: [PATCH 67/70] Added test for arguments parsing --- bin/earthdiags | 2 +- earthdiagnostics/config.py | 7 ++++- earthdiagnostics/earthdiags.py | 35 ++++++++++++++--------- test/unit/test_config.py | 8 +++++- test/unit/test_earthdiags.py | 52 +++++++++++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 17 deletions(-) diff --git a/bin/earthdiags b/bin/earthdiags index 174528e3..200086ee 100644 --- a/bin/earthdiags +++ b/bin/earthdiags @@ -18,7 +18,7 @@ def main(): """ Entry point for the Earth Diagnostics """ - if not EarthDiags.parse_args(): + if not EarthDiags.parse_args(sys.argv[1:]): os._exit(1) os._exit(0) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 7ef15246..734f94db 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -6,6 +6,7 @@ import six from bscearth.utils.config_parser import ConfigParser from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str, add_years, add_months, add_days from bscearth.utils.log import Log +import bscearth.utils.path from earthdiagnostics import cdftools from earthdiagnostics.frequency import Frequency, Frequencies @@ -109,9 +110,13 @@ class Config(object): ---------- path: str """ + config_file_path = bscearth.utils.path.expand_path(path) + if not os.path.isfile(config_file_path): + Log.critical('Configuration file {0} can not be found', config_file_path) + raise ValueError('Configuration file {0} can not be found'.format(config_file_path)) parser = ConfigParser() parser.optionxform = str - parser.read(path) + parser.read(config_file_path) # Read diags config self.data_adaptor = parser.get_choice_option('DIAGNOSTICS', 'DATA_ADAPTOR', ('CMOR', 'THREDDS', 'OBSRECON'), diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 1652ac09..86a5e6bc 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -3,6 +3,7 @@ """Entry point for EarthDiagnostics""" import argparse import os +import sys import shutil import tempfile from distutils.spawn import find_executable @@ -75,7 +76,7 @@ class EarthDiags(object): self.config.experiment.startdates, self.config.experiment.members) @staticmethod - def parse_args(): + def parse_args(args): """ Entry point for the Earth Diagnostics. @@ -104,13 +105,9 @@ class EarthDiags(object): parser.add_argument('-f', '--configfile', default='diags.conf', type=str) - args = parser.parse_args() + args = parser.parse_args(args) if args.doc: - Log.info('Opening documentation...') - doc_path = os.path.join('http://earthdiagnostics.readthedocs.io/en/latest') - Utils.execute_shell_command(('xdg-open', doc_path)) - Log.result('Documentation opened!') - return True + return EarthDiags.open_documentation() Log.set_console_level(args.logconsole) Log.set_file_level(args.logfile) @@ -123,13 +120,9 @@ class EarthDiags(object): if args.logfilepath: Log.set_file(bscearth.utils.path.expand_path(args.logfilepath)) - config_file_path = bscearth.utils.path.expand_path(args.configfile) - if not os.path.isfile(config_file_path): - Log.critical('Configuration file {0} can not be found', config_file_path) - return False try: diags = EarthDiags() - diags.read_config(config_file_path) + diags.read_config(args.configfile) if args.clean: result = diags.clean() elif args.report: @@ -143,6 +136,22 @@ class EarthDiags(object): return result + @staticmethod + def open_documentation(): + """ + Open Earthdiagnostics online documentation + + Returns + ------- + bool: + True if successful + """ + Log.info('Opening documentation...') + doc_path = os.path.join('http://earthdiagnostics.readthedocs.io/en/latest') + Utils.execute_shell_command(('xdg-open', doc_path)) + Log.result('Documentation opened!') + return True + def _create_dic_variables(self): self.dic_variables = dict() self.dic_variables['x'] = 'i' @@ -427,7 +436,7 @@ class EarthDiags(object): def main(): """Main for earthdiagnostics""" - if not EarthDiags.parse_args(): + if not EarthDiags.parse_args(sys.argv[1:]): exit(1) diff --git a/test/unit/test_config.py b/test/unit/test_config.py index eb9dfd6a..8f7a610d 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -483,7 +483,8 @@ class TestConfig(TestCase): with mock.patch('earthdiagnostics.config.ExperimentConfig', new=mock_new_exp): with mock.patch('earthdiagnostics.config.CMORConfig'): with mock.patch('earthdiagnostics.config.THREDDSConfig'): - config.parse('path') + with mock.patch('os.path.isfile'): + config.parse('path') def test_diags(self): config = Config() @@ -491,6 +492,11 @@ class TestConfig(TestCase): self._parse(config) self.assertEqual(config.get_commands(), (['diag1', 'diag2,opt1,opt2'])) + def test_file_not_found(self): + config = Config() + with self.assertRaises(ValueError): + config.parse('path') + def test_parse(self): config = Config() self._parse(config) diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py index 5693ae9d..c1f3a0cc 100644 --- a/test/unit/test_earthdiags.py +++ b/test/unit/test_earthdiags.py @@ -4,8 +4,11 @@ import os import tempfile from earthdiagnostics.earthdiags import EarthDiags -from earthdiagnostics.utils import TempFile +from earthdiagnostics.utils import TempFile, Utils from earthdiagnostics import cdftools +from bscearth.utils.log import Log + +import mock from mock import Mock @@ -78,3 +81,50 @@ class TestEarthDiags(TestCase): 'x': 'i', 'y': 'j', 'z': 'lev'}) + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.open_documentation') + def test_parse_args_open_doc(self, open_doc): + EarthDiags.parse_args(['--doc']) + open_doc.assert_called_once() + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.clean') + def test_parse_args_clean(self, clean, read_config): + EarthDiags.parse_args(['--clean']) + clean.assert_called_once() + read_config.assert_called_once_with('diags.conf') + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.report') + def test_parse_args_report(self, report, read_config): + EarthDiags.parse_args(['--report']) + report.assert_called_once() + read_config.assert_called_once_with('diags.conf') + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.run') + def test_parse_args_run(self, run, read_config): + EarthDiags.parse_args([]) + run.assert_called_once() + read_config.assert_called_once_with('diags.conf') + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.run') + def test_log_levels(self, run, read_config): + log_file = tempfile.mktemp() + for level in ('EVERYTHING', 'DEBUG', 'INFO', 'RESULT', 'USER_WARNING', + 'WARNING', 'ERROR', 'CRITICAL', 'NO_LOG'): + EarthDiags.parse_args(['-lc', level, '-lf', level, '-log', log_file]) + self.assertEqual(getattr(Log, level), Log.console_handler.level) + self.assertEqual(getattr(Log, level), Log.file_handler.level) + if os.path.isfile(log_file): + os.remove(log_file) + + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.read_config') + @mock.patch('earthdiagnostics.earthdiags.EarthDiags.run') + def test_log_cdo_nco(self, run, read_config): + EarthDiags.parse_args(['-lc', 'DEBUG']) + run.assert_called_once() + read_config.assert_called_once_with('diags.conf') + self.assertTrue(Utils.cdo.debug) + self.assertTrue(Utils.nco.debug) -- GitLab From 5f3ae98c944d6adbf32b0709d75b1c3182612af4 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 2 Mar 2018 13:58:08 +0100 Subject: [PATCH 68/70] Updated siasiesiv to use pure python and allow multiple basins definitions --- diags.conf | 3 +- earthdiagnostics/constants.py | 6 ++ earthdiagnostics/datafile.py | 7 +- earthdiagnostics/datamanager.py | 2 +- earthdiagnostics/diagnostic.py | 27 +++++- earthdiagnostics/ocean/siasiesiv.py | 138 ++++++++++++++++------------ earthdiagnostics/utils.py | 9 +- 7 files changed, 125 insertions(+), 67 deletions(-) diff --git a/diags.conf b/diags.conf index f5db5eba..64dc8269 100644 --- a/diags.conf +++ b/diags.conf @@ -92,7 +92,8 @@ OCEAN_TIMESTEP = 3 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment EXPID = mswep -STARTDATES = {19970101,20161231,1D} +# STARTDATES = {19970101,20161231,1D} +STARTDATES = 19970322 20010425 MEMBERS = 0 MEMBER_DIGITS = 1 CHUNK_SIZE = 1 diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index a244ca5c..e4679e74 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -26,6 +26,12 @@ class Basin(object): def __str__(self): return self._name + def __repr__(self): + return str(self) + + def __hash__(self): + return hash(str(self)) + @property def name(self): """ diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 23a3a006..8b34b78e 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -620,7 +620,12 @@ class NetCDFFile(DataFile): if not self.local_file: self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) - Utils.copy_file(self.remote_file, self.local_file) + try: + Utils.copy_file(self.remote_file, self.local_file, retrials=1) + except Utils.CopyException: + Utils.get_file_hash(self.remote_file, use_stored=False, save=True) + Utils.copy_file(self.remote_file, self.local_file, retrials=2) + if self.data_convention == 'meteofrance': Log.debug('Converting variable names from meteofrance convention') alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index ebf6e8a8..1c9593ba 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -69,7 +69,7 @@ class DataManager(object): def _apply_fxh(folder_name, timestep, frequency=None): is_base_frequency = frequency is not None and frequency.frequency.endswith('hr') if not is_base_frequency and timestep > 0: - return '{0}_f{1}h'.format(folder_name, timestep) + return '{0}_s0-{1}h'.format(folder_name, timestep) return folder_name def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 252c1255..b1cc4980 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -749,6 +749,32 @@ class DiagnosticFrequencyOption(DiagnosticOption): return Frequency.parse(self._check_default(option_value)) +class DiagnosticBasinListOption(DiagnosticOption): + """Class to parse list of basins options""" + + def parse(self, option_value): + """ + Parse option value + + Parameters + ---------- + option_value: str + + Returns + ------- + Basin + + """ + option_value = self._check_default(option_value) + basins = [] + for value in option_value.split(':'): + basin = Basins().parse(value) + if basin is None: + raise DiagnosticOptionError('Basin {0} not recognized'.format(value)) + basins.append(basin) + return basins + + class DiagnosticBasinOption(DiagnosticOption): """Class to parse basin options""" @@ -771,7 +797,6 @@ class DiagnosticBasinOption(DiagnosticOption): raise DiagnosticOptionError('Basin {0} not recognized'.format(value)) return basin - class DiagnosticComplexStrOption(DiagnosticOption): """ Class to parse complex string options diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 53b1c728..f2434cd3 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,13 +1,16 @@ # coding=utf-8 """Compute the sea ice extent , area and volume in both hemispheres or a specified region""" import os +import six -import netCDF4 +import iris +import iris.analysis +import iris.coords import numpy as np from bscearth.utils.log import Log from earthdiagnostics.constants import Basins -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticBoolOption +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinListOption, DiagnosticBoolOption from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import Utils, TempFile @@ -27,7 +30,7 @@ class Siasiesiv(Diagnostic): chunk: init domain: ModellingRealm variable: str - basin: Basin + basin: list of Basin mask: numpy.array omit_vol: bool """ @@ -39,22 +42,26 @@ class Siasiesiv(Diagnostic): e2t = None gphit = None - def __init__(self, data_manager, startdate, member, chunk, basin, mask, var_manager, omit_vol): + def __init__(self, data_manager, startdate, member, chunk, masks, var_manager, omit_vol): Diagnostic.__init__(self, data_manager) - self.basin = basin self.startdate = startdate self.member = member self.chunk = chunk - self.mask = mask + self.masks = masks self.generated = {} self.var_manager = var_manager self.omit_volume = omit_vol self.sic_varname = self.var_manager.get_variable('sic').short_name self.sit_varname = self.var_manager.get_variable('sit').short_name + self.results = {} + for var in ('siarean', 'siareas', 'sivoln', 'sivols', 'siextentn', 'siextents'): + self.results[var] = {} + def __str__(self): return 'Siasiesiv Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ - 'Basin: {0.basin} Omit volume: {0.omit_volume}'.format(self) + 'Basins: {1} Omit volume: {0.omit_volume}'.format(self, + ','.join(str(basin) for basin in self.masks.keys())) @classmethod def generate_jobs(cls, diags, options): @@ -67,25 +74,26 @@ class Siasiesiv(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticBasinOption('basin', Basins().Global), + options_available = (DiagnosticBasinListOption('basins', [Basins().Global]), DiagnosticBoolOption('omit_volume', False)) options = cls.process_options(options, options_available) - if options['basin'] is None: - Log.error('Basin not recognized') + if not options['basins']: + Log.error('Basins not recognized') return () - mask = np.asfortranarray(Utils.get_mask(options['basin'])) + masks = {} + for basin in options['basins']: + masks[basin] = Utils.get_mask(basin) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, options['basin'], mask, + job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, masks, diags.config.var_manager, options['omit_volume'])) - mesh_handler = Utils.open_cdf('mesh_hgr.nc') - Siasiesiv.e1t = np.asfortranarray(mesh_handler.variables['e1t'][0, :]) - Siasiesiv.e2t = np.asfortranarray(mesh_handler.variables['e2t'][0, :]) - Siasiesiv.gphit = np.asfortranarray(mesh_handler.variables['gphit'][0, :]) - mesh_handler.close() + + e1t = iris.load_cube('mesh_hgr.nc', 'e1t') + e2t = iris.load_cube('mesh_hgr.nc', 'e2t') + Siasiesiv.area = e1t * e2t return job_list @@ -110,57 +118,65 @@ class Siasiesiv(Diagnostic): self._declare_var('siextentn') def _declare_var(self, var_name): - self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, self.startdate, self.member, - self.chunk, region=self.basin.name) + self.generated[var_name] = {} + for basin in self.masks.keys(): + self.generated[var_name][basin] = self.declare_chunk(ModelingRealms.seaIce, var_name, + self.startdate, self.member,self.chunk, + region=basin.name) def compute(self): """Run the diagnostic""" - import earthdiagnostics.cdftoolspython as cdftoolspython - - sic_handler = Utils.open_cdf(self.sic.local_file) - Utils.convert_units(sic_handler.variables[self.sic_varname], '1.0') - sic = np.asfortranarray(sic_handler.variables[self.sic_varname][:]) - timesteps = sic_handler.dimensions['time'].size - sic_handler.close() + handler = Utils.open_cdf(self.sic.local_file) + handler.variables[self.sic_varname].coordinates = 'time lat lon leadtime time_centered' + handler.close() + sic = iris.load_cube(self.sic.local_file) + sic.convert_units('1.0') + sic *= Siasiesiv.area.data + extent = sic.data > 0.15 - if self.omit_volume: - sit = sic - else: - sit_handler = Utils.open_cdf(self.sit.local_file) - sit = np.asfortranarray(sit_handler.variables[self.sit_varname][:]) - sit_handler.close() + if not self.omit_volume: + handler = Utils.open_cdf(self.sit.local_file) + handler.variables[self.sit_varname].coordinates = 'time lat lon leadtime time_centered' + handler.close() + sit = iris.load_cube(self.sic.local_file) - result = np.empty((8, timesteps)) - for t in range(0, timesteps): - result[:, t] = cdftoolspython.icediag.icediags(Siasiesiv.e1t, Siasiesiv.e2t, self.mask, - Siasiesiv.gphit, sit[t, :], sic[t, :]) + for basin, mask in six.iteritems(self.masks): + self.results['siarean'][basin] = self.sum(sic, mask, north=True) + self.results['siareas'][basin] = self.sum(sic, mask, north=False) - self._extract_variable_and_rename(result[5, :], 'siareas', '10^9 m2') - self._extract_variable_and_rename(result[7, :], 'siextents', '10^9 m2') + if not self.omit_volume: + volume = sic * sit + self.results['sivoln'][basin] = self.sum(volume, mask, north=True) + self.results['sivols'][basin] = self.sum(volume, mask, north=False) - self._extract_variable_and_rename(result[1, :], 'siarean', '10^9 m2') - self._extract_variable_and_rename(result[3, :], 'siextentn', '10^9 m2') + extent_mask = mask * extent + self.results['siextentn'][basin] = self.sum(sic, extent_mask, north=True) + self.results['siextents'][basin] = self.sum(sic, extent_mask, north=False) - if not self.omit_volume: - self._extract_variable_and_rename(result[4, :], 'sivols', '10^9 m3') - self._extract_variable_and_rename(result[0, :], 'sivoln', '10^9 m3') + self.save() - def _extract_variable_and_rename(self, values, cmor_name, units): + def sum(self, data, mask, north=True): + if north: + condition = data.coord('latitude').points > 0 + else: + condition = data.coord('latitude').points < 0 + weights = iris.util.broadcast_to_shape(condition, data.shape, data.coord_dims('latitude')) * mask + return data.collapsed(('latitude', 'longitude'), iris.analysis.SUM, weights=weights) + + def save(self): + for var, basins in six.iteritems(self.results): + for basin, result in six.iteritems(basins): + if result.data.max() == 0: + continue + result.var_name = var + if var.startswith('sivol'): + result.units = 'm^3' + else: + result.units = 'm^2' + result.add_aux_coord(iris.coords.AuxCoord(basin.name, var_name='region')) + self._save_file(result, self.generated[var][basin], basin) + + def _save_file(self, data, generated_file, basin): temp = TempFile.get() - reference_handler = Utils.open_cdf(self.sic.local_file) - os.remove(temp) - handler = netCDF4.Dataset(temp, 'w') - - Utils.copy_variable(reference_handler, handler, 'time', add_dimensions=True) - Utils.copy_variable(reference_handler, handler, 'time_bnds', add_dimensions=True, must_exist=False) - Utils.copy_variable(reference_handler, handler, 'leadtime', add_dimensions=True, must_exist=False) - reference_handler.close() - - new_var = handler.createVariable(cmor_name, float, 'time', fill_value=1.0e20) - new_var.units = units - new_var.short_name = cmor_name - new_var.valid_min = 0.0 - new_var[:] = values - new_var.valid_max = np.max(values) - handler.close() - self.generated[cmor_name].set_local_file(temp) + iris.save(data, temp) + generated_file.set_local_file(temp, region=basin) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index e5325c35..a570f39a 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -79,7 +79,7 @@ class Utils(object): mask_handler = Utils.open_cdf('mask.nc') mask = mask_handler.variables['tmask'][0, 0, :] mask_handler.close() - return mask + return np.squeeze(mask) @staticmethod def setminmax(filename, variable_list): @@ -341,7 +341,7 @@ class Utils(object): while hash_original != hash_destiny: if retrials == 0: - raise Exception('Can not copy {0} to {1}'.format(source, destiny)) + raise Utils.CopyException('Can not copy {0} to {1}'.format(source, destiny)) Log.debug('Copying... {0}', datetime.datetime.now()) shutil.copyfile(source, destiny) Log.debug('Hashing copy ... {0}', datetime.datetime.now()) @@ -865,6 +865,11 @@ class Utils(object): pass + class CopyException(Exception): + """Exception raised when copy fails""" + + pass + class TempFile(object): """Class to manage temporal files""" -- GitLab From ea7e65b2f459455acd1d716d711a17b94c49e207 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 9 Mar 2018 18:06:30 +0100 Subject: [PATCH 69/70] Updated siasiesiv to use pure python and allow multiple basins definitions and multiple minor fixes --- earthdiagnostics/cmorizer.py | 10 ++++-- earthdiagnostics/cmormanager.py | 41 ++++++++++++------------ earthdiagnostics/datafile.py | 48 +++++++++++++---------------- earthdiagnostics/datamanager.py | 2 +- earthdiagnostics/diagnostic.py | 3 +- earthdiagnostics/ocean/siasiesiv.py | 35 ++++++++++++++------- environment.yml | 2 +- launch_diags.sh | 2 +- test/unit/ocean/test_siasiesiv.py | 4 +-- test/unit/test_cmormanager.py | 3 +- test/unit/test_workmanager.py | 5 ++- 11 files changed, 86 insertions(+), 69 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 7770f86f..42099f32 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -5,6 +5,7 @@ import os import shutil import uuid import traceback +import pygrib from datetime import datetime from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months @@ -242,7 +243,13 @@ class Cmorizer(object): for grid in ('SH', 'GG'): Log.info('Processing {0} variables', grid) - if not os.path.exists(self._get_original_grib_path(chunk_start, grid)): + first_grib = self._get_original_grib_path(chunk_start, grid) + if not os.path.exists(first_grib): + continue + var_list = Utils.cdo.showvar(input=first_grib)[0] + codes = {int(var.replace('var', '')) for var in var_list.split()} + if not codes.intersection(self.config.cmor.get_requested_codes()): + Log.info('No requested variables found in {0}. Skipping...', grid) continue self._cmorize_grib_file(chunk_end, chunk_start, grid) except Exception as ex: @@ -343,7 +350,6 @@ class Cmorizer(object): def _get_atmos_timestep(self, gribfile): Log.info('Getting timestep...') - import pygrib grib_handler = pygrib.open(gribfile) dates = set() for mes in grib_handler: diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index b88e7dca..9d2fa84f 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -496,8 +496,25 @@ class CMORManager(DataManager): return for startdate, member in self.experiment.get_member_list(): - if not self._unpack_cmor_files(startdate, member): - self._cmorize_member(startdate, member) + Log.info('Checking data for startdate {0} member {1}', startdate, member) + if not self.config.cmor.force: + cmorized = False + for chunk in range(1, self.experiment.num_chunks + 1): + if not self.config.cmor.chunk_cmorization_requested(chunk): + Log.debug('Skipping chunk {0}', chunk) + continue + if not self.config.cmor.force_untar: + Log.debug('Checking chunk {0}...', chunk) + for domain in (ModelingRealms.atmos, ModelingRealms.ocean, ModelingRealms.seaIce): + if self.is_cmorized(startdate, member, chunk, domain): + Log.debug('Chunk {0} ready', chunk) + continue + if self._unpack_chunk(startdate, member, chunk): + cmorized = True + if cmorized: + Log.info('Startdate {0} member {1} ready', startdate, member) + return + self._cmorize_member(startdate, member) def is_cmorized(self, startdate, member, chunk, domain): """ @@ -571,27 +588,7 @@ class CMORManager(DataManager): Log.result('CMORized startdate {0} member {1}! Elapsed time: {2}\n\n', startdate, member_str, datetime.now() - start_time) - def _unpack_cmor_files(self, startdate, member): - if self.config.cmor.force: - return False - cmorized = False - for chunk in range(1, self.experiment.num_chunks + 1): - if not self.config.cmor.force_untar: - if self.is_cmorized(startdate, member, chunk, ModelingRealms.atmos) or \ - self.is_cmorized(startdate, member, chunk, ModelingRealms.ocean): - cmorized = True - continue - - if self._unpack_chunk(startdate, member, chunk): - cmorized = True - if cmorized: - Log.info('Startdate {0} member {1} ready', startdate, member) - return cmorized - def _unpack_chunk(self, startdate, member, chunk): - if not self.config.cmor.chunk_cmorization_requested(chunk): - return True - filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar.gz') if len(filepaths) > 0: Log.info('Unzipping cmorized data for {0} {1} {2}...', startdate, member, chunk) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 8b34b78e..086273e1 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -6,6 +6,7 @@ import shutil from datetime import datetime import iris +import iris.coords import numpy as np from bscearth.utils.log import Log @@ -238,8 +239,8 @@ class DataFile(Publisher): self.lon_name = 'longitude' self.lat_name = 'latitude' else: - self.lon_name = 'longitude' - self.lat_name = 'latitude' + self.lon_name = 'lon' + self.lat_name = 'lat' Utils.convert2netcdf4(self.local_file) if rename_var: @@ -290,7 +291,7 @@ class DataFile(Publisher): if diagnostic in self._modifiers: self._modifiers.remove(diagnostic) if region is not None: - self.region = region.name + self.region = region else: self.region = None self.local_file = local_file @@ -304,7 +305,7 @@ class DataFile(Publisher): def _correct_metadata(self): handler = Utils.open_cdf(self.local_file) var_handler = handler.variables[self.final_name] - coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name, 'leadtime'}, + coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name, 'leadtime', 'region', 'time_centered'}, set(handler.variables.keys())) var_handler.coordinates = ' '.join(coords) if not self.cmor_var: @@ -400,10 +401,7 @@ class DataFile(Publisher): regions = handler.variables['region'][...].tolist() if len(regions) > 1: ordered_regions = sorted(regions) - print(regions) - print(ordered_regions) new_indexes = [regions.index(region) for region in ordered_regions] - print(new_indexes) for var in handler.variables.values(): if 'region' not in var.dimensions: @@ -424,34 +422,32 @@ class DataFile(Publisher): self._fix_values_metadata(var_type, temp) Utils.nco.ncks(input=temp, output=temp, options=['--mk_rec_dmn region']) - handler = Utils.open_cdf(temp) - var_handler = handler.variables[self.final_name] - if hasattr(var_handler, 'valid_min'): - del var_handler.valid_min - if hasattr(var_handler, 'valid_max'): - del var_handler.valid_max - handler.sync() cubes = iris.load(self.local_file) for cube in cubes: if self.final_name == cube.var_name: - value = cube.data + value = cube break - if isinstance(value, np.ma.MaskedArray): - value = np.ma.getdata(value) - var_region = handler.variables['region'] - basin_index = np.where(var_region[:] == self.region) - if len(basin_index[0]) == 0: - var_region[var_region.shape[0]] = self.region - basin_index = var_region.shape[0] - 1 - else: - basin_index = basin_index[0][0] - handler.variables[self.final_name][..., basin_index] = np.multiply(np.ones(value.shape), value) - handler.close() + for index_region, region in enumerate(value.coord('region').points): + handler = Utils.open_cdf(temp) + region_slice = value.data[index_region, ...] + original_regions = handler.variables['region'][...] + var = handler.variables[self.final_name] + if region in original_regions: + var[np.where(original_regions == region)[0][0], ...] = region_slice + else: + var[original_regions.shape[0], ...] = region_slice + handler.variables[-1] = region + handler.close() + + # handler.close() Utils.move_file(temp, self.local_file) def _add_region_dimension_to_var(self): handler = Utils.open_cdf(self.local_file) + if 'region' in handler.variables: + handler.close() + return handler.createDimension('region') var_region = handler.createVariable('region', str, 'region') var_region[0] = self.region diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 1c9593ba..ebf6e8a8 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -69,7 +69,7 @@ class DataManager(object): def _apply_fxh(folder_name, timestep, frequency=None): is_base_frequency = frequency is not None and frequency.frequency.endswith('hr') if not is_base_frequency and timestep > 0: - return '{0}_s0-{1}h'.format(folder_name, timestep) + return '{0}_f{1}h'.format(folder_name, timestep) return folder_name def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index b1cc4980..099ef4f1 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -92,7 +92,7 @@ class Diagnostic(Publisher): if file_generated.has_modifiers(): Log.warning('Can not skip diagnostics run when data is going to be modified: {0}'.format(self)) return False - return False + return True def __repr__(self): """Full string representation. Defaults to str""" @@ -797,6 +797,7 @@ class DiagnosticBasinOption(DiagnosticOption): raise DiagnosticOptionError('Basin {0} not recognized'.format(value)) return basin + class DiagnosticComplexStrOption(DiagnosticOption): """ Class to parse complex string options diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index f2434cd3..168a8eda 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -1,12 +1,12 @@ # coding=utf-8 """Compute the sea ice extent , area and volume in both hemispheres or a specified region""" -import os import six +import numpy as np import iris import iris.analysis import iris.coords -import numpy as np +import iris.util from bscearth.utils.log import Log from earthdiagnostics.constants import Basins @@ -118,11 +118,9 @@ class Siasiesiv(Diagnostic): self._declare_var('siextentn') def _declare_var(self, var_name): - self.generated[var_name] = {} - for basin in self.masks.keys(): - self.generated[var_name][basin] = self.declare_chunk(ModelingRealms.seaIce, var_name, - self.startdate, self.member,self.chunk, - region=basin.name) + self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, + self.startdate, self.member, self.chunk, + region=self.masks.keys()) def compute(self): """Run the diagnostic""" @@ -130,6 +128,8 @@ class Siasiesiv(Diagnostic): handler.variables[self.sic_varname].coordinates = 'time lat lon leadtime time_centered' handler.close() sic = iris.load_cube(self.sic.local_file) + if sic.units.origin == '%' and sic.data.max() < 2: + sic.units = '1.0' sic.convert_units('1.0') sic *= Siasiesiv.area.data extent = sic.data > 0.15 @@ -165,6 +165,7 @@ class Siasiesiv(Diagnostic): def save(self): for var, basins in six.iteritems(self.results): + results = iris.cube.CubeList() for basin, result in six.iteritems(basins): if result.data.max() == 0: continue @@ -174,9 +175,21 @@ class Siasiesiv(Diagnostic): else: result.units = 'm^2' result.add_aux_coord(iris.coords.AuxCoord(basin.name, var_name='region')) - self._save_file(result, self.generated[var][basin], basin) + results.append(result) + if not results: + continue + self._save_file(results.merge_cube(), self.generated[var], basins) - def _save_file(self, data, generated_file, basin): + def _save_file(self, data, generated_file, basins): temp = TempFile.get() - iris.save(data, temp) - generated_file.set_local_file(temp, region=basin) + region = data.coord('region').points + data.remove_coord('region') + iris.save(data, temp, zlib=True) + Utils.rename_variable(temp, 'dim0', 'region', False, True) + handler = Utils.open_cdf(temp) + var = handler.createVariable('region2', str, ('region',)) + var[...] = region + handler.close() + Utils.rename_variable(temp, 'region2', 'region', True, False) + + generated_file.set_local_file(temp, region=basins.keys()) diff --git a/environment.yml b/environment.yml index a642f28b..42dcdbbe 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,7 @@ channels: - conda-forge dependencies: -- iris +- iris<2 - netcdf4 - numpy - cdo=1.9.2 diff --git a/launch_diags.sh b/launch_diags.sh index df9a004b..1ff20d77 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -7,7 +7,7 @@ -PATH_TO_CONF_FILE=~rfernand/run_earthdiagnostics/m04p/diags.conf +PATH_TO_CONF_FILE=~/earthdiags_confs/ruben/diags_a0dc_DEBUG.conf PATH_TO_DIAGNOSTICS=~jvegas/PycharmProjects/earthdiagnostics PATH_TO_CONDAENV=/home/Earth/jvegas/.conda/envs/earthdiagnostics3/ diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py index c2d7cbab..5ad93a26 100644 --- a/test/unit/ocean/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -17,8 +17,8 @@ class TestSiasiesiv(TestCase): self.mask = Mock() self.var_manager = Mock() - self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask, self.var_manager, False) + self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, {Basins().Global: []}, self.var_manager, False) def test_str(self): self.assertEqual(str(self.psi), - 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global Omit volume: False') + 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basins: Global Omit volume: False') diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 0117df3f..9c32c184 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -28,6 +28,7 @@ class TestCMORManager(TestCase): self.config.experiment.calendar = 'standard' self.config.experiment.atmos_timestep = 3 self.config.experiment.ocean_timestep = 6 + self.config.experiment.get_member_str.return_value = 'r1i1p1' self.config.cmor.initialization_number = 1 self.config.cmor.version = '' @@ -41,6 +42,7 @@ class TestCMORManager(TestCase): self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) self.config.data_dir = self.tmp_dir + self.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') def tearDown(self): shutil.rmtree(self.tmp_dir) @@ -531,7 +533,6 @@ class TestCMORManager(TestCase): original_cmor_path = os.path.join(self.config.data_dir, self.config.experiment.expid, 'original_files', 'cmorfiles') os.makedirs(original_cmor_path) - self.config.experiment.get_member_str.return_value = 'r1i1p1' self.config.experiment.get_chunk_start_str.return_value = '20000101' cmor_prefix = 'CMORT_{0}_{1}_{2}_{3}-'.format(self.config.experiment.expid, '20000101', 'r1i1p1', '20000101') diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 030fb3ea..121e5617 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -30,7 +30,7 @@ class TestDownloader(TestCase): self.downloader.shutdown() datafile.download.assert_called_once() - def test_download(self): + def test_download_fails(self): datafile = Mock() def _download(): @@ -320,6 +320,9 @@ class TestWorkManager(TestCase): def declare_data_generated(self): pass + def can_skip_run(self): + return False + self.work_manager.jobs = [FailDiag(self.data_manager)] self.work_manager.run() -- GitLab From b9d52989421e5da6f5917d5d5021e609240dc571 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 12 Mar 2018 15:45:22 +0100 Subject: [PATCH 70/70] Siasiesiv now output all veriables, not only the non-zero --- earthdiagnostics/ocean/siasiesiv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 168a8eda..d60d2bb7 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -167,8 +167,6 @@ class Siasiesiv(Diagnostic): for var, basins in six.iteritems(self.results): results = iris.cube.CubeList() for basin, result in six.iteritems(basins): - if result.data.max() == 0: - continue result.var_name = var if var.startswith('sivol'): result.units = 'm^3' -- GitLab