From c6b130cd6ee9dfc38cf4865331e6035110d02117 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 22 Nov 2017 15:04:41 +0100 Subject: [PATCH 01/25] Added test for cmormananger --- earthdiagnostics/cmorizer.py | 2 +- earthdiagnostics/cmormanager.py | 29 ++++++++++---------- test/unit/test_cmormanager.py | 47 +++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 test/unit/test_cmormanager.py diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 4bfde9de..97c19717 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -1,7 +1,6 @@ # coding=utf-8 import glob import os -import pygrib import shutil import uuid from datetime import datetime @@ -338,6 +337,7 @@ class Cmorizer(object): def _get_atmos_timestep(self, gribfile): Log.info('Getting timestep...') + import pygrib grib_handler = pygrib.open(gribfile) dates = set() try: diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index cfa67cae..265fc677 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -23,11 +23,14 @@ class CMORManager(DataManager): def __init__(self, config): super(CMORManager, self).__init__(config) self._dic_cmorized = dict() + self.find_model_data() + self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid) + + def find_model_data(self): data_folders = self.config.data_dir.split(':') experiment_folder = self.experiment.model.lower() if experiment_folder.startswith('ec-earth'): experiment_folder = 'ecearth' - self.config.data_dir = None for data_folder in data_folders: if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): @@ -38,14 +41,12 @@ class CMORManager(DataManager): self.config.data_dir = test_folder break - test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) + test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) if os.path.isdir(os.path.join(test_folder, self.experiment.expid)): self.config.data_dir = test_folder break - if not self.config.data_dir: raise Exception('Can not find model data') - self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, @@ -74,8 +75,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param vartype: - :param domain: CMOR domain + :param vartype: + :param domain: CMOR domain :type domain: Domain :param var: variable name :type var: str @@ -104,8 +105,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param year: - :param diagnostic: + :param year: + :param diagnostic: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -137,8 +138,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param region: + :param diagnostic: + :param region: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -179,8 +180,8 @@ class CMORManager(DataManager): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - :param diagnostic: - :param year: + :param diagnostic: + :param year: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -215,7 +216,7 @@ class CMORManager(DataManager): grid=None, year=None, date_str=None): """ Returns the path to a concrete file - :param cmor_var: + :param cmor_var: :param startdate: file's startdate :type startdate: str :param member: file's member @@ -332,7 +333,7 @@ class CMORManager(DataManager): """ Creates the link of a given file from the CMOR repository. - :param cmor_var: + :param cmor_var: :param move_old: :param date_str: :param year: if frequency is yearly, this parameter is used to give the corresponding year diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py new file mode 100644 index 00000000..9c5fbb5f --- /dev/null +++ b/test/unit/test_cmormanager.py @@ -0,0 +1,47 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.cmormanager import CMORManager +from earthdiagnostics.modelingrealm import ModelingRealms +from mock import Mock +import tempfile +import os +import shutil + + +class TestCMORManager(TestCase): + + def setUp(self): + self.config = Mock() + self.config.data_convention = 'specs' + self.config.experiment.expid = 'expid' + self.config.experiment.model = 'model' + self.config.experiment.atmos_timestep = 6 + self.config.experiment.ocean_timestep = 6 + + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + self.config.data_dir = self.tmp_dir + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_find_data(self): + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'expid')) + + def test_find_data_fail(self): + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + with self.assertRaises(Exception): + CMORManager(self.config) + + def test_find_data_with_model(self): + os.makedirs(os.path.join(self.tmp_dir, 'model', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'model', 'expid')) + + def test_get_varfolder(self): + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.get_varfolder(ModelingRealms.ocean, 'var'), + 'var_f6h') + -- GitLab From f551c2f46e7e8d76e8f291582cf0255c3529fcfc Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 09:31:48 +0100 Subject: [PATCH 02/25] Added test for paths in specs format --- earthdiagnostics/cmormanager.py | 6 ++ test/unit/ocean/test_interpolatecdo.py | 4 +- test/unit/test_cmormanager.py | 131 +++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 265fc677..d66690e3 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -240,6 +240,12 @@ class CMORManager(DataManager): :param cmor_var: variable instance describing the selected variable :type cmor_var: Variable """ + + options = sum(x is not None for x in (chunk, year, date_str)) + if options == 0: + raise ValueError('You must provide chunk, year or date_str') + elif options > 1: + raise ValueError('You must provide only one parameter in chunk, year or date_str') if not frequency: frequency = self.config.frequency diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py index ba238f28..b7b87cf1 100644 --- a/test/unit/ocean/test_interpolatecdo.py +++ b/test/unit/ocean/test_interpolatecdo.py @@ -28,8 +28,10 @@ class TestInterpolate(TestCase): @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO.get_sample_grid_file') @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) @patch('os.remove') - def test_generate_jobs(self, mock_weights, mock_grid_file, mock_remove): + @patch('earthdiagnostics.utils.TempFile.get') + def test_generate_jobs(self, mock_weights, mock_grid_file, mock_remove, mock_get): mock_weights.return_value = None + mock_get.return_value = 'path_to_weights' jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var']) self.assertEqual(len(jobs), 2) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 9c5fbb5f..6e563344 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -13,10 +13,19 @@ class TestCMORManager(TestCase): def setUp(self): self.config = Mock() self.config.data_convention = 'specs' + self.config.data_type = 'exp' self.config.experiment.expid = 'expid' self.config.experiment.model = 'model' + self.config.experiment.experiment_name = 'expname' + self.config.experiment.institute = 'institute' + self.config.experiment.member_count_start = 0 self.config.experiment.atmos_timestep = 6 self.config.experiment.ocean_timestep = 6 + self.config.experiment.chunk_size = 12 + self.config.experiment.calendar = 'standard' + + self.config.cmor.initialization_number = 1 + self.config.cmor.version = '' self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) @@ -40,8 +49,130 @@ class TestCMORManager(TestCase): cmor_manager = CMORManager(self.config) self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'model', 'expid')) + def test_find_data_with_ecearth_fix(self): + self.config.experiment.model = 'EC-Earth' + os.makedirs(os.path.join(self.tmp_dir, 'ecearth', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'ecearth', 'expid')) + + def test_find_data_with_type_and_model(self): + os.makedirs(os.path.join(self.tmp_dir, 'exp', 'model', self.config.experiment.expid)) + os.rmdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) + cmor_manager = CMORManager(self.config) + self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'exp', 'model', 'expid')) + def test_get_varfolder(self): cmor_manager = CMORManager(self.config) self.assertEqual(cmor_manager.get_varfolder(ModelingRealms.ocean, 'var'), 'var_f6h') + def test_get_file_path_specs(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_version(self): + self.config.cmor.version = 'version' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/version/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_grid(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency, 'grid') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/grid/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) + + def test_get_file_path_specs_year(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'year' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'year' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, year='1998') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/year/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_1998.nc')) + + frequency.frequency = 'other' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'other' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + + def test_get_file_path_raise_incomaptible_date_info(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'monthly' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, date_str='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + None, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998', date_str='1998') + + def test_get_file_path_specs_date_str(self): + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + frequency.frequency = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, date_str='date_str') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/frequency/' + 'ocean/var/r2i1p1/' + 'var_Omon_model_expname_S19900101_r2i1p1_date_str.nc')) + -- GitLab From d1561e452a9247029a80e9a8dbfbab1b73107ed7 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 09:37:04 +0100 Subject: [PATCH 03/25] Removed unused code --- earthdiagnostics/cmormanager.py | 31 ------------------- earthdiagnostics/constants.py | 4 +-- earthdiagnostics/diagnostic.py | 1 - earthdiagnostics/general/dailymean.py | 1 - earthdiagnostics/general/module.py | 1 - earthdiagnostics/general/monthlymean.py | 1 - earthdiagnostics/general/relink.py | 2 -- earthdiagnostics/general/scale.py | 1 - earthdiagnostics/general/select_levels.py | 1 - .../general/simplify_dimensions.py | 3 +- earthdiagnostics/ocean/mask_land.py | 3 +- test/unit/general/test_relinkall.py | 2 -- test/unit/ocean/test_vertical_gradient.py | 2 -- 13 files changed, 4 insertions(+), 49 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index d66690e3..b39ecd73 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -602,37 +602,6 @@ class CMORManager(DataManager): self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) - def _fix_ij_swap(self, cmorfile): - return - original_handler = Utils.openCdf(cmorfile) - if original_handler.dimensions['i'].size < original_handler.dimensions['j'].size: - temp = TempFile.get() - new_handler = Utils.openCdf(temp, 'w') - for attribute in original_handler.ncattrs(): - original = getattr(original_handler, attribute) - setattr(new_handler, attribute, - Utils.convert_to_ASCII_if_possible(original)) - for dimension in original_handler.dimensions.keys(): - if dimension == 'i': - new_name = 'j' - elif dimension == 'j': - new_name = 'i' - else: - new_name = dimension - new_handler.createDimension(new_name, original_handler.dimensions[dimension].size) - for variable in original_handler.variables.keys(): - original_var = original_handler.variables[variable] - translated_dimensions = Utils._translate(original_var.dimensions, - {'i': 'j', 'j': 'i'}) - new_var = new_handler.createVariable(variable, original_var.datatype, - translated_dimensions) - Utils.copy_attributes(new_var, original_var) - new_var[:] = original_var[:] - original_handler.close() - new_handler.close() - Utils.move_file(temp, cmorfile, save_hash=True) - Log.debug('File {0} translated', cmorfile) - def _get_startdate_path(self, startdate): """ Returns the path to the startdate's CMOR folder diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 229a61b0..d7935838 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -2,8 +2,6 @@ """ Contains the enumeration-like classes used by the diagnostics """ -import netCDF4 - from singleton import SingletonType @@ -133,7 +131,7 @@ class Basins(object): def get_available_basins(self, handler): """ - + :param handler: :type handler: netCDF4.Dataset """ diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 0b1dbf57..c49dea84 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -245,7 +245,6 @@ class Diagnostic(Publisher): def _updated_request(self, request): if self.status != DiagnosticStatus.WAITING: return - from datafile import LocalStatus if request.local_status == LocalStatus.FAILED: self.message = 'Required file {0} is not available'.format(request.remote_file) self.status = DiagnosticStatus.FAILED diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index 7fb4736e..1411b0ed 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -5,7 +5,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class DailyMean(Diagnostic): diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index f72aa5f5..e8cf6d85 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm import numpy as np diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index dca5e730..6062d075 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -5,7 +5,6 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, Diagnostic DiagnosticFrequencyOption, DiagnosticVariableOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm class MonthlyMean(Diagnostic): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index 60c69f4c..b2545e1b 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -1,8 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption, \ DiagnosticVariableOption -from earthdiagnostics.modelingrealm import ModelingRealm -from earthdiagnostics.variable import VariableManager class Relink(Diagnostic): diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 116a978d..46b7af47 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils -from earthdiagnostics.modelingrealm import ModelingRealm import math diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index 1d2fb9ca..69931b87 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -1,7 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption, DiagnosticIntOption -from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.box import Box diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index c903af0a..c9c39e21 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -3,7 +3,6 @@ import numpy as np from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption -from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.utils import Utils, TempFile @@ -127,7 +126,7 @@ class SimplifyDimensions(Diagnostic): '{0}_vertices'.format(self.lon_name), '{0}_vertices'.format(self.lat_name)): continue Utils.copy_variable(handler, new_file, var, new_names={'i': self.lon_name, 'j': self.lat_name}) - + self._create_var(self.lon_name, lon_values, handler, new_file) self._create_var(self.lat_name, lat_values, handler, new_file) handler.close() diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index a7af9aaa..24370378 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -25,6 +25,7 @@ class MaskLand(Diagnostic): """ alias = 'maskland' + "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, mask, grid): Diagnostic.__init__(self, data_manager) @@ -83,7 +84,7 @@ class MaskLand(Diagnostic): mask_file.close() return mask - "Diagnostic alias for the configuration file" + def request_data(self): self.var_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, diff --git a/test/unit/general/test_relinkall.py b/test/unit/general/test_relinkall.py index cf8c9a16..cda393c8 100644 --- a/test/unit/general/test_relinkall.py +++ b/test/unit/general/test_relinkall.py @@ -6,8 +6,6 @@ from earthdiagnostics.box import Box from earthdiagnostics.general.relinkall import RelinkAll from mock import Mock, patch -from earthdiagnostics.modelingrealm import ModelingRealms - class TestRelinkAll(TestCase): diff --git a/test/unit/ocean/test_vertical_gradient.py b/test/unit/ocean/test_vertical_gradient.py index e2d9d22d..892160a2 100644 --- a/test/unit/ocean/test_vertical_gradient.py +++ b/test/unit/ocean/test_vertical_gradient.py @@ -1,8 +1,6 @@ # coding=utf-8 from unittest import TestCase from earthdiagnostics.ocean.verticalgradient import VerticalGradient -from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption from mock import Mock, patch -- GitLab From 015082a5e79766fedcf774fdf8176a336d115800 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 12:55:07 +0100 Subject: [PATCH 04/25] Added tests for PRIMAVERA paths --- earthdiagnostics/cmormanager.py | 3 +- test/unit/test_cmormanager.py | 96 +++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index b39ecd73..e6ffeebb 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -306,6 +306,8 @@ class CMORManager(DataManager): if self.config.cmor.version: folder_path = os.path.join(folder_path, self.config.cmor.version) else: + if not self.config.cmor.version: + raise ValueError('CMOR version is mandatory for PRIMAVERA and CMIP6') if not grid: if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: grid = self.config.cmor.default_ocean_grid @@ -598,7 +600,6 @@ class CMORManager(DataManager): if not filename.endswith('.nc') or filename.startswith('.'): return cmorfile = os.path.join(filepath, filename) - self._fix_ij_swap(cmorfile) self.create_link(domain, cmorfile, frequency, var, "", False, vartype=VariableType.MEAN) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 6e563344..20b30d30 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -26,6 +26,9 @@ class TestCMORManager(TestCase): self.config.cmor.initialization_number = 1 self.config.cmor.version = '' + self.config.cmor.default_ocean_grid = 'ocean_grid' + self.config.cmor.default_atmos_grid = 'atmos_grid' + self.config.cmor.activity = 'activity' self.tmp_dir = tempfile.mkdtemp() os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) @@ -176,3 +179,96 @@ class TestCMORManager(TestCase): 'ocean/var/r2i1p1/' 'var_Omon_model_expname_S19900101_r2i1p1_date_str.nc')) + def test_get_file_path_primavera(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' + 'r2i1p1f1/Omon/var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) + + def test_get_file_path_no_version_primavera(self): + self._configure_primavera() + self.config.cmor.version = '' + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + 0, frequency) + + def _configure_primavera(self): + self.config.data_convention = 'primavera' + self.config.cmor.version = 'version' + + def test_get_file_path_primavera_grid(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, + frequency, 'grid') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' + 'Omon/var/grid/version/' + 'var_Omon_model_expname_r2i1p1f1_grid_198901-198912.nc')) + + def test_get_file_path_primavera_year(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'year' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'year' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, year='1998') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/' + 'var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_1998.nc')) + + frequency.frequency = 'other' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'other' + self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', cmor_var, + 1, frequency, year='1998') + + def test_get_file_path_primavera_date_str(self): + self._configure_primavera() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + frequency.frequency = 'frequency' + file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, + frequency, date_str='date_str') + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' + 'Omon/var/ocean_grid/version/' + 'var_Omon_model_expname_r2i1p1f1_ocean_grid_date_str.nc')) -- GitLab From 7977fa13f633c50664eb0431650fb645f1e34a67 Mon Sep 17 00:00:00 2001 From: jvegasbsc Date: Wed, 29 Nov 2017 13:14:14 +0100 Subject: [PATCH 05/25] Added test for file_exists --- earthdiagnostics/cmormanager.py | 15 +++------------ test/unit/test_cmormanager.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index e6ffeebb..9b6e0d1d 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -54,21 +54,12 @@ class CMORManager(DataManager): cmor_var = self.variable_list.get_variable(var) filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) - # noinspection PyBroadException if possible_versions is None: - # noinspection PyBroadException - try: - return os.path.isfile(filepath) - except Exception: - return False + return os.path.isfile(filepath) else: for version in possible_versions: - # noinspection PyBroadException - try: - if os.path.isfile(filepath.replace(self.config.cmor.version, version)): - return True - except Exception: - pass + if os.path.isfile(filepath.replace(self.config.cmor.version, version)): + return True return False def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 20b30d30..e56deea5 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -3,6 +3,7 @@ from unittest import TestCase from earthdiagnostics.cmormanager import CMORManager from earthdiagnostics.modelingrealm import ModelingRealms from mock import Mock +import mock import tempfile import os import shutil @@ -272,3 +273,22 @@ class TestCMORManager(TestCase): os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' 'Omon/var/ocean_grid/version/' 'var_Omon_model_expname_r2i1p1f1_ocean_grid_date_str.nc')) + + def test_file_exists(self): + with mock.patch('os.path.isfile') as isfile: + cmor_manager = CMORManager(self.config) + isfile.return_value = True + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) + isfile.return_value = False + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1)) + + + def test_file_exists_multiple_versions(self): + with mock.patch('os.path.isfile') as isfile: + cmor_manager = CMORManager(self.config) + isfile.return_value = True + self.assertTrue(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + possible_versions=('version1', 'version2'))) + isfile.return_value = False + self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, + possible_versions=('version1', 'version2'))) -- GitLab From c6b210c380ea2cbe0f87424b9f41ef69a79d1db3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 11:06:55 +0100 Subject: [PATCH 06/25] Added mask option to scale --- earthdiagnostics/general/scale.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 46b7af47..da35ebad 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -1,7 +1,10 @@ # coding=utf-8 +import math + +import numpy as np + from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils -import math class Scale(Diagnostic): @@ -32,7 +35,7 @@ class Scale(Diagnostic): "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, value, offset, domain, variable, grid, - min_limit, max_limit, frequency): + min_limit, max_limit, frequency, apply_mask): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -45,6 +48,7 @@ class Scale(Diagnostic): self.min_limit = min_limit self.max_limit = max_limit self.frequency = frequency + self.apply_mask = apply_mask self.original_values = None @@ -76,14 +80,16 @@ class Scale(Diagnostic): DiagnosticOption('grid', ''), DiagnosticFloatOption('min_limit', float('nan')), DiagnosticFloatOption('max_limit', float('nan')), - DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency])) + DiagnosticListFrequenciesOption('frequencies', [diags.config.frequency]), + DiagnosticBoolOption('apply_mask', False)) options = cls.process_options(options, options_available) job_list = list() for frequency in options['frequencies']: for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Scale(diags.data_manager, startdate, member, chunk, options['value'], options['offset'], options['domain'], options['variable'], - options['grid'], options['min_limit'], options['max_limit'], frequency)) + options['grid'], options['min_limit'], options['max_limit'], frequency, + options['apply_mask'])) return job_list def request_data(self): @@ -101,10 +107,17 @@ class Scale(Diagnostic): variable_file = self.variable_file.local_file handler = Utils.openCdf(variable_file) - var_handler = handler.variables[self.variable] - self.original_values = var_handler[:] + var = handler.variables[self.variable] + self.original_values = var[:] + if self.apply_mask: + mask = Utils.get_mask(Basins().Global).astype(float) + mask[mask == 0] = np.nan + var[:] = mask * var[:] if self._check_limits(): - var_handler[:] = self.original_values * self.value + self.offset + values = self.original_values * self.value + self.offset + if self.apply_mask: + values[np.isnan(values)] = 0 + var[:] = values handler.close() self.corrected.set_local_file(self.variable_file.local_file, self) -- GitLab From 9c1fbc23a5f9e2c4d66d4b0526d5bcef6720cc83 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 11:37:57 +0100 Subject: [PATCH 07/25] Added meteofrance path convention --- earthdiagnostics/cmormanager.py | 13 ++++++++++++- launch_diags.sh | 6 +++--- test/unit/test_cmormanager.py | 30 +++++++++++++++++++++++++----- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 9b6e0d1d..5c7298bd 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -284,6 +284,8 @@ class CMORManager(DataManager): self.experiment.experiment_name, self._get_member_str(member), grid, time_bound) + elif self.config.data_convention in ('meteofrance',): + file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, startdate[0:6], self._get_member_str(member)) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name @@ -296,7 +298,8 @@ class CMORManager(DataManager): folder_path = os.path.join(folder_path, self._get_member_str(member)) if self.config.cmor.version: folder_path = os.path.join(folder_path, self.config.cmor.version) - else: + + elif self.config.data_convention in ('primavera', 'cmip6'): if not self.config.cmor.version: raise ValueError('CMOR version is mandatory for PRIMAVERA and CMIP6') if not grid: @@ -311,6 +314,12 @@ class CMORManager(DataManager): folder_path = os.path.join(self._get_startdate_path(startdate), self._get_member_str(member), table_name, var, grid, self.config.cmor.version) + elif self.config.data_convention == 'meteofrance': + folder_path = os.path.join(self.config.data_dir, + 'H{0}'.format(chr(64 + int(startdate[4:6]))), + startdate[0:4]) + else: + raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return folder_path def _get_chunk_time_bounds(self, startdate, chunk): @@ -617,6 +626,8 @@ class CMORManager(DataManager): template = 'r{0}i{1}p1' elif self.config.data_convention in ('primavera', 'cmip6'): template = 'r{0}i{1}p1f1' + elif self.config.data_convention == 'meteofrance': + return str(member) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) diff --git a/launch_diags.sh b/launch_diags.sh index 477a1ca4..bcfa6b07 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -21,6 +21,6 @@ set -xv source activate diags -export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} -cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ -./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} + export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} + cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ + ./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index e56deea5..258fc791 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -1,12 +1,14 @@ # coding=utf-8 +import os +import shutil +import tempfile from unittest import TestCase + +import mock +from mock import Mock + from earthdiagnostics.cmormanager import CMORManager from earthdiagnostics.modelingrealm import ModelingRealms -from mock import Mock -import mock -import tempfile -import os -import shutil class TestCMORManager(TestCase): @@ -215,6 +217,9 @@ class TestCMORManager(TestCase): self.config.data_convention = 'primavera' self.config.cmor.version = 'version' + def _configure_meteofrance(self): + self.config.data_convention = 'meteofrance' + def test_get_file_path_primavera_grid(self): self._configure_primavera() cmor_manager = CMORManager(self.config) @@ -292,3 +297,18 @@ class TestCMORManager(TestCase): isfile.return_value = False self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1,1, possible_versions=('version1', 'version2'))) + + def test_get_file_path_meteofrance(self): + self._configure_meteofrance() + cmor_manager = CMORManager(self.config) + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'day' + file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 1, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'HA/2011/soicecov_day_201101_16.nc')) -- GitLab From 071765a5c614b63c21594a0722541c08772fa3d1 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 15:11:15 +0100 Subject: [PATCH 08/25] Fixed meteofrance format --- earthdiagnostics/cmorizer.py | 2 +- earthdiagnostics/cmormanager.py | 10 +++++++--- test/unit/test_cmormanager.py | 7 ++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 97c19717..baf5f3de 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -363,7 +363,7 @@ class Cmorizer(object): Utils.convert2netcdf4(filename) frequency = self._get_nc_file_frequency(filename) - Utils.rename_variables(filename, Cmorizer.ALT_COORD_NAMES, False, True) + Utils.rename_variables(filename, self.alt_coord_names, False, True) self._remove_valid_limits(filename) self._add_common_attributes(filename, frequency) self._update_time_variables(filename) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 5c7298bd..27261987 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -285,7 +285,8 @@ class CMORManager(DataManager): self._get_member_str(member), grid, time_bound) elif self.config.data_convention in ('meteofrance',): - file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, startdate[0:6], self._get_member_str(member)) + time_bound = self._get_chunk_time_bounds(startdate, chunk) + file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self._get_member_str(member)) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) return file_name @@ -315,7 +316,7 @@ class CMORManager(DataManager): table_name, var, grid, self.config.cmor.version) elif self.config.data_convention == 'meteofrance': - folder_path = os.path.join(self.config.data_dir, + folder_path = os.path.join(self.config.data_dir, self.experiment.experiment_name, 'H{0}'.format(chr(64 + int(startdate[4:6]))), startdate[0:4]) else: @@ -331,7 +332,10 @@ class CMORManager(DataManager): separator = '_' else: separator = '-' - time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, + if self.config.data_convention == 'meteofrance': + time_bound = "{0:04}{1:02}".format(chunk_start.year, chunk_start.month) + else: + time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, chunk_end.month, separator) return time_bound diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 258fc791..8367545b 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -311,4 +311,9 @@ class TestCMORManager(TestCase): file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 1, frequency) self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'HA/2011/soicecov_day_201101_16.nc')) + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_16.nc')) + + file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 2, + frequency) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_16.nc')) -- GitLab From 92c42599c74215f49e19b6dbea9140c7a15e8b70 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 15:12:43 +0100 Subject: [PATCH 09/25] Added option meteofrance to config --- earthdiagnostics/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index f3c8d638..f5d4c3c4 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -23,7 +23,7 @@ class Config(object): :param path: path to the conf file :type path: str """ - + def __init__(self, path): parser = ConfigParser() @@ -62,7 +62,8 @@ class Config(object): "Custom mask regions 3D file to use" self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', - ('specs', 'primavera', 'cmip6', 'preface'), 'specs', + ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), + 'specs', ignore_case=True) if self.data_convention in ('primavera', 'cmip6'): @@ -130,10 +131,10 @@ class Config(object): :rtype: list(str) """ return self._real_commands - + class CMORConfig(object): - + def __init__(self, parser, var_manager): self.force = parser.get_bool_option('CMOR', 'FORCE', False) self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) -- GitLab From be9a2b61858a069ae6d31477adf978e1b51ea68b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 15:18:00 +0100 Subject: [PATCH 10/25] Added tables for meteofrance --- earthdiagnostics/cmor_tables/meteofrance.csv | 1 + earthdiagnostics/variable_alias/meteofrance.csv | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 earthdiagnostics/cmor_tables/meteofrance.csv create mode 100644 earthdiagnostics/variable_alias/meteofrance.csv diff --git a/earthdiagnostics/cmor_tables/meteofrance.csv b/earthdiagnostics/cmor_tables/meteofrance.csv new file mode 100644 index 00000000..90f01d91 --- /dev/null +++ b/earthdiagnostics/cmor_tables/meteofrance.csv @@ -0,0 +1 @@ +Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables diff --git a/earthdiagnostics/variable_alias/meteofrance.csv b/earthdiagnostics/variable_alias/meteofrance.csv new file mode 100644 index 00000000..07aef840 --- /dev/null +++ b/earthdiagnostics/variable_alias/meteofrance.csv @@ -0,0 +1,4 @@ +Aliases,Shortname,Basin,Grid +iiceconc:siconc:soicecov:ileadfra,sic,, +ci,sic,,ifs +es,sbl,, \ No newline at end of file -- GitLab From 021d67abaa22dae912fa6c4e3cad912757c74af6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Thu, 30 Nov 2017 17:55:58 +0100 Subject: [PATCH 11/25] Added sic ans sit names for meteofrance --- earthdiagnostics/cmor_tables/meteofrance.csv | 2 ++ earthdiagnostics/cmormanager.py | 3 +++ earthdiagnostics/variable_alias/meteofrance.csv | 4 ++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/cmor_tables/meteofrance.csv b/earthdiagnostics/cmor_tables/meteofrance.csv index 90f01d91..086e2ec0 100644 --- a/earthdiagnostics/cmor_tables/meteofrance.csv +++ b/earthdiagnostics/cmor_tables/meteofrance.csv @@ -1 +1,3 @@ Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables +iiceconc:siconc:soicecov:ileadfra:ci,soicecov,sea_ice_area_fraction,Sea Ice Area Fraction,seaIce,,%,,,, +iicethic:sithic,sogsit__,sea_ice_thickness,Sea Ice Thickness,seaIce,,m,,,, diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 27261987..66c75082 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -392,6 +392,9 @@ class CMORManager(DataManager): """ # Check if cmorized and convert if not + if self.config.data_convention == 'meteofrance': + return + for startdate, member in self.experiment.get_member_list(): if not self._unpack_cmor_files(startdate, member): self._cmorize_member(startdate, member) diff --git a/earthdiagnostics/variable_alias/meteofrance.csv b/earthdiagnostics/variable_alias/meteofrance.csv index 07aef840..abb99d5e 100644 --- a/earthdiagnostics/variable_alias/meteofrance.csv +++ b/earthdiagnostics/variable_alias/meteofrance.csv @@ -1,4 +1,4 @@ Aliases,Shortname,Basin,Grid -iiceconc:siconc:soicecov:ileadfra,sic,, +iiceconc:siconc:soicecov:ileadfra,soicecov,, ci,sic,,ifs -es,sbl,, \ No newline at end of file +es,sbl,, -- GitLab From dff07441c14411b583e6ea633daf062790f97487 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Fri, 1 Dec 2017 17:16:33 +0100 Subject: [PATCH 12/25] Removed cfunits dependency --- earthdiagnostics/utils.py | 16 ++++++++-------- environment.yml | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 7c70b56e..026c1f7d 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -10,6 +10,7 @@ import tarfile import tempfile from contextlib import contextmanager +import cf_units import iris import iris.exceptions import netCDF4 @@ -18,7 +19,6 @@ import six import xxhash from bscearth.utils.log import Log from cdo import Cdo -from cfunits import Units from nco import Nco from earthdiagnostics.constants import Basins @@ -666,15 +666,15 @@ class Utils(object): if hasattr(var_handler, 'calendar'): old_calendar = var_handler.calendar - new_unit = Units(new_units, calendar=calendar) - old_unit = Units(var_handler.units, calendar=old_calendar) - var_handler[:] = Units.conform(var_handler[:], old_unit, new_unit, inplace=True) + new_unit = cf_units.Unit(new_units, calendar=calendar) + old_unit = cf_units.Unit(var_handler.units, calendar=old_calendar) + var_handler[:] = old_unit.convert(var_handler[:], new_unit, inplace=True) if 'valid_min' in var_handler.ncattrs(): - var_handler.valid_min = Units.conform(float(var_handler.valid_min), old_unit, new_unit, - inplace=True) + var_handler.valid_min = old_unit.convert(float(var_handler.valid_min), new_unit, + inplace=True) if 'valid_max' in var_handler.ncattrs(): - var_handler.valid_max = Units.conform(float(var_handler.valid_max), old_unit, new_unit, - inplace=True) + var_handler.valid_max = old_unit.convert(float(var_handler.valid_max), new_unit, + inplace=True) var_handler.units = new_units @staticmethod diff --git a/environment.yml b/environment.yml index 4fa44d13..14d62005 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,6 @@ dependencies: - openpyxl - mock - cmake -- cfunits - coverage - pip: -- GitLab From 18c8b79b9541b2f1e316744f27d7bb5b6fb5b4f3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 10:28:06 +0100 Subject: [PATCH 13/25] Fixed scale tests --- earthdiagnostics/general/scale.py | 10 ++++---- environment.yml | 2 +- test/unit/general/test_scale.py | 42 ++++++++++++++++++------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index da35ebad..099d6774 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -53,14 +53,14 @@ class Scale(Diagnostic): self.original_values = None def __str__(self): - return 'Scale output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Scale value: {5} Offset: {6} Variable: {3}:{4} ' \ - 'Frequency: {7}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, - self.value, self.offset, self.frequency) + return 'Scale output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Scale value: {0.value} Offset: {0.offset} Variable: {0.domain}:{0.variable} ' \ + 'Frequency: {0.frequency} Apply mask: {0.apply_mask}'.format(self) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency + self.domain == other.domain and self.variable == other.variable and self.frequency == other.frequency and \ + self.apply_mask == other.apply_mask and self.value == other.value and self.offset == other.offset @classmethod def generate_jobs(cls, diags, options): diff --git a/environment.yml b/environment.yml index 14d62005..f804b6aa 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - openpyxl - mock - cmake -- coverage +5- coverage - pip: - bscearth.utils diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py index e7697cc2..b2fcd0d0 100644 --- a/test/unit/general/test_scale.py +++ b/test/unit/general/test_scale.py @@ -1,12 +1,12 @@ # coding=utf-8 from unittest import TestCase -from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError -from earthdiagnostics.box import Box -from earthdiagnostics.general.scale import Scale -from earthdiagnostics.frequency import Frequencies from mock import Mock, patch +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.scale import Scale from earthdiagnostics.modelingrealm import ModelingRealms @@ -32,40 +32,48 @@ class TestScale(TestCase): jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', '', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', '', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - float('nan'), float('nan'), Frequencies.monthly)) + float('nan'), float('nan'), Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.monthly)) + 0, 100, Frequencies.monthly, False)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.monthly)) + 0, 100, Frequencies.monthly, False)) jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.three_hourly)) + 0, 100, Frequencies.three_hourly, False)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly, False)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', + True]) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly, True)) self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', - 0, 100, Frequencies.three_hourly)) + 0, 100, Frequencies.three_hourly, True)) with self.assertRaises(DiagnosticOptionError): Scale.generate_jobs(self.diags, ['diagnostic']) - with self.assertRaises(DiagnosticOptionError): - Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', - 'extra']) + with self.assertRaises(DiagnosticOptionError): + Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', 'True', + 'extra']) def test_str(self): mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, - Frequencies.three_hourly) + Frequencies.three_hourly, False) self.assertEquals(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' - 'Variable: atmos:var Frequency: 3hr') + 'Variable: atmos:var Frequency: 3hr Apply mask: False') -- GitLab From 51b317d0a6ea94f64acbc559b97461b87954bf68 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 10:37:41 +0100 Subject: [PATCH 14/25] Fixed member meteofrance --- earthdiagnostics/cmormanager.py | 2 +- test/unit/test_cmormanager.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 66c75082..a0237d94 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -634,7 +634,7 @@ class CMORManager(DataManager): elif self.config.data_convention in ('primavera', 'cmip6'): template = 'r{0}i{1}p1f1' elif self.config.data_convention == 'meteofrance': - return str(member) + return '{0:02d}'.format(member) else: raise Exception('Data convention {0} not supported'.format(self.config.data_convention)) diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 8367545b..f536dd64 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -308,12 +308,12 @@ class TestCMORManager(TestCase): frequency = Mock() frequency.__str__ = Mock() frequency.__str__.return_value = 'day' - file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 1, + file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, frequency) self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_16.nc')) + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_01.nc')) - file_path = cmor_manager.get_file_path('20110101', 16, ModelingRealms.ocean, 'soicecov', cmor_var, 2, + file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 2, frequency) self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_16.nc')) + os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_01.nc')) -- GitLab From 5c5861196ad76909b8accc001edbcd78f1153c0b Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 12:57:32 +0100 Subject: [PATCH 15/25] Add fix for meteofrance var_names --- earthdiagnostics/CDFTOOLS_meteofrance.namlist | 211 ++++++++++++++++++ earthdiagnostics/datafile.py | 5 + 2 files changed, 216 insertions(+) create mode 100644 earthdiagnostics/CDFTOOLS_meteofrance.namlist diff --git a/earthdiagnostics/CDFTOOLS_meteofrance.namlist b/earthdiagnostics/CDFTOOLS_meteofrance.namlist new file mode 100644 index 00000000..8f747650 --- /dev/null +++ b/earthdiagnostics/CDFTOOLS_meteofrance.namlist @@ -0,0 +1,211 @@ + ! Thu Jun 30 16:19:27 2016 + ! Namelist automatically generated by PrintCdfNames + ! Do not edit without changing its name ... + ! ------------------------------------------ + &NAMDIM + CN_X = "i" + , + CN_Y = "j" + , + CN_Z = "lev" + , + CN_T = "time" + + / + &NAMDIMVAR + CN_VLON2D = "lon" + , + CN_VLAT2D = "lat" + , + CN_VDEPTHT = "lev" + , + CN_VDEPTHU = "lev" + , + CN_VDEPTHV = "lev" + , + CN_VDEPTHW = "lev" + , + CN_VTIMEC = "time" + , + CN_MISSING_VALUE = "_FillValue" + + / + &NAMMETRICS + CN_VE1T = "e1t" + , + CN_VE1U = "e1u" + , + CN_VE1V = "e1v" + , + CN_VE1F = "e1f" + , + CN_VE2T = "e2t" + , + CN_VE2U = "e2u" + , + CN_VE2V = "e2v" + , + CN_VE2F = "e2f" + , + CN_VE3T = "e3t" + , + CN_VE3W = "e3w" + , + CN_VFF = "ff" + , + CN_GLAMT = "glamt" + , + CN_GLAMU = "glamu" + , + CN_GLAMV = "glamv" + , + CN_GLAMF = "glamf" + , + CN_GPHIT = "gphit" + , + CN_GPHIU = "gphiu" + , + CN_GPHIV = "gphiv" + , + CN_GPHIF = "gphif" + , + CN_GDEPT = "gdept" + , + CN_GDEPW = "gdepw" + , + CN_HDEPT = "hdept" + , + CN_HDEPW = "hdepw" + + / + &NAMVARS + CN_VOTEMPER = "thetao" + , + CN_VOSALINE = "so" + , + CN_VOZOCRTX = "uo" + , + CN_VOMECRTY = "vo" + , + CN_VOMEEIVV = "vomeeivv" + , + CN_VOVECRTZ = "vovecrtz" + , + CN_SOSSHEIG = "sossheig" + , + CN_SOMXL010 = "mlotst" + , + CN_SOMXLT02 = "somxlt02" + , + CN_SOHEFLDO = "sohefldo" + , + CN_SOLHFLUP = "solhflup" + , + CN_SOSBHFUP = "sosbhfup" + , + CN_SOLWFLDO = "solwfldo" + , + CN_SOSHFLDO = "soshfldo" + , + CN_SOWAFLUP = "sowaflup" + , + CN_SOWAFLCD = "sowaflcd" + , + CN_SOWAFLDP = "sowafldp" + , + CN_IOWAFLUP = "iowaflup" + , + CN_ZOMSFATL = "zomsfatl" + , + CN_ZOMSFGLO = "zomsfglo" + , + CN_ZOMSFPAC = "zomsfpac" + , + CN_ZOMSFINP = "zomsfinp" + , + CN_ZOMSFIND = "zomsfind" + , + CN_ZOISOATL = "zoisoatl" + , + CN_ZOISOGLO = "zoisoglo" + , + CN_ZOISOPAC = "zoisopac" + , + CN_ZOISOINP = "zoisoinp" + , + CN_ZOISOIND = "zoisoind" + , + CN_VOZOUT = "vozout" + , + CN_VOMEVT = "vomevt" + , + CN_VOZOUS = "vozous" + , + CN_VOMEVS = "vomevs" + , + CN_SOZOUT = "sozout" + , + CN_SOMEVT = "somevt" + , + CN_SOZOUS = "sozous" + , + CN_SOMEVS = "somevs" + , + CN_SOZOUTRP = "sozoutrp" + , + CN_SOMEVTRP = "somevtrp" + , + CN_SOICECOV = "soicecov" + , + CN_VOSIGMA0 = "vosigma0" + , + CN_VOSIGMAI = "vosigmai" + , + CN_VOSIGNTR = "vosigntr" + , + CN_VODEPISO = "vodepiso" + , + CN_ISOTHICK = "isothick" + , + CN_IICETHIC = "iicethic" + , + CN_ILEADFRA = "ileadfra" + , + CN_INVCFC = "INVCFC" + , + CN_CFC11 = "CFC11" + , + CN_PENDEP = "pendep" + + / + &NAMBATHY + CN_FBATHYMET = "bathy_meter.nc" + , + CN_FBATHYLEV = "bathy_level.nc" + , + CN_BATHYMET = "Bathymetry" + , + CN_BATHYLEV = "bathy_level" + , + CN_MBATHY = "mbathy" + + / + ! Namelist entry namsqdvar needs manual formating before + ! it can be used as input : put variables names in between ' + ! and separate variables by , + &NAMSQDVAR + NN_SQDVAR = 4, + CN_SQDVAR = "vozocrtx vomecrty vovecrtz sossheig" , + / + &NAMMESHMASK + CN_FZGR = "mesh_zgr.nc" + , + CN_FHGR = "mesh_hgr.nc" + , + CN_FMSK = "mask.nc" + , + CN_FCOO = "coordinates.nc" + , + CN_FBASINS = "new_maskglo.nc", + + / diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 3b38e077..a0a7eb7d 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -511,6 +511,11 @@ class NetCDFFile(DataFile): self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) Utils.copy_file(self.remote_file, self.local_file) + if self.data_convention == 'meteofrance': + self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', + 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', + 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', + 'y': 'j'} Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From 3f699a713c1ca8dfd393040f68c1baac8074e01c Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 4 Dec 2017 13:05:12 +0100 Subject: [PATCH 16/25] Fixed environment file --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index f804b6aa..14d62005 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - openpyxl - mock - cmake -5- coverage +- coverage - pip: - bscearth.utils -- GitLab From adb435ab835b8417aa947ea107b317c331c09122 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 09:47:59 +0100 Subject: [PATCH 17/25] Added rename call --- earthdiagnostics/datafile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index a0a7eb7d..300eb7cd 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -516,6 +516,7 @@ class NetCDFFile(DataFile): 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} + Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From 2c1843b64864ec7db48437dadd72cc2e833337e8 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:06:34 +0100 Subject: [PATCH 18/25] Added ncdumps for debugging --- earthdiagnostics/datafile.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 300eb7cd..3f253b73 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -516,7 +516,11 @@ class NetCDFFile(DataFile): 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} + Log.info('Original file') + Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) + Log.info('Renamed file') + Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From abc7050ec8b31b274099defa6c9096bdc2104437 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:11:58 +0100 Subject: [PATCH 19/25] More debug info --- earthdiagnostics/datafile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 3f253b73..202dbd2d 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -511,7 +511,9 @@ class NetCDFFile(DataFile): self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) Utils.copy_file(self.remote_file, self.local_file) + Log.warning(self.data_convention) if self.data_convention == 'meteofrance': + Log.info('Converting variable names from meteofrance convention') self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', -- GitLab From dd75ec5ef8cdf930e97d4a4499247447f86d7201 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:17:03 +0100 Subject: [PATCH 20/25] Now setting data_convention --- earthdiagnostics/datafile.py | 6 ++++-- earthdiagnostics/datamanager.py | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 202dbd2d..4606f58b 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -139,17 +139,19 @@ class DataFile(Publisher): self.dispatch(self) @classmethod - def from_storage(cls, filepath): + def from_storage(cls, filepath, data_convention): file_object = cls() file_object.remote_file = filepath file_object.local_status = LocalStatus.PENDING + file_object.data_convention = data_convention return file_object @classmethod - def to_storage(cls, remote_file): + def to_storage(cls, remote_file, data_convention): new_object = cls() new_object.remote_file = remote_file new_object.storage_status = StorageStatus.PENDING + new_object.data_convention = data_convention return new_object def download(self): diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index b4ab0124..81fd3740 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -29,7 +29,7 @@ class DataManager(object): def _get_file_from_storage(self, filepath): if filepath not in self.requested_files: - self.requested_files[filepath] = NCfile.from_storage(filepath) + self.requested_files[filepath] = NCfile.from_storage(filepath, self.config.data_convention) file_object = self.requested_files[filepath] file_object.local_satatus = LocalStatus.PENDING return self.requested_files[filepath] @@ -37,7 +37,7 @@ class DataManager(object): def _declare_generated_file(self, remote_file, domain, final_var, cmor_var, data_convention, region, diagnostic, grid, var_type, original_var): if remote_file not in self.requested_files: - self.requested_files[remote_file] = NCfile.to_storage(remote_file) + self.requested_files[remote_file] = NCfile.to_storage(remote_file, data_convention) file_object = self.requested_files[remote_file] file_object.diagnostic = diagnostic file_object.var_type = var_type @@ -48,7 +48,6 @@ class DataManager(object): file_object.final_name = final_var file_object.cmor_var = cmor_var file_object.region = region - file_object.data_convention = data_convention file_object.storage_status = StorageStatus.PENDING return file_object -- GitLab From 0712af4a17d5566097f2e30dfeb1cc1d0d787ba0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:28:48 +0100 Subject: [PATCH 21/25] Cleaned debug info --- earthdiagnostics/datafile.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 4606f58b..6c7237bc 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -513,18 +513,13 @@ class NetCDFFile(DataFile): self.local_file = TempFile.get() Utils.get_file_hash(self.remote_file, use_stored=True, save=True) Utils.copy_file(self.remote_file, self.local_file) - Log.warning(self.data_convention) if self.data_convention == 'meteofrance': - Log.info('Converting variable names from meteofrance convention') - self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds', + Log.debug('Converting variable names from meteofrance convention') + self.alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} - Log.info('Original file') - Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) - Log.info('Renamed file') - Utils.execute_shell_command('ncdump -h {0}'.format(self.local_file), Log.INFO) Log.info('File {0} ready!', self.remote_file) self.local_status = LocalStatus.READY -- GitLab From 4d725816a0e609cce4ae3bc42e17e349e87af644 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:32:23 +0100 Subject: [PATCH 22/25] Fix check compression --- earthdiagnostics/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 026c1f7d..29d86dda 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -440,7 +440,9 @@ class Utils(object): handler = Utils.openCdf(filetoconvert) if not handler.file_format == 'NETCDF4': is_compressed = False + handler.close() else: + handler.close() ncdump_result = Utils.execute_shell_command('ncdump -hs {0}'.format(filetoconvert), Log.NO_LOG) ncdump_result = ncdump_result[0].replace('\t', '').split('\n') for var in handler.variables: @@ -450,8 +452,6 @@ class Utils(object): if not '{0}:_Shuffle = "true" ;'.format(var) in ncdump_result: is_compressed = False break - - handler.close() return is_compressed -- GitLab From 3eb9e269ed4411e7477ac207a905ed313e9822f2 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 10:40:04 +0100 Subject: [PATCH 23/25] Avoid collisions between time_centered and time_counter --- earthdiagnostics/datafile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 6c7237bc..b28f319a 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -516,7 +516,6 @@ class NetCDFFile(DataFile): if self.data_convention == 'meteofrance': Log.debug('Converting variable names from meteofrance convention') self.alt_coord_names = {'time_counter': 'time', 'time_counter_bounds': 'time_bnds', - 'time_centered': 'time', 'time_centered_bnds': 'time_bnds', 'tbnds': 'bnds', 'nav_lat': 'lat', 'nav_lon': 'lon', 'x': 'i', 'y': 'j'} Utils.rename_variables(self.local_file, self.alt_coord_names, must_exist=False, rename_dimension=True) -- GitLab From 51824178c178c588b64671afd8f8313bc8821783 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 11:02:00 +0100 Subject: [PATCH 24/25] Add HDF5 flag --- earthdiagnostics/earthdiags.py | 1 + earthdiagnostics/utils.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 44c7fba5..c8a489b9 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -46,6 +46,7 @@ class EarthDiags(object): def __init__(self, config_file): Log.info('Initialising Earth Diagnostics Version {0}', EarthDiags.version) self.config = Config(config_file) + os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' TempFile.scratch_folder = self.config.scratch_dir cdftools.path = self.config.cdftools_path diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 29d86dda..b91fda94 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -390,7 +390,8 @@ class Utils(object): Log.log.log(log_level, line) output.append(line) if process.returncode != 0: - raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), process.returncode)) + raise Utils.ExecutionError('Error executing {0}\n Return code: {1}'.format(' '.join(command), + str(process.returncode))) return output _cpu_count = None -- GitLab From 22693537b523e9f4f89d490c74690703646ccb84 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Tue, 5 Dec 2017 11:50:07 +0100 Subject: [PATCH 25/25] Now user can specify STARTDATES in the format {START,END,INTERVAL} --- earthdiagnostics/config.py | 37 ++++++++++++++++++++++++++++++++----- test/unit/test_config.py | 21 ++++++++++++++++++++- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index f5d4c3c4..e0dfa50b 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -3,7 +3,7 @@ import os import six from bscearth.utils.config_parser import ConfigParser -from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str, add_years, add_months, add_days from bscearth.utils.log import Log from earthdiagnostics import cdftools @@ -290,25 +290,52 @@ class ExperimentConfig(object): mem = mem[len(self.member_prefix):] members.append(int(mem)) self.members = members - + self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') startdates = parser.get_list_option('EXPERIMENT', 'STARTDATES') import exrex self.startdates = [] for startdate_pattern in startdates: - for startdate in exrex.generate(startdate_pattern): - self.startdates.append(startdate) + if startdate_pattern[0] == '{' and startdate_pattern[-1] == '}': + self._read_startdates(startdate_pattern[1:-1]) + else: + for startdate in exrex.generate(startdate_pattern): + self.startdates.append(startdate) self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') self.chunk_list = parser.get_int_list_option('EXPERIMENT', 'CHUNK_LIST', []) - self.calendar = parser.get_option('EXPERIMENT', 'CALENDAR', 'standard') + self.model = parser.get_option('EXPERIMENT', 'MODEL') self.model_version = parser.get_option('EXPERIMENT', 'MODEL_VERSION', '') self.atmos_grid = parser.get_option('EXPERIMENT', 'ATMOS_GRID', '') self.atmos_timestep = parser.get_int_option('EXPERIMENT', 'ATMOS_TIMESTEP', 6) self.ocean_timestep = parser.get_int_option('EXPERIMENT', 'OCEAN_TIMESTEP', 6) + def _read_startdates(self, pattern): + pattern = pattern.split(',') + start = parse_date(pattern[0].strip()) + end = parse_date(pattern[1].strip()) + interval = pattern[2].strip() + if len(interval) == 1: + factor = 1 + else: + factor = int(interval[0:-1]) + interval = interval[-1].upper() + while start <= end: + self.startdates.append(date2str(start)) + if interval == 'Y': + start = add_years(start, factor) + elif interval == 'M': + start = add_months(start, factor, cal=self.calendar) + elif interval == 'W': + start = add_days(start, factor * 7, cal=self.calendar) + elif interval == 'D': + start = add_days(start, factor, cal=self.calendar) + else: + raise ConfigException('Interval {0} not supported in STARTDATES definition: {1}', interval, pattern) + + def get_chunk_list(self): """ Return a list with all the chunks diff --git a/test/unit/test_config.py b/test/unit/test_config.py index a1c8d25b..b113d8bd 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -306,9 +306,28 @@ class TestExperimentConfig(TestCase): self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200[0-2](02|05|08|11)01') config = ExperimentConfig(self.mock_parser) - print(config.startdates) self.assertEquals(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', u'20020801', u'20021101']) + def test_auto_startdates(self): + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,1Y}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20011101,6M}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20010501', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,1W}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7D}') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20001108', '20001115', '20001122', '20001129']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '{20001101,20001201,7F}') + with self.assertRaises(ConfigException): + ExperimentConfig(self.mock_parser) -- GitLab