From 6ea3de445de9fd75232ab9e19677df06bfb16dd0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 27 Jan 2020 12:25:26 +0100 Subject: [PATCH 1/4] Update CI --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e6fe6598..9c0a667e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,8 @@ test: - git submodule update --init --recursive - conda env update -f environment.yml -n earthdiagnostics3 python=3.7 - source activate earthdiagnostics3 + - git clone https://earth.bsc.es/gitlab/es/diagonals.git ../diagonals + - pip install ../diagonals - pip install .[develop] - python setup.py test -- GitLab From a37c213dd393175d5c62274a041377280a5549d6 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 27 Jan 2020 12:36:27 +0100 Subject: [PATCH 2/4] Update CI --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c0a667e..684f1213 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,7 @@ test: - git submodule update --init --recursive - conda env update -f environment.yml -n earthdiagnostics3 python=3.7 - source activate earthdiagnostics3 + - rm -r ../diagonals - git clone https://earth.bsc.es/gitlab/es/diagonals.git ../diagonals - pip install ../diagonals - pip install .[develop] -- GitLab From 612bfe0000efe94b54a99562b61dde11972a00a0 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 27 Jan 2020 14:24:23 +0100 Subject: [PATCH 3/4] Fix update regions when - in var name --- earthdiagnostics/datafile.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index f9640240..1be79913 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -438,6 +438,12 @@ class DataFile(Publisher): final_cube = cube_list.merge_cube() temp = TempFile.get() iris.save(final_cube, temp, zlib=True) + if '-' in final_cube.var_name: + Utils.rename_variable( + temp, + final_cube.var_name.replace('-', '_'), + final_cube.var_name, must_exist=False + ) Utils.move_file(temp, self.local_file) self._correct_metadata() -- GitLab From 26c2ba7b91db2970f8370214a8505737b59435ab Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Mon, 27 Jan 2020 15:13:11 +0100 Subject: [PATCH 4/4] Fix datafile management of lat lon names --- earthdiagnostics/datafile.py | 48 +++++++++++++++++------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 1be79913..accc2c49 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -65,8 +65,6 @@ class DataFile(Publisher): self.job_added = False self._modifiers = [] self._size = None - self.lon_name = None - self.lat_name = None def __str__(self): return 'Data file for {0}'.format(self.remote_file) @@ -239,13 +237,6 @@ class DataFile(Publisher): This includes renaming the variable if necessary, updating the metadata and adding the history and managing the possibility of multiple regions """ - if self.data_convention in ('primavera', 'cmip6'): - self.lon_name = 'longitude' - self.lat_name = 'latitude' - else: - self.lon_name = 'lon' - self.lat_name = 'lat' - if rename_var: original_name = rename_var else: @@ -309,7 +300,8 @@ class DataFile(Publisher): var_handler = handler.variables[self.final_name] coords = set.intersection( { - 'time', 'lev', self.lat_name, self.lon_name, + 'time', 'lev', + self.data_convention.lat_name, self.data_convention.lon_name, 'leadtime', 'region', 'time_centered' }, set(handler.variables.keys()) @@ -365,16 +357,18 @@ class DataFile(Publisher): ) def _fix_coordinate_variables_metadata(self, handler): + lat_name = self.data_convention.lat_name + lon_name = self.data_convention.lon_name if 'lev' in handler.variables: handler.variables['lev'].short_name = 'lev' if self.domain == ModelingRealms.ocean: handler.variables['lev'].standard_name = 'depth' - if self.lon_name in handler.variables: - handler.variables[self.lon_name].short_name = self.lon_name - handler.variables[self.lon_name].standard_name = 'longitude' - if self.lat_name in handler.variables: - handler.variables[self.lat_name].short_name = self.lat_name - handler.variables[self.lat_name].standard_name = 'latitude' + if lon_name in handler.variables: + handler.variables[lon_name].short_name = lon_name + handler.variables[lon_name].standard_name = 'longitude' + if lat_name in handler.variables: + handler.variables[lat_name].short_name = lat_name + handler.variables[lat_name].standard_name = 'latitude' def _fix_units(self, var_handler): if 'units' not in var_handler.ncattrs(): @@ -417,9 +411,13 @@ class DataFile(Publisher): try: old_cube = iris.load_cube(self.remote_file) except Exception: - old_cube = iris.load_cube( - self.remote_file.replace('/cmorfiles/', '/diags/') - ) + try: + old_cube = iris.load_cube( + self.remote_file.replace('/cmorfiles/', '/diags/') + ) + except Exception: + # Bad data, overwrite + return new_data = {} for region_slice in cube.slices_over('region'): Log.debug(region_slice.coord('region').points[0]) @@ -451,12 +449,12 @@ class DataFile(Publisher): variables = dict() variables['x'] = 'i' variables['y'] = 'j' - variables['nav_lat_grid_V'] = self.lat_name - variables['nav_lon_grid_V'] = self.lon_name - variables['nav_lat_grid_U'] = self.lat_name - variables['nav_lon_grid_U'] = self.lon_name - variables['nav_lat_grid_T'] = self.lat_name - variables['nav_lon_grid_T'] = self.lon_name + variables['nav_lat_grid_V'] = self.data_convention.lat_name + variables['nav_lon_grid_V'] = self.data_convention.lon_name + variables['nav_lat_grid_U'] = self.data_convention.lat_name + variables['nav_lon_grid_U'] = self.data_convention.lon_name + variables['nav_lat_grid_T'] = self.data_convention.lat_name + variables['nav_lon_grid_T'] = self.data_convention.lon_name Utils.rename_variables(self.local_file, variables, False) def add_diagnostic_history(self): -- GitLab