diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e6fe6598640321a8250e5fce9d5b035e76ccb79f..684f121341e24b837b644b54cb7e14ed4fd0cdbf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,9 @@ test: - git submodule update --init --recursive - conda env update -f environment.yml -n earthdiagnostics3 python=3.7 - source activate earthdiagnostics3 + - rm -r ../diagonals + - git clone https://earth.bsc.es/gitlab/es/diagonals.git ../diagonals + - pip install ../diagonals - pip install .[develop] - python setup.py test diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index f964024057b9a336678036e0c998fdbd4e4233b7..accc2c4996e8031a257d6d6a60917d7a07fbbc02 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -65,8 +65,6 @@ class DataFile(Publisher): self.job_added = False self._modifiers = [] self._size = None - self.lon_name = None - self.lat_name = None def __str__(self): return 'Data file for {0}'.format(self.remote_file) @@ -239,13 +237,6 @@ class DataFile(Publisher): This includes renaming the variable if necessary, updating the metadata and adding the history and managing the possibility of multiple regions """ - if self.data_convention in ('primavera', 'cmip6'): - self.lon_name = 'longitude' - self.lat_name = 'latitude' - else: - self.lon_name = 'lon' - self.lat_name = 'lat' - if rename_var: original_name = rename_var else: @@ -309,7 +300,8 @@ class DataFile(Publisher): var_handler = handler.variables[self.final_name] coords = set.intersection( { - 'time', 'lev', self.lat_name, self.lon_name, + 'time', 'lev', + self.data_convention.lat_name, self.data_convention.lon_name, 'leadtime', 'region', 'time_centered' }, set(handler.variables.keys()) @@ -365,16 +357,18 @@ class DataFile(Publisher): ) def _fix_coordinate_variables_metadata(self, handler): + lat_name = self.data_convention.lat_name + lon_name = self.data_convention.lon_name if 'lev' in handler.variables: handler.variables['lev'].short_name = 'lev' if self.domain == ModelingRealms.ocean: handler.variables['lev'].standard_name = 'depth' - if self.lon_name in handler.variables: - handler.variables[self.lon_name].short_name = self.lon_name - handler.variables[self.lon_name].standard_name = 'longitude' - if self.lat_name in handler.variables: - handler.variables[self.lat_name].short_name = self.lat_name - handler.variables[self.lat_name].standard_name = 'latitude' + if lon_name in handler.variables: + handler.variables[lon_name].short_name = lon_name + handler.variables[lon_name].standard_name = 'longitude' + if lat_name in handler.variables: + handler.variables[lat_name].short_name = lat_name + handler.variables[lat_name].standard_name = 'latitude' def _fix_units(self, var_handler): if 'units' not in var_handler.ncattrs(): @@ -417,9 +411,13 @@ class DataFile(Publisher): try: old_cube = iris.load_cube(self.remote_file) except Exception: - old_cube = iris.load_cube( - self.remote_file.replace('/cmorfiles/', '/diags/') - ) + try: + old_cube = iris.load_cube( + self.remote_file.replace('/cmorfiles/', '/diags/') + ) + except Exception: + # Bad data, overwrite + return new_data = {} for region_slice in cube.slices_over('region'): Log.debug(region_slice.coord('region').points[0]) @@ -438,6 +436,12 @@ class DataFile(Publisher): final_cube = cube_list.merge_cube() temp = TempFile.get() iris.save(final_cube, temp, zlib=True) + if '-' in final_cube.var_name: + Utils.rename_variable( + temp, + final_cube.var_name.replace('-', '_'), + final_cube.var_name, must_exist=False + ) Utils.move_file(temp, self.local_file) self._correct_metadata() @@ -445,12 +449,12 @@ class DataFile(Publisher): variables = dict() variables['x'] = 'i' variables['y'] = 'j' - variables['nav_lat_grid_V'] = self.lat_name - variables['nav_lon_grid_V'] = self.lon_name - variables['nav_lat_grid_U'] = self.lat_name - variables['nav_lon_grid_U'] = self.lon_name - variables['nav_lat_grid_T'] = self.lat_name - variables['nav_lon_grid_T'] = self.lon_name + variables['nav_lat_grid_V'] = self.data_convention.lat_name + variables['nav_lon_grid_V'] = self.data_convention.lon_name + variables['nav_lat_grid_U'] = self.data_convention.lat_name + variables['nav_lon_grid_U'] = self.data_convention.lon_name + variables['nav_lat_grid_T'] = self.data_convention.lat_name + variables['nav_lon_grid_T'] = self.data_convention.lon_name Utils.rename_variables(self.local_file, variables, False) def add_diagnostic_history(self):