diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c5b843a22f1041193551e84ad2f3fac189c06f20..46393bc61b9feb23420e155b8cbbfc8ba764b318 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,21 +26,21 @@ test_python2: - source activate earthdiagnostics2 - python run_test.py -test_python3: - stage: test - script: - - git submodule sync --recursive - - git submodule update --init --recursive - - conda env update -f environment.yml -n earthdiagnostics3 python=3.6 - - source activate earthdiagnostics3 - - python run_test.py +#test_python3: +# stage: test +# script: +# - git submodule sync --recursive +# - git submodule update --init --recursive +# - conda env update -f environment.yml -n earthdiagnostics3 python=3.6 +# - source activate earthdiagnostics3 +# - python run_test.py report_codacy: stage: report script: - source activate earthdiagnostics3 - pip install codacy-coverage --upgrade - - python-codacy-coverage -r test/report/python3/coverage.xml + - python-codacy-coverage -r test/report/python2/coverage.xml clean: stage: clean diff --git a/VERSION b/VERSION index 7a113d6c3a110bd924a12ac1cac92cf8f78e11fc..53f0501f4c2ace879ebefbe4942c2977dd23603d 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc10 +3.0.0rc11 diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 7e80ebb04ebeb2150129ac1d3a85d47acf15c37a..73da16ba7f90f894f4c5a755d25ebda3b40944fe 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -5,8 +5,8 @@ import os import shutil import uuid import traceback -import pygrib -from datetime import datetime +import eccodes +from datetime import datetime, timedelta import six from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months @@ -352,16 +352,19 @@ class Cmorizer(object): def _get_atmos_timestep(self, gribfile): Log.info('Getting timestep...') - grib_handler = pygrib.open(gribfile) - dates = set() - for mes in grib_handler: - dates.add(mes.validDate) + with eccodes.GribFile(gribfile) as grib: + dates = set() + for mes in grib: + try: + dates.add(mes['validDate']) + except Exception: + dates.add(parse_date(str(mes['date'])) + timedelta(hours=int(mes['stepRange']))) + print(dates) dates = list(dates) dates.sort() atmos_timestep = dates[1] - dates[0] atmos_timestep = int(atmos_timestep.total_seconds() / 3600) self.experiment.atmos_timestep = atmos_timestep - grib_handler.close() return atmos_timestep def _cmorize_nc_file(self, filename): diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index c1df586be11cb9b4e14bccff64e36dce22c6d8d3..18e8455116620399e40b7d11817f45a5ae033ded 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -145,7 +145,7 @@ class Siasiesiv(Diagnostic): handler = Utils.open_cdf(self.sit.local_file) handler.variables[self.sit_varname].coordinates = coordinates handler.close() - sit = iris.load_cube(self.sic.local_file) + sit = iris.load_cube(self.sit.local_file) for basin, mask in six.iteritems(self.masks): self.results['siarean'][basin] = self.sum(sic, mask, north=True) diff --git a/environment.yml b/environment.yml index c64b1ce5a52dfc80936e9fb0859de1b35b9baf1f..504f10397d8e5985d596edd9423877071e984bb1 100644 --- a/environment.yml +++ b/environment.yml @@ -11,14 +11,15 @@ dependencies: - cdo=1.9.2 - nco - python-cdo +- python-eccodes - coverage -- pygrib - psutil - six - cf_units - openpyxl + +# testing - mock -- cmake - coverage - pytest - pytest-cov @@ -30,5 +31,7 @@ dependencies: - nco - exrex - xxhash + + # testing - pytest-profiling - dummydata diff --git a/fix_oiday.py b/fix_oiday.py new file mode 100644 index 0000000000000000000000000000000000000000..d649c2d43ce8ae82d177735990fb499919ae548f --- /dev/null +++ b/fix_oiday.py @@ -0,0 +1,23 @@ +import glob +import shutil +import os +import netCDF4 + +def main(): + + for path in glob.glob('/esarchive/exp/ecearth/a0pe/cmorfiles/BSC/EC-EARTH3/a0pe/*/day/*/*/*/*_Oday_*'): + day_path =path.replace('_OIday_', '_day_') + if os.path.isfile(day_path): + print('del {0}'.format(path)) + os.remove(path) + else: + print('move {0}'.format(path)) + handler = netCDF4.Dataset(path, 'a') + handler.table_id = "Table day (December 2013)" + handler.close() + shutil.move(path, day_path) + + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 81ea7ac88c192975f366114399a03b20da94823f..0dc78f1028a50de42c9a55ed58744a0ecacdb0d2 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ setup( keywords=['climate', 'weather', 'diagnostic'], setup_requires=['pyproj'], install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo>=1.3.4', 'nco>=0.0.3', 'iris>=1.12.0', 'coverage', - 'pygrib', 'openpyxl', 'mock', 'futures', 'cf_units', 'xxhash', 'six', 'psutil', + 'openpyxl', 'mock', 'futures', 'xxhash', 'six', 'psutil', 'eccodes', 'exrex'], packages=find_packages(), include_package_data=True,