diff --git a/.gitmodules b/.gitmodules index ee454f263f45cb841deeeca4641091235517f4ca..9009fcce0a1d3b740c2178b8c76c25d5e2651015 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "earthdiagnostics/cmor_tables/cmip6"] path = earthdiagnostics/cmor_tables/cmip6 url = https://github.com/jvegasbsc/cmip6-cmor-tables.git +[submodule "earthdiagnostics/cmor_tables/primavera"] + path = earthdiagnostics/cmor_tables/primavera + url = https://github.com/jonseddon/cmip6-cmor-tables.git diff --git a/VERSION b/VERSION index 996c69056fb9ede4b8ff8de2ef3916945a484432..e3ab2e1647bd77cac63f130c36b9a7641126e11d 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0b44 +3.0.0b45 diff --git a/doc/source/conf.py b/doc/source/conf.py index d749566967f93f1f463d5de176753fb529985382..9cba69d08275cf054174283fe07fc5ef73c8872b 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b44' +release = '3.0.0b45' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index ed0000d9001949b30945bae659f339eded7a885c..7e31e14c4941733d1b52c318ca8e3476fec22e2c 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cmor_tables/default.csv b/earthdiagnostics/cmor_tables/default.csv index fabfbfbb20c504562c5ffe82db00f5c1ea94d6ad..9224e2c95a3a517bf6aa1af7275bfc8e4f83de54 100644 --- a/earthdiagnostics/cmor_tables/default.csv +++ b/earthdiagnostics/cmor_tables/default.csv @@ -19,7 +19,8 @@ lcc,cll,low_cloud_area_fraction,Low cloud fraction,atmos,,,,,, mcc,clm,medium_cloud_area_fraction,Medium cloud fraction,atmos,,,,,, ciwc,cli,mass_fraction_of_cloud_ice_in_air,Mass fraction of cloud ice,atmos,,,,,, tcc,clt,cloud_area_fraction,Total cloud fraction,atmos,,,,,, -clwc,clw,mass_fraction_of_cloud_liquid_water_in_air,Mass fraction of cloud liquid water,atmos,,,,,, +clw,clw,mass_fraction_of_cloud_liquid_water_in_air,Mass fraction of cloud liquid water,atmos,,,,,, +clwc,clwc,mass_fraction_of_convective_cloud_liquid_water_in_air,Mass fraction of convective cloud liquid water,atmos,,,,,, tcw,clwvi,atmosphere_cloud_condensed_water_content,Condensed water path,atmos,,,,,, iicedive:sidive,divice,Strain Rate Divergence of Sea Ice,Divergence_of_sea_ice_velocity,seaIce,,,,,, e,evspsbl,water_evaporation_flux,Evaporation,atmos,,,,,, diff --git a/earthdiagnostics/cmor_tables/primavera b/earthdiagnostics/cmor_tables/primavera new file mode 160000 index 0000000000000000000000000000000000000000..cbc0ea84e67caa713638e219d31a11dcc4b45039 --- /dev/null +++ b/earthdiagnostics/cmor_tables/primavera @@ -0,0 +1 @@ +Subproject commit cbc0ea84e67caa713638e219d31a11dcc4b45039 diff --git a/earthdiagnostics/cmor_tables/primavera.xlsx b/earthdiagnostics/cmor_tables/primavera.xlsx deleted file mode 100644 index cd537007ea823e4d846766b6fe2ccf42f2d5afb3..0000000000000000000000000000000000000000 Binary files a/earthdiagnostics/cmor_tables/primavera.xlsx and /dev/null differ diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 6be46960233e9aaa5614c013c659008e247d63d3..4addc828ac4f1d655b2209ab4af92d19bed5c0cf 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -322,7 +322,10 @@ class Cmorizer(object): for variable in handler.variables.keys(): if variable in Cmorizer.NON_DATA_VARIABLES: continue - self.extract_variable(filename, handler, frequency, variable) + try: + self.extract_variable(filename, handler, frequency, variable) + except Exception as ex: + Log.error('Variable {0} can not be cmorized: {1}', variable, ex) Log.result('File {0} cmorized!', filename) handler.close() os.remove(filename) diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 40b30b77ab276f9858c6ce9e71f9e7a22fb9c4c4..bdc06a651488c3220425b9f191bb350bb4777e35 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -7,6 +7,7 @@ from datetime import datetime import numpy as np import os import re +from bscearth.utils.log import Log from cfunits import Units from earthdiagnostics.utils import Utils, TempFile @@ -404,7 +405,9 @@ class NetCDFFile(object): def _convert_units(self, var_handler): try: Utils.convert_units(var_handler, self.cmor_var.units) - except ValueError: + except ValueError as ex: + Log.warning('Can not convert {3} from {0} to {1}: {2}', var_handler.units, self.cmor_var.units, ex, + self.cmor_var.short_name) factor, offset = UnitConversion.get_conversion_factor_offset(var_handler.units, self.cmor_var.units) @@ -414,8 +417,6 @@ class NetCDFFile(object): if 'valid_max' in var_handler.ncattrs(): var_handler.valid_max = float(var_handler.valid_max) * factor + offset - - def _rename_coordinate_variables(self): variables = dict() variables['x'] = 'i' diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 40dd535324e8053a4e3b653362a0f056b756238e..bb39bf42a418b2a5d4809f32125d9563be246bcb 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -586,6 +586,8 @@ class Utils(object): @staticmethod def give_group_write_permissions(path): st = os.stat(path) + if st.st_mode & stat.S_IWGRP: + return os.chmod(path, st.st_mode | stat.S_IWGRP) @staticmethod diff --git a/earthdiagnostics/variable_alias/default.csv b/earthdiagnostics/variable_alias/default.csv index f78af47e0ccbb8fe8b9bfd0e4a80507211af9695..4a46d64d9832fbb0ac17699fad9d5d4a7054d1a5 100644 --- a/earthdiagnostics/variable_alias/default.csv +++ b/earthdiagnostics/variable_alias/default.csv @@ -296,5 +296,4 @@ difvho,difvho,, vovematr,wmo,, qtr_ice,qtr,, var78,tclw,, -var79,tciw,, -clwc,clw,, \ No newline at end of file +var79,tciw,, \ No newline at end of file diff --git a/model_diags.conf b/model_diags.conf index 0ac9a54e9f8fe1b83ed0a85a7a24ca59f22a0953..b55c2bc026459ee0da1c4f1b60e85e36ff0b043e 100644 --- a/model_diags.conf +++ b/model_diags.conf @@ -111,14 +111,18 @@ CHUNKS = [CMOR] # If true, recreates CMOR files regardless of presence. Default = False -FORCE = False +# FORCE = False + # If true, CMORizes ocean files. Default = True -OCEAN_FILES = True -FILTER_FILES = +# OCEAN_FILES = True + +# FILTER_FILES = + # If true, CMORizes atmosphere files. Default = True -ATMOSPHERE_FILES = False +# ATMOSPHERE_FILES = True + # You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var -VARIABLE_LIST = +# VARIABLE_LIST = # Variables to be CMORized from the grib atmospheric files, separated by comma. # You can also specify the levels to extract using the following syntax