Commit 638dd6e9 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Merge branch 'master' into 'production'

Master

See merge request !52
parents 7d299458 c8e5966f
......@@ -447,37 +447,8 @@ class Cmorizer(object):
return
temp = TempFile.get()
handler = Utils.open_cdf(file_path)
coords = [self.lon_name, self.lat_name, 'time']
if 'leadtime' in handler.variables.keys():
coords.append('leadtime')
if var_cmor.domain == ModelingRealms.ocean:
lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev',
'depth': 'lev'}
elif var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]:
lev_dimensions = {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth',
'depth_4': 'sdepth'}
elif var_cmor.domain == ModelingRealms.atmos:
lev_dimensions = {'depth': 'plev'}
else:
lev_dimensions = {}
for lev_dim in lev_dimensions.keys():
if lev_dim in handler.variables[variable].dimensions:
coords.append(lev_dim)
handler.variables[variable].coordinates = ' '.join(set(coords))
handler.close()
cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable))
for lev_original, lev_target in six.iteritems(lev_dimensions):
try:
cube.coord(lev_original).var_name = lev_target
except iris.exceptions.CoordinateNotFoundError:
pass
iris.save(cube, temp, zlib=True)
lev_dimensions = self._set_coordinates_attribute(file_path, var_cmor, variable)
self._rename_level_coords(file_path, lev_dimensions, temp, variable)
if alias.basin is None:
region = None
......@@ -516,6 +487,41 @@ class Cmorizer(object):
region_str = ''
Log.info('Variable {0.domain}:{0.short_name} processed{1}', var_cmor, region_str)
def _rename_level_coords(self, file_path, lev_dimensions, temp, variable):
cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable))
for lev_original, lev_target in six.iteritems(lev_dimensions):
try:
cube.coord(lev_original).var_name = lev_target
except iris.exceptions.CoordinateNotFoundError:
pass
iris.save(cube, temp, zlib=True)
def _set_coordinates_attribute(self, file_path, var_cmor, variable):
handler = Utils.open_cdf(file_path)
coords = [self.lon_name, self.lat_name, 'time']
if 'leadtime' in handler.variables.keys():
coords.append('leadtime')
lev_dimensions = self._get_lev_dimensions(var_cmor)
for lev_dim in lev_dimensions.keys():
if lev_dim in handler.variables[variable].dimensions:
coords.append(lev_dim)
handler.variables[variable].coordinates = ' '.join(set(coords))
handler.close()
return lev_dimensions
def _get_lev_dimensions(self, var_cmor):
if var_cmor.domain == ModelingRealms.ocean:
lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev',
'depth': 'lev'}
elif var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]:
lev_dimensions = {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth',
'depth_4': 'sdepth'}
elif var_cmor.domain == ModelingRealms.atmos:
lev_dimensions = {'depth': 'plev'}
else:
lev_dimensions = {}
return lev_dimensions
def _get_date_str(self, file_path):
file_parts = os.path.basename(file_path).split('_')
valid_starts = (self.experiment.expid, 'MMA', 'MMASH', 'MMAGG', 'MMO')
......
......@@ -138,32 +138,10 @@ class Config(object):
self.mask_regions = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS', '')
self.mask_regions_3d = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS_3D', '')
data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION',
('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'),
'specs',
ignore_case=True)
if data_convention == 'specs':
self.data_convention = SPECSConvention(data_convention, self)
elif data_convention == 'primavera':
self.data_convention = PrimaveraConvention(data_convention, self)
elif data_convention == 'cmip6':
self.data_convention = CMIP6Convention(data_convention, self)
elif data_convention == 'preface':
self.data_convention = PrefaceConvention(data_convention, self)
elif data_convention == 'meteofrance':
self.data_convention = MeteoFranceConvention(data_convention, self)
self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks)
namelist_file = os.path.join(os.path.dirname(__file__),
'CDFTOOLS_{0}.namlist'.format(self.data_convention.name))
Log.debug(namelist_file)
if os.path.isfile(namelist_file):
Log.debug('Setting namelist {0}', namelist_file)
os.environ['NAM_CDF_NAMES'] = namelist_file
self._parse_dataconvention(parser)
self.var_manager = VariableManager()
self.var_manager.load_variables(data_convention)
self.var_manager.load_variables(self.data_convention.name)
self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS')
self.skip_diags_done = parser.get_bool_option('DIAGNOSTICS', 'SKIP_DIAGS_DONE', True)
self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY'))
......@@ -185,6 +163,16 @@ class Config(object):
Log.debug('Preparing command list')
commands = self._diags.split()
self._real_commands = list()
self._apply_aliases(commands)
Log.debug('Command list ready ')
self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid)
self.cmor = CMORConfig(parser, self.var_manager)
self.thredds = THREDDSConfig(parser)
self.report = ReportConfig(parser)
def _apply_aliases(self, commands):
for command in commands:
command = command.strip()
if command.startswith('#'):
......@@ -196,13 +184,28 @@ class Config(object):
self._real_commands.append(add_command)
else:
self._real_commands.append(command)
Log.debug('Command list ready ')
self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid)
self.cmor = CMORConfig(parser, self.var_manager)
self.thredds = THREDDSConfig(parser)
self.report = ReportConfig(parser)
def _parse_dataconvention(self, parser):
data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION',
('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'),
'specs',
ignore_case=True)
if data_convention == 'specs':
self.data_convention = SPECSConvention(data_convention, self)
elif data_convention == 'primavera':
self.data_convention = PrimaveraConvention(data_convention, self)
elif data_convention == 'cmip6':
self.data_convention = CMIP6Convention(data_convention, self)
elif data_convention == 'preface':
self.data_convention = PrefaceConvention(data_convention, self)
elif data_convention == 'meteofrance':
self.data_convention = MeteoFranceConvention(data_convention, self)
self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks)
namelist_file = os.path.join(os.path.dirname(__file__),
'CDFTOOLS_{0}.namlist'.format(self.data_convention.name))
Log.debug(namelist_file)
Log.debug('Setting namelist {0}', namelist_file)
os.environ['NAM_CDF_NAMES'] = namelist_file
def get_commands(self):
"""
......
"""Module to manage the different data conventions supported by EarthDiagnostics"""
import os
import shutil
import re
......@@ -24,6 +25,21 @@ class DataConvention(object):
self.lock = threading.Lock()
self._checked_vars = list()
def get_scratch_masks(self, scratch_masks):
"""
Get the final scratch_masks path
Parameters
----------
scratch_masks: str
Returns
-------
str
"""
return scratch_masks
def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency,
grid=None, year=None, date_str=None):
"""
......@@ -262,6 +278,8 @@ class DataConvention(object):
self.lock.release()
def _get_time_component(self, chunk, date_str, frequency, startdate, year):
if len([x for x in (chunk, date_str, year) if x is not None]) > 1:
raise ValueError('Only one of the parameters chunk, year or date_str may be provided')
if chunk is not None:
time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency)
elif year:
......@@ -296,16 +314,54 @@ class DataConvention(object):
return current_count
def is_cmorized(self, startdate, member, chunk, domain):
"""
Check if a given chunk is cmorized for a given domain
Parameters
----------
startdate: str
member: str
chunk: int
domain: ModelingRealm
Returns
-------
bool
Raises
------
NotImplementedError:
If not implemented by the derived classes
"""
raise NotImplementedError
class Cmor2Convention(DataConvention):
"""Base class for CMOR2-based conventions"""
def get_scratch_masks(self, scratch_masks):
return scratch_masks
def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ):
"""
Get filename for a given configuration
Parameters
----------
startdate: str
member: int
domain: ModelingRealm
var: str
cmor_var: Variable
frequency: Frequency
chunk: int or None
year: int or None
date_str: str or None
grid: str or None
Returns
-------
str
"""
if cmor_var is None:
cmor_table = domain.get_table(frequency, self.config.data_convention)
else:
......@@ -329,6 +385,23 @@ class Cmor2Convention(DataConvention):
return folder_path
def get_member_str(self, member):
"""
Transalate member number to member string
Parameters
----------
member: int
Returns
-------
str
Raises
------
NotImplementedError:
If not implemented by derived classes
"""
template = 'r{0}i{1}p1'
return template.format(member + 1 - self.config.experiment.member_count_start,
self.config.cmor.initialization_number)
......@@ -346,14 +419,29 @@ class Cmor2Convention(DataConvention):
for name in os.listdir(os.path.join(path, freq, domain, var, member)):
filepath = os.path.join(path, freq, domain, var, member, name)
if os.path.isfile(filepath):
self.create_link(domain, filepath, frequency, var, "", False,
self.create_link(ModelingRealms.parse(domain), filepath, frequency, var, "", False,
vartype=VariableType.MEAN)
else:
for filename in os.listdir(filepath):
self.create_link(domain, os.path.join(filepath, filename), frequency, var,
"", False, vartype=VariableType.MEAN)
self.create_link(ModelingRealms.parse(domain), os.path.join(filepath, filename),
frequency, var, "", False, vartype=VariableType.MEAN)
def is_cmorized(self, startdate, member, chunk, domain):
"""
Check if a given chunk is cmorized for a given domain
Parameters
----------
startdate: str
member: str
chunk: int
domain: ModelingRealm
Returns
-------
bool
"""
startdate_path = self.get_startdate_path(startdate)
if not os.path.isdir(startdate_path):
return False
......@@ -371,6 +459,18 @@ class SPECSConvention(Cmor2Convention):
"""Base class for CMOR2-based conventions"""
def get_startdate_path(self, startdate):
"""
Return the path to the startdate's CMOR folder
Parameters
----------
startdate: str
Returns
-------
str
"""
return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles',
self.config.experiment.institute,
self.config.experiment.model, self.experiment_name(startdate), 'S' + startdate)
......@@ -386,11 +486,24 @@ class PrefaceConvention(Cmor2Convention):
config: Config
"""
def __init__(self, name, config):
super(PrefaceConvention, self).__init__(name, config)
self.time_separator = '_'
def get_startdate_path(self, startdate):
"""
Return the path to the startdate's CMOR folder
Parameters
----------
startdate: str
Returns
-------
str
"""
return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles',
self.config.experiment.institute,
self.experiment_name(startdate), 'S' + startdate)
......@@ -406,15 +519,51 @@ class Cmor3Convention(DataConvention):
config: Config
"""
def __init__(self, name, config):
super(Cmor3Convention, self).__init__(name, config)
self.lat_name = 'latitude'
self.lon_name = 'longitude'
def get_scratch_masks(self, scratch_masks):
"""
Get the final scratch_masks path
Adds a folder matching the convention name to the configured path
Parameters
----------
scratch_masks: str
Returns
-------
str
"""
return os.path.join(scratch_masks, self.name)
def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ):
"""
Get filename for a given configuration
Parameters
----------
startdate: str
member: int
domain: ModelingRealm
var: str
cmor_var: Variable
frequency: Frequency
chunk: int or None
year: int or None
date_str: str or None
grid: str or None
Returns
-------
str
"""
if cmor_var is None:
cmor_table = domain.get_table(frequency, self.config.data_convention)
else:
......@@ -497,11 +646,38 @@ class Cmor3Convention(DataConvention):
False, vartype=VariableType.MEAN)
def get_member_str(self, member):
"""
Transalate member number to member string
Parameters
----------
member: int
Returns
-------
str
"""
template = 'r{0}i{1}p1f1'
return template.format(member + 1 - self.config.experiment.member_count_start,
self.config.cmor.initialization_number)
def is_cmorized(self, startdate, member, chunk, domain):
"""
Check if a given chunk is cmorized for a given domain
Parameters
----------
startdate: str
member: str
chunk: int
domain: ModelingRealm
Returns
-------
bool
"""
startdate_path = self.get_startdate_path(startdate)
if not os.path.isdir(startdate_path):
return False
......@@ -522,11 +698,13 @@ class Cmor3Convention(DataConvention):
class CMIP6Convention(Cmor3Convention):
"""Class managing CMIP6 file conventions"""
pass
class PrimaveraConvention(Cmor3Convention):
"""Class managing Primavera file conventions"""
pass
......@@ -534,6 +712,27 @@ class MeteoFranceConvention(DataConvention):
"""Class managing MeteoFrance file conventions"""
def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,):
"""
Get filename for a given configuration
Parameters
----------
startdate: str
member: int
domain: ModelingRealm
var: str
cmor_var: Variable
frequency: Frequency
chunk: int or None
year: int or None
date_str: str or None
grid: str or None
Returns
-------
str
"""
if year is not None:
raise ValueError('Year not supported with MeteoFrance convention')
if date_str is not None:
......@@ -551,6 +750,18 @@ class MeteoFranceConvention(DataConvention):
return folder_path
def get_member_str(self, member):
"""
Transalate member number to member string
Parameters
----------
member: int
Returns
-------
str
"""
return '{0:02d}'.format(member)
def _get_chunk_time_bounds(self, startdate, chunk, frequency):
......@@ -561,10 +772,57 @@ class MeteoFranceConvention(DataConvention):
return time_bound
def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype):
"""
Create file link
In this convention, it does nothing
Parameters
----------
domain: ModelingRealm
filepath: str
frequency: Frequency
var: str
grid: str
move_old: bool
vartype: VariableType
"""
pass
def create_links(self, startdate, member=None):
"""
Create links for a given startdate or member
In this convention, it does nothing
Parameters
----------
startdate: str
member: int or None
"""
pass
def is_cmorized(self, startdate, member, chunk, domain):
"""
Check if a given chunk is cmorized for a given domain
Parameters
----------
startdate: str
member: str
chunk: int
domain: ModelingRealm
Returns
-------
bool
Raises
------
NotImplementedError:
If not implemented by the derived classes
"""
return True
......@@ -660,16 +660,7 @@ class NetCDFFile(DataFile):
if self.region:
try:
cubes = iris.load(self.remote_file)
for cube in cubes:
try:
if isinstance(self.region, six.string_types):
regions = {self.region.name}
else:
regions = {basin.name for basin in self.region}
if regions.issubset(set(cube.coord('region').points)):
self.storage_status = StorageStatus.READY
except iris.exceptions.CoordinateNotFoundError:
pass
self._check_regions(cubes)
except iris.exceptions.TranslationError as ex:
# If the check goes wrong, we must execute everything
os.remove(self.remote_file)
......@@ -678,6 +669,18 @@ class NetCDFFile(DataFile):
else:
self.storage_status = StorageStatus.READY
def _check_regions(self, cubes):
for cube in cubes: