Commit 3cfddd94 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Merge branch 'master' into 'production'

Master

See merge request !48
parents e145aafe 80ecde22
...@@ -6,7 +6,7 @@ import shutil ...@@ -6,7 +6,7 @@ import shutil
import uuid import uuid
import traceback import traceback
import pygrib import pygrib
from datetime import datetime, timedelta from datetime import datetime
import six import six
from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months
...@@ -583,7 +583,7 @@ class Cmorizer(object): ...@@ -583,7 +583,7 @@ class Cmorizer(object):
def _fix_time_coord(self, cube, var_code): def _fix_time_coord(self, cube, var_code):
time = cube.coord('time') time = cube.coord('time')
target_units = 'days since 1950-01-01 00:00:00'.format(parse_date(self.startdate)) target_units = 'days since 1950-01-01 00:00:00'
time.convert_units(cf_units.Unit(target_units, calendar=time.units.calendar)) time.convert_units(cf_units.Unit(target_units, calendar=time.units.calendar))
time.units = target_units time.units = target_units
......
...@@ -2,8 +2,6 @@ ...@@ -2,8 +2,6 @@
"""Classes to manage cmorized datasets""" """Classes to manage cmorized datasets"""
import glob import glob
import os import os
import re
import shutil
from datetime import datetime from datetime import datetime
......
...@@ -8,7 +8,6 @@ from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, da ...@@ -8,7 +8,6 @@ from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, da
from bscearth.utils.log import Log from bscearth.utils.log import Log
import bscearth.utils.path import bscearth.utils.path
from earthdiagnostics import cdftools
from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.frequency import Frequency, Frequencies
from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.modelingrealm import ModelingRealm
from earthdiagnostics.variable import VariableManager from earthdiagnostics.variable import VariableManager
...@@ -153,7 +152,7 @@ class Config(object): ...@@ -153,7 +152,7 @@ class Config(object):
elif data_convention == 'preface': elif data_convention == 'preface':
self.data_convention = PrefaceConvention(data_convention, self) self.data_convention = PrefaceConvention(data_convention, self)
elif data_convention == 'meteofrance': elif data_convention == 'meteofrance':
self.data_convention == MeteoFranceConvention(data_convention, self) self.data_convention = MeteoFranceConvention(data_convention, self)
self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks)
namelist_file = os.path.join(os.path.dirname(__file__), namelist_file = os.path.join(os.path.dirname(__file__),
......
...@@ -13,6 +13,7 @@ from earthdiagnostics.utils import Utils ...@@ -13,6 +13,7 @@ from earthdiagnostics.utils import Utils
class DataConvention(object): class DataConvention(object):
"""Base class to manage filename conventions"""
def __init__(self, name, config): def __init__(self, name, config):
self.config = config self.config = config
...@@ -62,9 +63,58 @@ class DataConvention(object): ...@@ -62,9 +63,58 @@ class DataConvention(object):
return filepath return filepath
def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ):
"""
Get filename for a given configuration
Parameters
----------
startdate: str
member: int
domain: ModelingRealm
var: str
cmor_var: Variable
frequency: Frequency
chunk: int or None
year: int or None
date_str: str or None
grid: str or None
Returns
-------
str
Raises
------
NotImplementedError:
If not implemented by derived classes
"""
raise NotImplementedError raise NotImplementedError
def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var):
"""
Get the folder path following current CMOR convention
Parameters
----------
startdate: str
member: int
domain: ModelingRealm
var: str
frequency: Frequency
grid: str
cmor_var: Variable
Returns
-------
str
Raises
------
NotImplementedError:
If not implemented by derived classes
"""
raise NotImplementedError raise NotImplementedError
def get_startdate_path(self, startdate): def get_startdate_path(self, startdate):
...@@ -78,6 +128,7 @@ class DataConvention(object): ...@@ -78,6 +128,7 @@ class DataConvention(object):
Returns Returns
------- -------
str str
""" """
return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.cmor.activity, return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.cmor.activity,
self.config.experiment.institute, self.config.experiment.model, self.config.experiment.institute, self.config.experiment.model,
...@@ -102,9 +153,35 @@ class DataConvention(object): ...@@ -102,9 +153,35 @@ class DataConvention(object):
return self.config.experiment.experiment_name return self.config.experiment.experiment_name
def get_member_str(self, member): def get_member_str(self, member):
"""
Transalate member number to member string
Parameters
----------
member: int
Returns
-------
str
Raises
------
NotImplementedError:
If not implemented by derived classes
"""
raise NotImplementedError raise NotImplementedError
def create_links(self, startdate, member=None): def create_links(self, startdate, member=None):
"""
Create links for a given startdate or member
Parameters
----------
startdate: str
member: int or None
"""
if member is not None: if member is not None:
member_str = self.get_member_str(member) member_str = self.get_member_str(member)
else: else:
...@@ -114,7 +191,7 @@ class DataConvention(object): ...@@ -114,7 +191,7 @@ class DataConvention(object):
self._link_startdate(path, member_str) self._link_startdate(path, member_str)
Log.debug('Links ready') Log.debug('Links ready')
def _link_startdate(self, path, member): def _link_startdate(self, path, member_str):
raise NotImplementedError raise NotImplementedError
def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype):
...@@ -223,6 +300,7 @@ class DataConvention(object): ...@@ -223,6 +300,7 @@ class DataConvention(object):
class Cmor2Convention(DataConvention): class Cmor2Convention(DataConvention):
"""Base class for CMOR2-based conventions"""
def get_scratch_masks(self, scratch_masks): def get_scratch_masks(self, scratch_masks):
return scratch_masks return scratch_masks
...@@ -290,6 +368,7 @@ class Cmor2Convention(DataConvention): ...@@ -290,6 +368,7 @@ class Cmor2Convention(DataConvention):
class SPECSConvention(Cmor2Convention): class SPECSConvention(Cmor2Convention):
"""Base class for CMOR2-based conventions"""
def get_startdate_path(self, startdate): def get_startdate_path(self, startdate):
return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles',
...@@ -298,7 +377,15 @@ class SPECSConvention(Cmor2Convention): ...@@ -298,7 +377,15 @@ class SPECSConvention(Cmor2Convention):
class PrefaceConvention(Cmor2Convention): class PrefaceConvention(Cmor2Convention):
"""
Class to manage Preface convention
Parameters
----------
name: str
config: Config
"""
def __init__(self, name, config): def __init__(self, name, config):
super(PrefaceConvention, self).__init__(name, config) super(PrefaceConvention, self).__init__(name, config)
self.time_separator = '_' self.time_separator = '_'
...@@ -310,7 +397,15 @@ class PrefaceConvention(Cmor2Convention): ...@@ -310,7 +397,15 @@ class PrefaceConvention(Cmor2Convention):
class Cmor3Convention(DataConvention): class Cmor3Convention(DataConvention):
"""
Base class for CMOR3-based conventions
Parameters
----------
name: str
config: Config
"""
def __init__(self, name, config): def __init__(self, name, config):
super(Cmor3Convention, self).__init__(name, config) super(Cmor3Convention, self).__init__(name, config)
self.lat_name = 'latitude' self.lat_name = 'latitude'
...@@ -426,14 +521,17 @@ class Cmor3Convention(DataConvention): ...@@ -426,14 +521,17 @@ class Cmor3Convention(DataConvention):
class CMIP6Convention(Cmor3Convention): class CMIP6Convention(Cmor3Convention):
"""Class managing CMIP6 file conventions"""
pass pass
class PrimaveraConvention(Cmor3Convention): class PrimaveraConvention(Cmor3Convention):
"""Class managing Primavera file conventions"""
pass pass
class MeteoFranceConvention(DataConvention): class MeteoFranceConvention(DataConvention):
"""Class managing MeteoFrance file conventions"""
def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,): def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,):
if year is not None: if year is not None:
...@@ -442,7 +540,7 @@ class MeteoFranceConvention(DataConvention): ...@@ -442,7 +540,7 @@ class MeteoFranceConvention(DataConvention):
raise ValueError('Date_str not supported with MeteoFrance convention') raise ValueError('Date_str not supported with MeteoFrance convention')
if chunk is None: if chunk is None:
raise ValueError('Chunk must be provided in MeteoFrance convention') raise ValueError('Chunk must be provided in MeteoFrance convention')
time_bound = self._get_chunk_time_bounds(startdate, chunk) time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency)
file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self.get_member_str(member)) file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self.get_member_str(member))
return file_name return file_name
...@@ -455,7 +553,7 @@ class MeteoFranceConvention(DataConvention): ...@@ -455,7 +553,7 @@ class MeteoFranceConvention(DataConvention):
def get_member_str(self, member): def get_member_str(self, member):
return '{0:02d}'.format(member) return '{0:02d}'.format(member)
def _get_chunk_time_bounds(self, startdate, chunk): def _get_chunk_time_bounds(self, startdate, chunk, frequency):
start = parse_date(startdate) start = parse_date(startdate)
chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month', chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month',
self.config.experiment.calendar) self.config.experiment.calendar)
......
...@@ -674,7 +674,7 @@ class NetCDFFile(DataFile): ...@@ -674,7 +674,7 @@ class NetCDFFile(DataFile):
# If the check goes wrong, we must execute everything # If the check goes wrong, we must execute everything
os.remove(self.remote_file) os.remove(self.remote_file)
except Exception as ex: except Exception as ex:
pass Log.debug('Exception when checking file {0}: {1}', self.remote_file, ex)
else: else:
self.storage_status = StorageStatus.READY self.storage_status = StorageStatus.READY
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
"""Base data manager for Earth diagnostics""" """Base data manager for Earth diagnostics"""
from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus, UnitConversion from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus, UnitConversion
from earthdiagnostics.modelingrealm import ModelingRealms
from earthdiagnostics.variable import VariableType from earthdiagnostics.variable import VariableType
......
...@@ -143,7 +143,7 @@ class WorkManager(object): ...@@ -143,7 +143,7 @@ class WorkManager(object):
return return
else: else:
step = 0.5 step = 0.5
for x in range(int(interval / step)): for _ in range(int(interval / step)):
if lock.acquire(blocking=False): if lock.acquire(blocking=False):
return return
sys.sleep(step) sys.sleep(step)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment