Commit 3cfddd94 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Merge branch 'master' into 'production'

Master

See merge request !48
parents e145aafe 80ecde22
......@@ -6,7 +6,7 @@ import shutil
import uuid
import traceback
import pygrib
from datetime import datetime, timedelta
from datetime import datetime
import six
from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months
......@@ -583,7 +583,7 @@ class Cmorizer(object):
def _fix_time_coord(self, cube, var_code):
time = cube.coord('time')
target_units = 'days since 1950-01-01 00:00:00'.format(parse_date(self.startdate))
target_units = 'days since 1950-01-01 00:00:00'
time.convert_units(cf_units.Unit(target_units, calendar=time.units.calendar))
time.units = target_units
......
......@@ -2,8 +2,6 @@
"""Classes to manage cmorized datasets"""
import glob
import os
import re
import shutil
from datetime import datetime
......
......@@ -8,7 +8,6 @@ from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, da
from bscearth.utils.log import Log
import bscearth.utils.path
from earthdiagnostics import cdftools
from earthdiagnostics.frequency import Frequency, Frequencies
from earthdiagnostics.modelingrealm import ModelingRealm
from earthdiagnostics.variable import VariableManager
......@@ -153,7 +152,7 @@ class Config(object):
elif data_convention == 'preface':
self.data_convention = PrefaceConvention(data_convention, self)
elif data_convention == 'meteofrance':
self.data_convention == MeteoFranceConvention(data_convention, self)
self.data_convention = MeteoFranceConvention(data_convention, self)
self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks)
namelist_file = os.path.join(os.path.dirname(__file__),
......
......@@ -13,6 +13,7 @@ from earthdiagnostics.utils import Utils
class DataConvention(object):
"""Base class to manage filename conventions"""
def __init__(self, name, config):
self.config = config
......@@ -62,9 +63,58 @@ class DataConvention(object):
return filepath
def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ):
"""
Get filename for a given configuration
Parameters
----------
startdate: str
member: int
domain: ModelingRealm
var: str
cmor_var: Variable
frequency: Frequency
chunk: int or None
year: int or None
date_str: str or None
grid: str or None
Returns
-------
str
Raises
------
NotImplementedError:
If not implemented by derived classes
"""
raise NotImplementedError
def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var):
"""
Get the folder path following current CMOR convention
Parameters
----------
startdate: str
member: int
domain: ModelingRealm
var: str
frequency: Frequency
grid: str
cmor_var: Variable
Returns
-------
str
Raises
------
NotImplementedError:
If not implemented by derived classes
"""
raise NotImplementedError
def get_startdate_path(self, startdate):
......@@ -78,6 +128,7 @@ class DataConvention(object):
Returns
-------
str
"""
return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.cmor.activity,
self.config.experiment.institute, self.config.experiment.model,
......@@ -102,9 +153,35 @@ class DataConvention(object):
return self.config.experiment.experiment_name
def get_member_str(self, member):
"""
Transalate member number to member string
Parameters
----------
member: int
Returns
-------
str
Raises
------
NotImplementedError:
If not implemented by derived classes
"""
raise NotImplementedError
def create_links(self, startdate, member=None):
"""
Create links for a given startdate or member
Parameters
----------
startdate: str
member: int or None
"""
if member is not None:
member_str = self.get_member_str(member)
else:
......@@ -114,7 +191,7 @@ class DataConvention(object):
self._link_startdate(path, member_str)
Log.debug('Links ready')
def _link_startdate(self, path, member):
def _link_startdate(self, path, member_str):
raise NotImplementedError
def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype):
......@@ -223,6 +300,7 @@ class DataConvention(object):
class Cmor2Convention(DataConvention):
"""Base class for CMOR2-based conventions"""
def get_scratch_masks(self, scratch_masks):
return scratch_masks
......@@ -290,6 +368,7 @@ class Cmor2Convention(DataConvention):
class SPECSConvention(Cmor2Convention):
"""Base class for CMOR2-based conventions"""
def get_startdate_path(self, startdate):
return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles',
......@@ -298,7 +377,15 @@ class SPECSConvention(Cmor2Convention):
class PrefaceConvention(Cmor2Convention):
"""
Class to manage Preface convention
Parameters
----------
name: str
config: Config
"""
def __init__(self, name, config):
super(PrefaceConvention, self).__init__(name, config)
self.time_separator = '_'
......@@ -310,7 +397,15 @@ class PrefaceConvention(Cmor2Convention):
class Cmor3Convention(DataConvention):
"""
Base class for CMOR3-based conventions
Parameters
----------
name: str
config: Config
"""
def __init__(self, name, config):
super(Cmor3Convention, self).__init__(name, config)
self.lat_name = 'latitude'
......@@ -426,14 +521,17 @@ class Cmor3Convention(DataConvention):
class CMIP6Convention(Cmor3Convention):
"""Class managing CMIP6 file conventions"""
pass
class PrimaveraConvention(Cmor3Convention):
"""Class managing Primavera file conventions"""
pass
class MeteoFranceConvention(DataConvention):
"""Class managing MeteoFrance file conventions"""
def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,):
if year is not None:
......@@ -442,7 +540,7 @@ class MeteoFranceConvention(DataConvention):
raise ValueError('Date_str not supported with MeteoFrance convention')
if chunk is None:
raise ValueError('Chunk must be provided in MeteoFrance convention')
time_bound = self._get_chunk_time_bounds(startdate, chunk)
time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency)
file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self.get_member_str(member))
return file_name
......@@ -455,7 +553,7 @@ class MeteoFranceConvention(DataConvention):
def get_member_str(self, member):
return '{0:02d}'.format(member)
def _get_chunk_time_bounds(self, startdate, chunk):
def _get_chunk_time_bounds(self, startdate, chunk, frequency):
start = parse_date(startdate)
chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month',
self.config.experiment.calendar)
......
......@@ -674,7 +674,7 @@ class NetCDFFile(DataFile):
# If the check goes wrong, we must execute everything
os.remove(self.remote_file)
except Exception as ex:
pass
Log.debug('Exception when checking file {0}: {1}', self.remote_file, ex)
else:
self.storage_status = StorageStatus.READY
......
......@@ -2,7 +2,6 @@
"""Base data manager for Earth diagnostics"""
from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus, UnitConversion
from earthdiagnostics.modelingrealm import ModelingRealms
from earthdiagnostics.variable import VariableType
......
......@@ -143,7 +143,7 @@ class WorkManager(object):
return
else:
step = 0.5
for x in range(int(interval / step)):
for _ in range(int(interval / step)):
if lock.acquire(blocking=False):
return
sys.sleep(step)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment