diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 31ddc967fab07ccfc596fba7cc1d75aa791006ae..9f7a9e1096417d2b932fa6bae1c482ecf947dc17 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -6,7 +6,7 @@ import shutil import uuid import traceback import pygrib -from datetime import datetime, timedelta +from datetime import datetime import six from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months @@ -583,7 +583,7 @@ class Cmorizer(object): def _fix_time_coord(self, cube, var_code): time = cube.coord('time') - target_units = 'days since 1950-01-01 00:00:00'.format(parse_date(self.startdate)) + target_units = 'days since 1950-01-01 00:00:00' time.convert_units(cf_units.Unit(target_units, calendar=time.units.calendar)) time.units = target_units diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index f367542a308bd6bf341f604a3e37069411c9c292..4142ab09af91b712d04a9994f15245d1b3e3acca 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -2,8 +2,6 @@ """Classes to manage cmorized datasets""" import glob import os -import re -import shutil from datetime import datetime diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 3cfcc74ab17ea5d14c5f45228c418123eae9a71f..ecd0003186f900e8a5d8a9b0a9ebf8a606abbd2c 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -8,7 +8,6 @@ from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, da from bscearth.utils.log import Log import bscearth.utils.path -from earthdiagnostics import cdftools from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.variable import VariableManager @@ -153,7 +152,7 @@ class Config(object): elif data_convention == 'preface': self.data_convention = PrefaceConvention(data_convention, self) elif data_convention == 'meteofrance': - self.data_convention == MeteoFranceConvention(data_convention, self) + self.data_convention = MeteoFranceConvention(data_convention, self) self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) namelist_file = os.path.join(os.path.dirname(__file__), diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 1f1bd51d2185423db53706e2bf6bfbf4273809e6..4207f844da4875e0f7205dfc3f4e07f44ccb5a41 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -13,6 +13,7 @@ from earthdiagnostics.utils import Utils class DataConvention(object): + """Base class to manage filename conventions""" def __init__(self, name, config): self.config = config @@ -62,9 +63,58 @@ class DataConvention(object): return filepath def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ raise NotImplementedError def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): + """ + Get the folder path following current CMOR convention + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + frequency: Frequency + grid: str + cmor_var: Variable + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ raise NotImplementedError def get_startdate_path(self, startdate): @@ -78,6 +128,7 @@ class DataConvention(object): Returns ------- str + """ return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.cmor.activity, self.config.experiment.institute, self.config.experiment.model, @@ -102,9 +153,35 @@ class DataConvention(object): return self.config.experiment.experiment_name def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ raise NotImplementedError def create_links(self, startdate, member=None): + """ + Create links for a given startdate or member + + Parameters + ---------- + startdate: str + member: int or None + + """ if member is not None: member_str = self.get_member_str(member) else: @@ -114,7 +191,7 @@ class DataConvention(object): self._link_startdate(path, member_str) Log.debug('Links ready') - def _link_startdate(self, path, member): + def _link_startdate(self, path, member_str): raise NotImplementedError def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): @@ -223,6 +300,7 @@ class DataConvention(object): class Cmor2Convention(DataConvention): + """Base class for CMOR2-based conventions""" def get_scratch_masks(self, scratch_masks): return scratch_masks @@ -290,6 +368,7 @@ class Cmor2Convention(DataConvention): class SPECSConvention(Cmor2Convention): + """Base class for CMOR2-based conventions""" def get_startdate_path(self, startdate): return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', @@ -298,7 +377,15 @@ class SPECSConvention(Cmor2Convention): class PrefaceConvention(Cmor2Convention): + """ + Class to manage Preface convention + + Parameters + ---------- + name: str + config: Config + """ def __init__(self, name, config): super(PrefaceConvention, self).__init__(name, config) self.time_separator = '_' @@ -310,7 +397,15 @@ class PrefaceConvention(Cmor2Convention): class Cmor3Convention(DataConvention): + """ + Base class for CMOR3-based conventions + Parameters + ---------- + name: str + config: Config + + """ def __init__(self, name, config): super(Cmor3Convention, self).__init__(name, config) self.lat_name = 'latitude' @@ -426,14 +521,17 @@ class Cmor3Convention(DataConvention): class CMIP6Convention(Cmor3Convention): + """Class managing CMIP6 file conventions""" pass class PrimaveraConvention(Cmor3Convention): + """Class managing Primavera file conventions""" pass class MeteoFranceConvention(DataConvention): + """Class managing MeteoFrance file conventions""" def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,): if year is not None: @@ -442,7 +540,7 @@ class MeteoFranceConvention(DataConvention): raise ValueError('Date_str not supported with MeteoFrance convention') if chunk is None: raise ValueError('Chunk must be provided in MeteoFrance convention') - time_bound = self._get_chunk_time_bounds(startdate, chunk) + time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency) file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self.get_member_str(member)) return file_name @@ -455,7 +553,7 @@ class MeteoFranceConvention(DataConvention): def get_member_str(self, member): return '{0:02d}'.format(member) - def _get_chunk_time_bounds(self, startdate, chunk): + def _get_chunk_time_bounds(self, startdate, chunk, frequency): start = parse_date(startdate) chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month', self.config.experiment.calendar) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index f0ec47872cf4e803cd1a2470cc05a52cb9f7b8e3..e5fdb82a50543037fae0a979f9e1a2042342bcb6 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -674,7 +674,7 @@ class NetCDFFile(DataFile): # If the check goes wrong, we must execute everything os.remove(self.remote_file) except Exception as ex: - pass + Log.debug('Exception when checking file {0}: {1}', self.remote_file, ex) else: self.storage_status = StorageStatus.READY diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 7e0b6e3158636339154d296f7f13a5f99bd00507..4077c24e163811621f9735c6fc524cbc5cbd389d 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -2,7 +2,6 @@ """Base data manager for Earth diagnostics""" from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus, UnitConversion -from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.variable import VariableType diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index f6bd3ae96e7235a6781de027e913b0b13b8baffa..25e1cc4abe644b01c6e2ce6dff624b1111a4ea0b 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -143,7 +143,7 @@ class WorkManager(object): return else: step = 0.5 - for x in range(int(interval / step)): + for _ in range(int(interval / step)): if lock.acquire(blocking=False): return sys.sleep(step)