From 03da07968a11961039694b385a7dfc8ba03961a3 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 11:01:56 +0200 Subject: [PATCH 1/2] Cleaned code --- earthdiagnostics/cmorizer.py | 4 ++-- earthdiagnostics/cmormanager.py | 2 -- earthdiagnostics/config.py | 3 +-- earthdiagnostics/data_convention.py | 10 ++++++---- earthdiagnostics/datafile.py | 2 +- earthdiagnostics/datamanager.py | 1 - earthdiagnostics/work_manager.py | 2 +- 7 files changed, 11 insertions(+), 13 deletions(-) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 31ddc967..9f7a9e10 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -6,7 +6,7 @@ import shutil import uuid import traceback import pygrib -from datetime import datetime, timedelta +from datetime import datetime import six from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months @@ -583,7 +583,7 @@ class Cmorizer(object): def _fix_time_coord(self, cube, var_code): time = cube.coord('time') - target_units = 'days since 1950-01-01 00:00:00'.format(parse_date(self.startdate)) + target_units = 'days since 1950-01-01 00:00:00' time.convert_units(cf_units.Unit(target_units, calendar=time.units.calendar)) time.units = target_units diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index f367542a..4142ab09 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -2,8 +2,6 @@ """Classes to manage cmorized datasets""" import glob import os -import re -import shutil from datetime import datetime diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 3cfcc74a..ecd00031 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -8,7 +8,6 @@ from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, da from bscearth.utils.log import Log import bscearth.utils.path -from earthdiagnostics import cdftools from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.variable import VariableManager @@ -153,7 +152,7 @@ class Config(object): elif data_convention == 'preface': self.data_convention = PrefaceConvention(data_convention, self) elif data_convention == 'meteofrance': - self.data_convention == MeteoFranceConvention(data_convention, self) + self.data_convention = MeteoFranceConvention(data_convention, self) self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) namelist_file = os.path.join(os.path.dirname(__file__), diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 1f1bd51d..9fb1d917 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -114,7 +114,7 @@ class DataConvention(object): self._link_startdate(path, member_str) Log.debug('Links ready') - def _link_startdate(self, path, member): + def _link_startdate(self, path, member_str): raise NotImplementedError def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): @@ -310,7 +310,7 @@ class PrefaceConvention(Cmor2Convention): class Cmor3Convention(DataConvention): - + """Class managing Primavera file conventions""" def __init__(self, name, config): super(Cmor3Convention, self).__init__(name, config) self.lat_name = 'latitude' @@ -426,10 +426,12 @@ class Cmor3Convention(DataConvention): class CMIP6Convention(Cmor3Convention): + """Class managing CMIP6 file conventions""" pass class PrimaveraConvention(Cmor3Convention): + """Class managing Primavera file conventions""" pass @@ -442,7 +444,7 @@ class MeteoFranceConvention(DataConvention): raise ValueError('Date_str not supported with MeteoFrance convention') if chunk is None: raise ValueError('Chunk must be provided in MeteoFrance convention') - time_bound = self._get_chunk_time_bounds(startdate, chunk) + time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency) file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self.get_member_str(member)) return file_name @@ -455,7 +457,7 @@ class MeteoFranceConvention(DataConvention): def get_member_str(self, member): return '{0:02d}'.format(member) - def _get_chunk_time_bounds(self, startdate, chunk): + def _get_chunk_time_bounds(self, startdate, chunk, frequency): start = parse_date(startdate) chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month', self.config.experiment.calendar) diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index f0ec4787..e5fdb82a 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -674,7 +674,7 @@ class NetCDFFile(DataFile): # If the check goes wrong, we must execute everything os.remove(self.remote_file) except Exception as ex: - pass + Log.debug('Exception when checking file {0}: {1}', self.remote_file, ex) else: self.storage_status = StorageStatus.READY diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 7e0b6e31..4077c24e 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -2,7 +2,6 @@ """Base data manager for Earth diagnostics""" from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus, UnitConversion -from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.variable import VariableType diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index f6bd3ae9..25e1cc4a 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -143,7 +143,7 @@ class WorkManager(object): return else: step = 0.5 - for x in range(int(interval / step)): + for _ in range(int(interval / step)): if lock.acquire(blocking=False): return sys.sleep(step) -- GitLab From 80ecde225b313af1f3e4e172b476387927bf5698 Mon Sep 17 00:00:00 2001 From: Javier Vegas-Regidor Date: Wed, 23 May 2018 11:16:28 +0200 Subject: [PATCH 2/2] Added docs --- earthdiagnostics/data_convention.py | 98 ++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py index 9fb1d917..4207f844 100644 --- a/earthdiagnostics/data_convention.py +++ b/earthdiagnostics/data_convention.py @@ -13,6 +13,7 @@ from earthdiagnostics.utils import Utils class DataConvention(object): + """Base class to manage filename conventions""" def __init__(self, name, config): self.config = config @@ -62,9 +63,58 @@ class DataConvention(object): return filepath def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + """ + Get filename for a given configuration + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + frequency: Frequency + chunk: int or None + year: int or None + date_str: str or None + grid: str or None + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ raise NotImplementedError def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): + """ + Get the folder path following current CMOR convention + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + frequency: Frequency + grid: str + cmor_var: Variable + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ raise NotImplementedError def get_startdate_path(self, startdate): @@ -78,6 +128,7 @@ class DataConvention(object): Returns ------- str + """ return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.cmor.activity, self.config.experiment.institute, self.config.experiment.model, @@ -102,9 +153,35 @@ class DataConvention(object): return self.config.experiment.experiment_name def get_member_str(self, member): + """ + Transalate member number to member string + + Parameters + ---------- + member: int + + Returns + ------- + str + + Raises + ------ + NotImplementedError: + If not implemented by derived classes + + """ raise NotImplementedError def create_links(self, startdate, member=None): + """ + Create links for a given startdate or member + + Parameters + ---------- + startdate: str + member: int or None + + """ if member is not None: member_str = self.get_member_str(member) else: @@ -223,6 +300,7 @@ class DataConvention(object): class Cmor2Convention(DataConvention): + """Base class for CMOR2-based conventions""" def get_scratch_masks(self, scratch_masks): return scratch_masks @@ -290,6 +368,7 @@ class Cmor2Convention(DataConvention): class SPECSConvention(Cmor2Convention): + """Base class for CMOR2-based conventions""" def get_startdate_path(self, startdate): return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', @@ -298,7 +377,15 @@ class SPECSConvention(Cmor2Convention): class PrefaceConvention(Cmor2Convention): + """ + Class to manage Preface convention + Parameters + ---------- + name: str + config: Config + + """ def __init__(self, name, config): super(PrefaceConvention, self).__init__(name, config) self.time_separator = '_' @@ -310,7 +397,15 @@ class PrefaceConvention(Cmor2Convention): class Cmor3Convention(DataConvention): - """Class managing Primavera file conventions""" + """ + Base class for CMOR3-based conventions + + Parameters + ---------- + name: str + config: Config + + """ def __init__(self, name, config): super(Cmor3Convention, self).__init__(name, config) self.lat_name = 'latitude' @@ -436,6 +531,7 @@ class PrimaveraConvention(Cmor3Convention): class MeteoFranceConvention(DataConvention): + """Class managing MeteoFrance file conventions""" def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,): if year is not None: -- GitLab