diff --git a/VERSION b/VERSION index 40020fcf5dc73df73f4e501cfa508d3f1744bf02..6f9b87cf6358b66ba00f4c1254c2f407fc509414 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0rc5 +3.0.0rc6 diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index f720696e20bea554d24e2ad9cb1dce149648d3f0..58bef080fc2998a349863262e460958e92090399 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -18,7 +18,6 @@ class CDFTools(object): def __init__(self, path=''): self.path = path - self.data_convention = '' # noinspection PyShadowingBuiltins def run(self, command, input_file, output_file=None, options=None, log_level=Log.INFO, input_option=None): diff --git a/earthdiagnostics/cmor_tables/primavera b/earthdiagnostics/cmor_tables/primavera index 5236593cf11fd3175dd1e48d201cd507bd2e37ac..f3e44bf8211642e519597f73a0510e13ec3d4725 160000 --- a/earthdiagnostics/cmor_tables/primavera +++ b/earthdiagnostics/cmor_tables/primavera @@ -1 +1 @@ -Subproject commit 5236593cf11fd3175dd1e48d201cd507bd2e37ac +Subproject commit f3e44bf8211642e519597f73a0510e13ec3d4725 diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 142f06ebfa7555ba2a64ed234ce282a40a0fe173..31ddc967fab07ccfc596fba7cc1d75aa791006ae 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -6,11 +6,18 @@ import shutil import uuid import traceback import pygrib -from datetime import datetime +from datetime import datetime, timedelta +import six from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months from bscearth.utils.log import Log from cdo import CDOException +import iris +import iris.coord_categorisation +import iris.analysis +import iris.util +import iris.exceptions +import cf_units from earthdiagnostics.datafile import NetCDFFile from earthdiagnostics.frequency import Frequency, Frequencies @@ -277,9 +284,7 @@ class Cmorizer(object): if not os.path.exists(next_gribfile): os.remove(gribfile) - self._ungrib_vars(gribfile, current_date.month, Frequency('{0}hr'.format(self.atmos_timestep))) - self._ungrib_vars(gribfile, current_date.month, Frequencies.daily) - self._ungrib_vars(gribfile, current_date.month, Frequencies.monthly) + self._ungrib_vars(gribfile, current_date.month) for splited_file in glob.glob('{0}_*.128.nc'.format(gribfile)): os.remove(splited_file) @@ -301,18 +306,18 @@ class Cmorizer(object): codes_str = ','.join([str(code) for code in codes]) try: if grid == 'SH': - Utils.cdo.splitparam(input='-sp2gpl -selmon,{2} -selcode,{0} {1} '.format(codes_str, full_file, month), + Utils.cdo.splitparam(input='-sp2gpl -selcode,{0} {1} '.format(codes_str, full_file), output=gribfile + '_', - options='-f nc4') + options='-f nc4 -t ecmwf') else: - Utils.cdo.splitparam(input='-selmon,{2} -selcode,{0} {1}'.format(codes_str, full_file, month), + Utils.cdo.splitparam(input='-selcode,{0} {1}'.format(codes_str, full_file), output=gribfile + '_', - options='-R -f nc4') + options='-R -f nc4 -t ecmwf') # total precipitation (remove negative values) if 228 in codes: Utils.cdo.setcode(228, - input='-setmisstoc,0 -setvrange,0,Inf ' + input='-chname,LSP,TP -setmisstoc,0 -setvrange,0,Inf ' '-add {0}_142.128.nc {0}_143.128.nc'.format(gribfile), output='{0}_228.128.nc'.format(gribfile), options='-f nc4') @@ -442,10 +447,37 @@ class Cmorizer(object): return temp = TempFile.get() - Utils.nco.ncks(input=file_path, output=temp, options=('-v {0}'.format(variable),)) - self._rename_level_variables(temp, var_cmor) + handler = Utils.open_cdf(file_path) + coords = [self.lon_name, self.lat_name, 'time'] + if 'leadtime' in handler.variables.keys(): + coords.append('leadtime') - self._add_coordinate_variables(file_path, temp) + if var_cmor.domain == ModelingRealms.ocean: + lev_dimensions = {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', + 'depth': 'lev'} + elif var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: + lev_dimensions = {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', + 'depth_4': 'sdepth'} + elif var_cmor.domain == ModelingRealms.atmos: + lev_dimensions = {'depth': 'plev'} + else: + lev_dimensions = {} + + for lev_dim in lev_dimensions.keys(): + if lev_dim in handler.variables[variable].dimensions: + coords.append(lev_dim) + + handler.variables[variable].coordinates = ' '.join(set(coords)) + handler.close() + + cube = iris.load_cube(file_path, iris.Constraint(cube_func=lambda c: c.var_name == variable)) + for lev_original, lev_target in six.iteritems(lev_dimensions): + try: + cube.coord(lev_original).var_name = lev_target + except iris.exceptions.CoordinateNotFoundError: + pass + + iris.save(cube, temp, zlib=True) if alias.basin is None: region = None @@ -512,138 +544,115 @@ class Cmorizer(object): raise Exception('File {0} start date is not a valid chunk start date'.format(file_path)) return chunk - def _add_coordinate_variables(self, file_path, temp): - handler_cmor = Utils.open_cdf(temp) - handler = Utils.open_cdf(file_path, 'r') - Utils.copy_variable(handler, handler_cmor, self.lon_name, False) - Utils.copy_variable(handler, handler_cmor, self.lat_name, False) - if 'time' in handler_cmor.dimensions.keys(): - Utils.copy_variable(handler, handler_cmor, 'leadtime', False) - handler_cmor.close() - handler.close() - - @staticmethod - def _rename_level_variables(temp, var_cmor): - if var_cmor.domain == ModelingRealms.ocean: - Utils.rename_variables(temp, {'deptht': 'lev', 'depthu': 'lev', 'depthw': 'lev', 'depthv': 'lev', - 'depth': 'lev'}, False) - if var_cmor.domain in [ModelingRealms.landIce, ModelingRealms.land]: - Utils.rename_variables(temp, {'depth': 'sdepth', 'depth_2': 'sdepth', 'depth_3': 'sdepth', - 'depth_4': 'sdepth'}, False) - if var_cmor.domain == ModelingRealms.atmos: - Utils.rename_variables(temp, {'depth': 'plev'}, False) - def _merge_grib_files(self, current_month, prev_gribfile, gribfile): Log.info('Merging data from different files...') - rules_path = os.path.join(self.config.scratch_dir, 'rules_files') - fd = open(rules_path, 'w') - fd.write('if (dataDate >= {0.year}{0.month:02}01) {{ write ; }}\n'.format(current_month)) - fd.close() - # get first timestep for each month from previous file (if possible) - if os.path.exists(self.path_icm): - os.remove(self.path_icm) - Utils.execute_shell_command('grib_filter -o {2} {3} ' - '{0} {1}'.format(prev_gribfile, gribfile, self.path_icm, rules_path)) - os.remove(rules_path) - Utils.remove_file(prev_gribfile) - - def _ungrib_vars(self, gribfile, month, frequency): - cdo_reftime = '{0.year}-{0.month:02}-{0.day:02},00:00'.format(parse_date(self.startdate)) - Log.info('Preparing {0} variables'.format(frequency)) - var_codes = self.config.cmor.get_variables(frequency) - for var_code in var_codes: - if not os.path.exists('{0}_{1}.128.nc'.format(gribfile, var_code)): + temp = TempFile.get(suffix='.grb') + Utils.cdo.selmon(current_month.month, input=prev_gribfile, output=temp) + Utils.cdo.mergetime(input=[temp, gribfile], + output=self.path_icm) + os.remove(prev_gribfile) + os.remove(temp) + + def _ungrib_vars(self, gribfile, month): + for var_code in self.cmor.get_requested_codes(): + file_path = '{0}_{1}.128.nc'.format(gribfile, var_code) + if not os.path.exists(file_path): continue - new_units = None - - cdo_operator = self._get_time_average(frequency, var_code) - cdo_operator = self._fix_time_shift(cdo_operator, var_code) - cdo_operator, new_units = self._change_units(cdo_operator, new_units, var_code) - - levels = self.config.cmor.get_levels(frequency, var_code) - if levels: - cdo_operator = "{0} -sellevel,{1}".format(cdo_operator, levels) - - Utils.cdo.setreftime(cdo_reftime, - input='{2} {0}_{1}.128.nc '.format(gribfile, var_code, cdo_operator), - output='{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency), - options='-t ecmwf') - h_var_file = '{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency) - - handler = Utils.open_cdf(h_var_file) - if new_units: - for var in handler.variables.values(): - if 'code' in var.ncattrs() and var.code == var_code: - var.units = new_units - break - - var_name = None - for key in handler.variables.keys(): - if key + '_2' in handler.variables and key not in handler.dimensions: - var_name = key - handler.close() + cube = iris.load_cube(file_path) + + cube = self._fix_time_coord(cube, var_code) + cube = cube.extract(iris.Constraint(month_number=month)) + cube = self._change_units(cube, var_code) + + for frequency in (Frequencies.monthly, Frequencies.daily, Frequency('{0}hr'.format(self.atmos_timestep))): + if var_code not in self.cmor.get_variables(frequency): + continue + time_cube = self._get_time_average(cube, frequency, var_code) + + levels = self.config.cmor.get_levels(frequency, var_code) + if levels: + time_cube = time_cube.extract(level=levels) - if var_name is not None: - Utils.nco.ncks(input='{0}_{1}_1m.nc'.format(gribfile, var_code), - output='{0}_{1}_1m.nc'.format(gribfile, var_code), - options=('-O -v {0}'.format(var_name))) + if cube.var_name.endswith('_2'): + time_cube.var_name = cube.var_name[:-2] + + out_file = '{0}_{1}_{2}.nc'.format(gribfile, var_code, frequency) + time_cube.remove_coord('month_number') + time_cube.remove_coord('day_of_month') + iris.save(time_cube, out_file, zlib=True) + + def _fix_time_coord(self, cube, var_code): + time = cube.coord('time') + target_units = 'days since 1950-01-01 00:00:00'.format(parse_date(self.startdate)) + time.convert_units(cf_units.Unit(target_units, calendar=time.units.calendar)) + time.units = target_units - def _fix_time_shift(self, cdo_operator, var_code): if var_code in (144, 146, 147, 169, 175, 176, 177, 179, 180, 181, 182, 201, 202, 205, 212, 228): - cdo_operator = '{0} -shifttime,-{1}hours'.format(cdo_operator, self.experiment.atmos_timestep) - return cdo_operator + time.points = time.points - (self.experiment.atmos_timestep / 24.0) + iris.coord_categorisation.add_day_of_month(cube, 'time') + iris.coord_categorisation.add_month_number(cube, 'time') + return cube @staticmethod - def _get_time_average(frequency, var_code): - cdo_operator = "" + def _get_time_average(cube, frequency, var_code): if frequency == Frequencies.monthly: if var_code == 201: - cdo_operator = "-monmean -daymax " + cube = cube.aggregated_by(['month_number', 'day_of_month'], iris.analysis.MAX) elif var_code == 202: - cdo_operator = "-monmean -daymin " - else: - cdo_operator = "-monmean " + cube = cube.aggregated_by(['month_number', 'day_of_month'], iris.analysis.MIN) + cube = cube.aggregated_by(['month_number'], iris.analysis.MEAN) elif frequency == Frequencies.daily: if var_code == 201: - cdo_operator = "-daymax " + cube = cube.aggregated_by(['month_number', 'day_of_month'], iris.analysis.MAX) elif var_code == 202: - cdo_operator = "-daymin " + cube = cube.aggregated_by(['month_number', 'day_of_month'], iris.analysis.MIN) else: - cdo_operator = "-daymean " - return cdo_operator + cube = cube.aggregated_by(['month_number', 'day_of_month'], iris.analysis.MEAN) + return cube - def _change_units(self, cdo_operator, new_units, var_code): + def _change_units(self, cube, var_code): + var_name = cube.var_name if var_code == 129: # geopotential - new_units = "m" - cdo_operator = "-divc,9.81 {0}".format(cdo_operator) + cube = cube / 9.81 + cube.units = 'm' elif var_code in (146, 147, 169, 175, 176, 177, 179, 212): # radiation - new_units = "W m-2" - cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) + cube = cube / (self.experiment.atmos_timestep * 3600) + cube.units = 'W m-2' elif var_code in (180, 181): # momentum flux - new_units = "N m-2" - cdo_operator = "-divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) + cube = cube / (self.experiment.atmos_timestep * 3600) + cube.units = 'N m-2' elif var_code in (144, 182, 205, 228): # precipitation/evaporation/runoff - new_units = "kg m-2 s-1" - cdo_operator = "-mulc,1000 -divc,{0} {1}".format(self.experiment.atmos_timestep * 3600, cdo_operator) - return cdo_operator, new_units + cube = cube * 1000 / (self.experiment.atmos_timestep * 3600) + cube.units = 'kg m-2 s-1' + cube.var_name = var_name + return cube def _merge_and_cmorize_atmos(self, chunk_start, chunk_end, grid, frequency): merged_file = 'MMA_{0}_{1}_{2}_{3}.nc'.format(frequency, date2str(chunk_start), date2str(chunk_end), grid) files = glob.glob(os.path.join(self.config.scratch_dir, '{0}_*_{1}.nc'.format(self._get_grib_filename(grid, chunk_start), frequency))) + + def _load_cube(cube, field, filename): + if 'history' in cube.attributes: + del cube.attributes['history'] + for first_file in files: - shutil.move(first_file, merged_file) - current_month = add_months(chunk_start, 1, self.experiment.calendar) + var_files = [] + current_month = chunk_start while current_month < chunk_end: - month_file = first_file.replace('+{0}.grb'.format(date2str(chunk_start)[:-2]), - '+{0}.grb'.format(date2str(current_month)[:-2])) - Utils.concat_variables(month_file, merged_file, True) + var_files.append(first_file.replace('+{0}.grb'.format(date2str(chunk_start)[:-2]), + '+{0}.grb'.format(date2str(current_month)[:-2]))) current_month = add_months(current_month, 1, self.experiment.calendar) - + var_cubes = iris.load(var_files, callback=_load_cube) + iris.util.unify_time_units(var_cubes) + var_cube = var_cubes.concatenate_cube() + iris.save(var_cube, merged_file, zlib=True) + for var_file in var_files: + os.remove(var_file) self._cmorize_nc_file(merged_file) def _update_time_variables(self, handler): diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 7cb0628a8084dcb45a696af229908c5e9352f81c..f367542a308bd6bf341f604a3e37069411c9c292 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -6,7 +6,7 @@ import re import shutil from datetime import datetime -from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, previous_day + from bscearth.utils.log import Log from earthdiagnostics.datafile import StorageStatus @@ -34,24 +34,7 @@ class CMORManager(DataManager): self._dic_cmorized = dict() self.find_model_data() self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid) - - def experiment_name(self, startdate): - """ - Get experiment name, appending startdate if needed - - Parameters - ---------- - startdate: str - - Returns - ------- - str - - """ - if self.config.cmor.append_startdate: - return '{}S{}'.format(self.config.experiment.experiment_name, startdate) - else: - return self.config.experiment.experiment_name + self.convention = self.config.data_convention def find_model_data(self): """ @@ -110,7 +93,8 @@ class CMORManager(DataManager): """ cmor_var = self.variable_list.get_variable(var) - filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) + filepath = self.convention.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, + None, None) if possible_versions is None: return os.path.isfile(filepath) @@ -145,7 +129,8 @@ class CMORManager(DataManager): frequency = self.config.frequency cmor_var = self.variable_list.get_variable(var) var = self._get_final_var_name(box, var) - filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) + filepath = self.convention.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, + None, None) return self._get_file_from_storage(filepath) @@ -210,7 +195,8 @@ class CMORManager(DataManager): var = cmor_var.short_name final_name = self._get_final_var_name(box, var) - filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, chunk, frequency, grid) + filepath = self.convention.get_file_path(startdate, member, domain, final_name, cmor_var, chunk, + frequency, grid) netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, region, diagnostic, grid, vartype, original_name) netcdf_file.frequency = frequency @@ -244,150 +230,13 @@ class CMORManager(DataManager): var = cmor_var.short_name final_name = self._get_final_var_name(box, var) - filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, None, Frequencies.yearly, grid, - year=year) + filepath = self.convention.get_file_path(startdate, member, domain, final_name, cmor_var, None, + Frequencies.yearly, grid, year=year) netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, None, diagnostic, grid, vartype, original_name) netcdf_file.frequency = Frequencies.yearly return netcdf_file - def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, - grid=None, year=None, date_str=None): - """ - Return the path to a concrete file - - Parameters - ---------- - startdate: str - member: int - domain: ModelingRealm - var: str - cmor_var: Variable - chunk: int or None - frequency: Frequency or str - grid: str or None - year: int or None - date_str: str or None - - Returns - ------- - str - - Raises - ------ - ValueError - If you provide two or more parameters from chunk, year or date_str or none at all - - """ - if frequency is None: - frequency = self.config.frequency - frequency = Frequency.parse(frequency) - options = sum(x is not None for x in (chunk, year, date_str)) - if options == 0: - raise ValueError('You must provide chunk, year or date_str') - elif options > 1: - raise ValueError('You must provide only one parameter in chunk, year or date_str') - if frequency is None: - frequency = self.config.frequency - else: - frequency = Frequency.parse(frequency) - - folder_path = self._get_full_cmor_folder_path(startdate, member, domain, var, frequency, grid, cmor_var) - file_name = self._get_cmor_file_name(startdate, member, domain, var, cmor_var, frequency, - chunk, year, date_str, grid) - - filepath = os.path.join(folder_path, file_name) - return filepath - - def _get_cmor_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): - if cmor_var is None: - cmor_table = domain.get_table(frequency, self.config.data_convention) - else: - cmor_table = cmor_var.get_table(frequency, self.config.data_convention) - - time_bound = self._get_time_component(chunk, date_str, frequency, startdate, year) - time_bound = '_{0}.nc'.format(time_bound) - - if self.config.data_convention in ('specs', 'preface'): - - file_name = '{0}_{1}_{2}_{3}_S{4}_{5}{6}'.format(var, cmor_table.name, self.experiment.model, - self.experiment_name(startdate), startdate, - self._get_member_str(member), time_bound) - elif self.config.data_convention in ('primavera', 'cmip6'): - if not grid: - if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: - grid = self.config.cmor.default_ocean_grid - else: - grid = self.config.cmor.default_atmos_grid - file_name = '{0}_{1}_{2}_{3}_{4}_{5}{6}'.format(var, cmor_table.name, self.experiment.model, - self.experiment_name(startdate), - self._get_member_str(member), - grid, time_bound) - # Meteofrance - else: - time_bound = self._get_chunk_time_bounds(startdate, chunk) - file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self._get_member_str(member)) - return file_name - - def _get_time_component(self, chunk, date_str, frequency, startdate, year): - if chunk is not None: - time_bound = self._get_chunk_time_bounds(startdate, chunk) - elif year: - if frequency != Frequencies.yearly: - raise ValueError('Year may be provided instead of chunk only if frequency is "yr"') - time_bound = str(year) - else: - time_bound = date_str - return time_bound - - def _get_full_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): - if self.config.data_convention in ('specs', 'preface'): - folder_path = os.path.join(self._get_startdate_path(startdate), str(frequency), domain.name, var) - if grid: - folder_path = os.path.join(folder_path, grid) - folder_path = os.path.join(folder_path, self._get_member_str(member)) - if self.config.cmor.version: - folder_path = os.path.join(folder_path, self.config.cmor.version) - - elif self.config.data_convention in ('primavera', 'cmip6'): - if not self.config.cmor.version: - raise ValueError('CMOR version is mandatory for PRIMAVERA and CMIP6') - if not grid: - if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: - grid = self.config.cmor.default_ocean_grid - else: - grid = self.config.cmor.default_atmos_grid - if cmor_var is None: - table_name = domain.get_table(frequency, self.config.data_convention).name - else: - table_name = cmor_var.get_table(frequency, self.config.data_convention).name - folder_path = os.path.join(self._get_startdate_path(startdate), self._get_member_str(member), - table_name, var, - grid, self.config.cmor.version) - elif self.config.data_convention == 'meteofrance': - folder_path = os.path.join(self.config.data_dir, self.experiment_name(startdate), - 'H{0}'.format(chr(64 + int(startdate[4:6]))), - startdate[0:4]) - else: - raise ValueError('Data convention {0} not supported'.format(self.config.data_convention)) - return folder_path - - def _get_chunk_time_bounds(self, startdate, chunk): - start = parse_date(startdate) - chunk_start = chunk_start_date(start, chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) - chunk_end = chunk_end_date(chunk_start, self.experiment.chunk_size, 'month', self.experiment.calendar) - chunk_end = previous_day(chunk_end, self.experiment.calendar) - if self.config.data_convention == 'preface': - separator = '_' - else: - separator = '-' - if self.config.data_convention == 'meteofrance': - time_bound = "{0:04}{1:02}".format(chunk_start.year, chunk_start.month) - else: - time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, - chunk_end.month, separator) - return time_bound - def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ @@ -410,75 +259,9 @@ class CMORManager(DataManager): """ if frequency is None: frequency = self.config.frequency - filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, - grid=grid, year=year, date_str=date_str) - self.create_link(domain, filepath, frequency, var, grid, move_old, vartype) - - def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): - """ - Create file link - - Parameters - ---------- - domain: ModelingRealm - filepath: str - frequency: Frequency - var: str - grid: str - move_old: bool - vartype: VariableType - - """ - if self.config.data_convention == 'meteofrance': - return - freq_str = frequency.folder_name(vartype) - - if not grid: - grid = 'original' - - variable_folder = self.get_varfolder(domain, var) - vargrid_folder = self.get_varfolder(domain, var, grid) - - self.lock.acquire() - try: - if grid == 'original': - link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) - Utils.create_folder_tree(link_path) - else: - link_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, vargrid_folder) - Utils.create_folder_tree(link_path) - default_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, variable_folder) - original_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, - vargrid_folder.replace('-{0}_f'.format(grid), '-original_f')) - - if os.path.islink(default_path): - os.remove(default_path) - elif os.path.isdir(default_path): - shutil.move(default_path, original_path) - os.symlink(link_path, default_path) - - if move_old and link_path not in self._checked_vars: - self._checked_vars.append(link_path) - old_path = os.path.join(self.config.data_dir, self.experiment.expid, freq_str, - 'old_{0}'.format(os.path.basename(link_path))) - regex = re.compile(var + '_[0-9]{6,8}\.nc') - for filename in os.listdir(link_path): - if regex.match(filename): - Utils.create_folder_tree(old_path) - Utils.move_file(os.path.join(link_path, filename), - os.path.join(old_path, filename)) - - link_path = os.path.join(link_path, os.path.basename(filepath)) - if os.path.lexists(link_path): - os.remove(link_path) - if not os.path.exists(filepath): - raise ValueError('Original file {0} does not exists'.format(filepath)) - relative_path = os.path.relpath(filepath, os.path.dirname(link_path)) - os.symlink(relative_path, link_path) - except Exception: - raise - finally: - self.lock.release() + filepath = self.convention.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, + grid=grid, year=year, date_str=date_str) + self.convention.create_link(domain, filepath, frequency, var, grid, move_old, vartype) def prepare(self): """ @@ -492,7 +275,7 @@ class CMORManager(DataManager): """ # Check if cmorized and convert if not - if self.config.data_convention == 'meteofrance': + if self.config.data_convention.name == 'meteofrance': return if self.config.cmor.skip_prepare: return @@ -510,10 +293,14 @@ class CMORManager(DataManager): continue if not self.config.cmor.force_untar: Log.debug('Checking chunk {0}...', chunk) + skip = False for domain in (ModelingRealms.atmos, ModelingRealms.ocean, ModelingRealms.seaIce): if self.is_cmorized(startdate, member, chunk, domain): Log.debug('Chunk {0} ready', chunk) - return + skip = True + break + if skip: + continue if self._unpack_chunk(startdate, member, chunk): cmorized = True if cmorized: @@ -542,47 +329,11 @@ class CMORManager(DataManager): identifier = (startdate, member, chunk) if identifier not in self._dic_cmorized: self._dic_cmorized[identifier] = {} - self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain) + self._dic_cmorized[identifier][domain] = self.convention.is_cmorized(startdate, member, chunk, domain) elif domain not in self._dic_cmorized[identifier]: - self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain) + self._dic_cmorized[identifier][domain] = self.convention.is_cmorized(startdate, member, chunk, domain) return self._dic_cmorized[identifier][domain] - def _is_cmorized(self, startdate, member, chunk, domain): - startdate_path = self._get_startdate_path(startdate) - if not os.path.isdir(startdate_path): - return False - count = 0 - if self.config.data_convention == 'specs': - for freq in os.listdir(startdate_path): - domain_path = os.path.join(startdate_path, freq, domain.name) - if os.path.isdir(domain_path): - count = self._check_var_presence(domain_path, count, startdate, member, domain, chunk, freq) - if count >= self.config.cmor.min_cmorized_vars: - return True - else: - member_path = os.path.join(startdate_path, self._get_member_str(member)) - if not os.path.isdir(member_path): - return False - freq = Frequencies.monthly - table = domain.get_table(freq, self.config.data_convention) - table_dir = os.path.join(member_path, table.name) - if not os.path.isdir(table_dir): - return False - count = self._check_var_presence(table_dir, count, startdate, member, domain, chunk, freq) - if count >= self.config.cmor.min_cmorized_vars: - return True - return False - - def _check_var_presence(self, folder, current_count, startdate, member, domain, chunk, freq): - for var in os.listdir(folder): - cmor_var = self.variable_list.get_variable(var, True) - var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq) - if os.path.isfile(var_path): - current_count += 1 - if current_count >= self.config.cmor.min_cmorized_vars: - break - return current_count - def _cmorize_member(self, startdate, member): start_time = datetime.now() member_str = self.experiment.get_member_str(member) @@ -604,7 +355,7 @@ class CMORManager(DataManager): Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) Utils.untar(filepaths, os.path.join(self.cmor_path, 'cmorfiles')) self._correct_paths(startdate) - self.create_links(startdate, member) + self.convention.create_links(startdate, member) return True return False @@ -629,7 +380,8 @@ class CMORManager(DataManager): self._fix_model_as_experiment_error(startdate) def _fix_model_as_experiment_error(self, startdate): - if self.experiment_name(startdate) != self.experiment.model: + experiment_name = self.convention.experiment_name(startdate) + if experiment_name != self.experiment.model: bad_path = os.path.join(self.cmor_path, self.experiment.institute, self.experiment.model, self.experiment.model) Log.debug('Correcting double model appearance') @@ -641,15 +393,15 @@ class CMORManager(DataManager): good = filepath good = good.replace('_{0}_output_'.format(self.experiment.model), '_{0}_{1}_S{2}_'.format(self.experiment.model, - self.experiment_name(startdate), + experiment_name, startdate)) good = good.replace('/{0}/{0}'.format(self.experiment.model), '/{0}/{1}'.format(self.experiment.model, - self.experiment_name(startdate))) + experiment_name)) Utils.move_file(filepath, good) - if self.experiment.model != self.experiment_name(startdate): + if self.experiment.model != experiment_name: os.rmdir(dirpath) Log.debug('Done') @@ -660,108 +412,6 @@ class CMORManager(DataManager): Utils.move_tree(bad_path, self.cmor_path) Log.debug('Done') - def create_links(self, startdate, member=None): - """ - Create links for a gicen startdate or member - - Parameters - ---------- - startdate: str - member: int or None - - Returns - ------- - ValueError: - If the data convention is not supported - - """ - if member is not None: - member_str = self._get_member_str(member) - else: - member_str = None - Log.info('Creating links for CMOR files ({0})', startdate) - path = self._get_startdate_path(startdate) - if self.config.data_convention.upper() in ('SPECS', 'APPLICATE'): - self._create_links_cmip5(member_str, path) - elif self.config.data_convention.upper() in ('CMIP6', 'PRIMAVERA'): - self._create_links_cmip6(member_str, path) - else: - raise ValueError('Dataset convention {0} not supported for massive ' - 'link creation'.format(self.config.data_convention)) - Log.debug('Links ready') - - def _create_links_cmip5(self, member_str, path): - for freq in os.listdir(path): - Log.debug('Creating links for frequency {0}', freq) - frequency = Frequency.parse(freq) - for domain in os.listdir(os.path.join(path, freq)): - Log.debug('Creating links for domain {0}', domain) - for var in os.listdir(os.path.join(path, freq, domain)): - for member in os.listdir(os.path.join(path, freq, domain, var)): - if member_str is not None and member_str != member: - continue - for name in os.listdir(os.path.join(path, freq, domain, var, member)): - filepath = os.path.join(path, freq, domain, var, member, name) - if os.path.isfile(filepath): - self.create_link(domain, filepath, frequency, var, "", False, - vartype=VariableType.MEAN) - else: - for filename in os.listdir(filepath): - self.create_link(domain, os.path.join(filepath, filename), frequency, var, "", - False, vartype=VariableType.MEAN) - - def _create_links_cmip6(self, member_str, path): - for member in os.listdir(path): - for table in os.listdir(os.path.join(path, member)): - frequency = self.variable_list.tables[table].frequency - domain = None - Log.debug('Creating links for table {0}', table) - for var in os.listdir(os.path.join(path, member, table)): - for grid in os.listdir(os.path.join(path, member, table, var)): - if member_str is not None and member_str != member: - continue - for name in os.listdir(os.path.join(path, member, table, var, grid)): - filepath = os.path.join(path, member, table, var, grid, name) - if os.path.isfile(filepath): - self.create_link(domain, filepath, frequency, var, "", False, - vartype=VariableType.MEAN) - else: - for filename in os.listdir(filepath): - cmorfile = os.path.join(filepath, filename) - self.create_link(domain, cmorfile, frequency, var, "", - False, vartype=VariableType.MEAN) - - def _get_startdate_path(self, startdate): - """ - Return the path to the startdate's CMOR folder - - Parameters - ---------- - startdate: str - - Returns - ------- - str - """ - if self.config.data_convention == 'specs': - return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, - self.experiment.model, self.experiment_name(startdate), 'S' + startdate) - elif self.config.data_convention == 'preface': - return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.experiment.institute, - self.experiment_name(startdate), 'S' + startdate) - else: - return os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles', self.config.cmor.activity, - self.experiment.institute, self.experiment.model, self.experiment_name(startdate)) - - def _get_member_str(self, member): - if self.config.data_convention in ('specs', 'preface'): - template = 'r{0}i{1}p1' - elif self.config.data_convention in ('primavera', 'cmip6'): - template = 'r{0}i{1}p1f1' - elif self.config.data_convention == 'meteofrance': - return '{0:02d}'.format(member) - return template.format(member + 1 - self.experiment.member_count_start, self.config.cmor.initialization_number) - class MergeYear(Diagnostic): """ diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 2b2dea62b1ddc969203a81a7c47d2695a55f7fcc..3cfcc74ab17ea5d14c5f45228c418123eae9a71f 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -12,6 +12,8 @@ from earthdiagnostics import cdftools from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.modelingrealm import ModelingRealm from earthdiagnostics.variable import VariableManager +from earthdiagnostics.data_convention import SPECSConvention, CMIP6Convention, PrimaveraConvention, \ + MeteoFranceConvention, PrefaceConvention class ConfigException(Exception): @@ -137,22 +139,32 @@ class Config(object): self.mask_regions = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS', '') self.mask_regions_3d = parser.get_path_option('DIAGNOSTICS', 'MASK_REGIONS_3D', '') - self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', - ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), - 'specs', - ignore_case=True) - - if self.data_convention in ('primavera', 'cmip6'): - self.scratch_masks = os.path.join(self.scratch_masks, self.data_convention) - cdftools.data_convention = self.data_convention - namelist_file = os.path.join(os.path.dirname(__file__), 'CDFTOOLS_{0}.namlist'.format(self.data_convention)) + data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', + ('specs', 'primavera', 'cmip6', 'preface', 'meteofrance'), + 'specs', + ignore_case=True) + + if data_convention == 'specs': + self.data_convention = SPECSConvention(data_convention, self) + elif data_convention == 'primavera': + self.data_convention = PrimaveraConvention(data_convention, self) + elif data_convention == 'cmip6': + self.data_convention = CMIP6Convention(data_convention, self) + elif data_convention == 'preface': + self.data_convention = PrefaceConvention(data_convention, self) + elif data_convention == 'meteofrance': + self.data_convention == MeteoFranceConvention(data_convention, self) + + self.scratch_masks = self.data_convention.get_scratch_masks(self.scratch_masks) + namelist_file = os.path.join(os.path.dirname(__file__), + 'CDFTOOLS_{0}.namlist'.format(self.data_convention.name)) Log.debug(namelist_file) if os.path.isfile(namelist_file): Log.debug('Setting namelist {0}', namelist_file) os.environ['NAM_CDF_NAMES'] = namelist_file self.var_manager = VariableManager() - self.var_manager.load_variables(self.data_convention) + self.var_manager.load_variables(data_convention) self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') self.skip_diags_done = parser.get_bool_option('DIAGNOSTICS', 'SKIP_DIAGS_DONE', True) self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) diff --git a/earthdiagnostics/data_convention.py b/earthdiagnostics/data_convention.py new file mode 100644 index 0000000000000000000000000000000000000000..1f1bd51d2185423db53706e2bf6bfbf4273809e6 --- /dev/null +++ b/earthdiagnostics/data_convention.py @@ -0,0 +1,472 @@ +import os +import shutil +import re +import threading + +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, previous_day, add_hours +from bscearth.utils.log import Log + +from earthdiagnostics.frequency import Frequency, Frequencies +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.variable import VariableType +from earthdiagnostics.utils import Utils + + +class DataConvention(object): + + def __init__(self, name, config): + self.config = config + self.name = name + self.lat_name = 'lat' + self.lon_name = 'lon' + self.time_separator = '-' + self.lock = threading.Lock() + self._checked_vars = list() + + def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, + grid=None, year=None, date_str=None): + """ + Return the path to a concrete file + + Parameters + ---------- + startdate: str + member: int + domain: ModelingRealm + var: str + cmor_var: Variable + chunk: int or None + frequency: Frequency or str + grid: str or None + year: int or None + date_str: str or None + + Returns + ------- + str + + Raises + ------ + ValueError + If you provide two or more parameters from chunk, year or date_str or none at all + + """ + if frequency is None: + frequency = self.config.frequency + frequency = Frequency.parse(frequency) + folder_path = self.get_cmor_folder_path(startdate, member, domain, var, frequency, grid, cmor_var) + file_name = self.get_file_name(startdate, member, domain, var, cmor_var, frequency, + chunk, year, date_str, grid) + + filepath = os.path.join(folder_path, file_name) + return filepath + + def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + raise NotImplementedError + + def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): + raise NotImplementedError + + def get_startdate_path(self, startdate): + """ + Return the path to the startdate's CMOR folder + + Parameters + ---------- + startdate: str + + Returns + ------- + str + """ + return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', self.config.cmor.activity, + self.config.experiment.institute, self.config.experiment.model, + self.experiment_name(startdate)) + + def experiment_name(self, startdate): + """ + Get experiment name, appending startdate if needed + + Parameters + ---------- + startdate: str + + Returns + ------- + str + + """ + if self.config.cmor.append_startdate: + return '{}S{}'.format(self.config.experiment.experiment_name, startdate) + else: + return self.config.experiment.experiment_name + + def get_member_str(self, member): + raise NotImplementedError + + def create_links(self, startdate, member=None): + if member is not None: + member_str = self.get_member_str(member) + else: + member_str = None + Log.info('Creating links for CMOR files ({0})', startdate) + path = self.get_startdate_path(startdate) + self._link_startdate(path, member_str) + Log.debug('Links ready') + + def _link_startdate(self, path, member): + raise NotImplementedError + + def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + """ + Create file link + + Parameters + ---------- + domain: ModelingRealm + filepath: str + frequency: Frequency + var: str + grid: str + move_old: bool + vartype: VariableType + + """ + freq_str = frequency.folder_name(vartype) + + if not grid: + grid = 'original' + + variable_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, + self.config.experiment.atmos_timestep) + vargrid_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep, + self.config.experiment.atmos_timestep, grid=grid) + + self.lock.acquire() + try: + expid = self.config.experiment.expid + if grid == 'original': + link_path = os.path.join(self.config.data_dir, expid, freq_str, variable_folder) + Utils.create_folder_tree(link_path) + else: + link_path = os.path.join(self.config.data_dir, expid, freq_str, vargrid_folder) + Utils.create_folder_tree(link_path) + default_path = os.path.join(self.config.data_dir, expid, freq_str, variable_folder) + original_path = os.path.join(self.config.data_dir, expid, freq_str, + vargrid_folder.replace('-{0}_f'.format(grid), '-original_f')) + + if os.path.islink(default_path): + os.remove(default_path) + elif os.path.isdir(default_path): + shutil.move(default_path, original_path) + os.symlink(link_path, default_path) + + if move_old and link_path not in self._checked_vars: + self._checked_vars.append(link_path) + old_path = os.path.join(self.config.data_dir, expid, freq_str, + 'old_{0}'.format(os.path.basename(link_path))) + regex = re.compile(var + '_[0-9]{6,8}[.]nc') + for filename in os.listdir(link_path): + if regex.match(filename): + Utils.create_folder_tree(old_path) + Utils.move_file(os.path.join(link_path, filename), + os.path.join(old_path, filename)) + + link_path = os.path.join(link_path, os.path.basename(filepath)) + if os.path.lexists(link_path): + os.remove(link_path) + if not os.path.exists(filepath): + raise ValueError('Original file {0} does not exists'.format(filepath)) + relative_path = os.path.relpath(filepath, os.path.dirname(link_path)) + os.symlink(relative_path, link_path) + except Exception: + raise + finally: + self.lock.release() + + def _get_time_component(self, chunk, date_str, frequency, startdate, year): + if chunk is not None: + time_bound = self._get_chunk_time_bounds(startdate, chunk, frequency) + elif year: + if frequency != Frequencies.yearly: + raise ValueError('Year may be provided instead of chunk only if frequency is "yr"') + time_bound = str(year) + elif date_str: + time_bound = date_str + else: + raise ValueError('Time info not provided') + return time_bound + + def _get_chunk_time_bounds(self, startdate, chunk, frequency): + start = parse_date(startdate) + chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month', + self.config.experiment.calendar) + chunk_end = chunk_end_date(chunk_start, self.config.experiment.chunk_size, 'month', + self.config.experiment.calendar) + chunk_end = previous_day(chunk_end, self.config.experiment.calendar) + time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, + chunk_end.month, self.time_separator) + return time_bound + + def _check_var_presence(self, folder, current_count, startdate, member, domain, chunk, freq): + for var in os.listdir(folder): + cmor_var = self.config.var_manager.get_variable(var, True) + var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq) + if os.path.isfile(var_path): + current_count += 1 + if current_count >= self.config.cmor.min_cmorized_vars: + break + return current_count + + def is_cmorized(self, startdate, member, chunk, domain): + raise NotImplementedError + + +class Cmor2Convention(DataConvention): + + def get_scratch_masks(self, scratch_masks): + return scratch_masks + + def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + if cmor_var is None: + cmor_table = domain.get_table(frequency, self.config.data_convention) + else: + cmor_table = cmor_var.get_table(frequency, self.config.data_convention) + + time_bound = self._get_time_component(chunk, date_str, frequency, startdate, year) + time_bound = '_{0}.nc'.format(time_bound) + + file_name = '{0}_{1}_{2}_{3}_S{4}_{5}{6}'.format(var, cmor_table.name, self.config.experiment.model, + self.experiment_name(startdate), startdate, + self.get_member_str(member), time_bound) + return file_name + + def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): + folder_path = os.path.join(self.get_startdate_path(startdate), str(frequency), domain.name, var) + if grid: + folder_path = os.path.join(folder_path, grid) + folder_path = os.path.join(folder_path, self.get_member_str(member)) + if self.config.cmor.version: + folder_path = os.path.join(folder_path, self.config.cmor.version) + return folder_path + + def get_member_str(self, member): + template = 'r{0}i{1}p1' + return template.format(member + 1 - self.config.experiment.member_count_start, + self.config.cmor.initialization_number) + + def _link_startdate(self, path, member_str): + for freq in os.listdir(path): + Log.debug('Creating links for frequency {0}', freq) + frequency = Frequency.parse(freq) + for domain in os.listdir(os.path.join(path, freq)): + Log.debug('Creating links for domain {0}', domain) + for var in os.listdir(os.path.join(path, freq, domain)): + for member in os.listdir(os.path.join(path, freq, domain, var)): + if member_str is not None and member_str != member: + continue + for name in os.listdir(os.path.join(path, freq, domain, var, member)): + filepath = os.path.join(path, freq, domain, var, member, name) + if os.path.isfile(filepath): + self.create_link(domain, filepath, frequency, var, "", False, + vartype=VariableType.MEAN) + else: + for filename in os.listdir(filepath): + self.create_link(domain, os.path.join(filepath, filename), frequency, var, + "", False, vartype=VariableType.MEAN) + + def is_cmorized(self, startdate, member, chunk, domain): + startdate_path = self.get_startdate_path(startdate) + if not os.path.isdir(startdate_path): + return False + count = 0 + for freq in os.listdir(startdate_path): + domain_path = os.path.join(startdate_path, freq, domain.name) + if os.path.isdir(domain_path): + count = self._check_var_presence(domain_path, count, startdate, member, domain, chunk, freq) + if count >= self.config.cmor.min_cmorized_vars: + return True + return False + + +class SPECSConvention(Cmor2Convention): + + def get_startdate_path(self, startdate): + return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', + self.config.experiment.institute, + self.config.experiment.model, self.experiment_name(startdate), 'S' + startdate) + + +class PrefaceConvention(Cmor2Convention): + + def __init__(self, name, config): + super(PrefaceConvention, self).__init__(name, config) + self.time_separator = '_' + + def get_startdate_path(self, startdate): + return os.path.join(self.config.data_dir, self.config.experiment.expid, 'cmorfiles', + self.config.experiment.institute, + self.experiment_name(startdate), 'S' + startdate) + + +class Cmor3Convention(DataConvention): + + def __init__(self, name, config): + super(Cmor3Convention, self).__init__(name, config) + self.lat_name = 'latitude' + self.lon_name = 'longitude' + + def get_scratch_masks(self, scratch_masks): + return os.path.join(scratch_masks, self.name) + + def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid, ): + if cmor_var is None: + cmor_table = domain.get_table(frequency, self.config.data_convention) + else: + cmor_table = cmor_var.get_table(frequency, self.config.data_convention) + + time_bound = self._get_time_component(chunk, date_str, frequency, startdate, year) + time_bound = '_{0}.nc'.format(time_bound) + + if not grid: + if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: + grid = self.config.cmor.default_ocean_grid + else: + grid = self.config.cmor.default_atmos_grid + file_name = '{0}_{1}_{2}_{3}_{4}_{5}{6}'.format(var, cmor_table.name, self.config.experiment.model, + self.experiment_name(startdate), + self.get_member_str(member), + grid, time_bound) + return file_name + + def _get_chunk_time_bounds(self, startdate, chunk, frequency): + start = parse_date(startdate) + chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month', + self.config.experiment.calendar) + chunk_end = chunk_end_date(chunk_start, self.config.experiment.chunk_size, 'month', + self.config.experiment.calendar) + + if frequency == Frequencies.monthly: + chunk_end = previous_day(chunk_end, self.config.experiment.calendar) + time_bound = "{0:04}{1:02}{4}{2:04}{3:02}".format(chunk_start.year, chunk_start.month, chunk_end.year, + chunk_end.month, self.time_separator) + elif frequency == Frequencies.daily: + chunk_end = previous_day(chunk_end, self.config.experiment.calendar) + time_bound = "{0.year:04}{0.month:02}{0.day:02}{2}" \ + "{1.year:04}{1.month:02}{1.day:02}".format(chunk_start, chunk_end, self.time_separator) + elif frequency.frequency.endswith('hr'): + chunk_end = add_hours(chunk_end, -int(frequency.frequency[:-2]), self.config.experiment.calendar) + time_bound = "{0.year:04}{0.month:02}{0.day:02}{0.hour:02}{0.minute:02}{2}" \ + "{1.year:04}{1.month:02}{1.day:02}{1.hour:02}{1.minute:02}".format(chunk_start, + chunk_end, + self.time_separator) + + return time_bound + + def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): + if not self.config.cmor.version: + raise ValueError('CMOR version is mandatory for PRIMAVERA and CMIP6') + if not grid: + if domain in [ModelingRealms.ocnBgchem, ModelingRealms.seaIce, ModelingRealms.ocean]: + grid = self.config.cmor.default_ocean_grid + else: + grid = self.config.cmor.default_atmos_grid + if cmor_var is None: + table_name = domain.get_table(frequency, self.config.data_convention).name + else: + table_name = cmor_var.get_table(frequency, self.config.data_convention).name + folder_path = os.path.join(self.get_startdate_path(startdate), self.get_member_str(member), + table_name, var, + grid, self.config.cmor.version) + return folder_path + + def _link_startdate(self, path, member_str): + for member in os.listdir(path): + for table in os.listdir(os.path.join(path, member)): + frequency = self.config.var_manager.tables[table].frequency + Log.debug('Creating links for table {0}', table) + for var in os.listdir(os.path.join(path, member, table)): + domain = self.config.var_manager.get_variable(var, silent=True).domain + for grid in os.listdir(os.path.join(path, member, table, var)): + if member_str is not None and member_str != member: + continue + for name in os.listdir(os.path.join(path, member, table, var, grid)): + filepath = os.path.join(path, member, table, var, grid, name) + if os.path.isfile(filepath): + self.create_link(domain, filepath, frequency, var, "", False, + vartype=VariableType.MEAN) + else: + for filename in os.listdir(filepath): + cmorfile = os.path.join(filepath, filename) + self.create_link(domain, cmorfile, frequency, var, "", + False, vartype=VariableType.MEAN) + + def get_member_str(self, member): + template = 'r{0}i{1}p1f1' + return template.format(member + 1 - self.config.experiment.member_count_start, + self.config.cmor.initialization_number) + + def is_cmorized(self, startdate, member, chunk, domain): + startdate_path = self.get_startdate_path(startdate) + if not os.path.isdir(startdate_path): + return False + count = 0 + member_path = os.path.join(startdate_path, self.get_member_str(member)) + if not os.path.isdir(member_path): + return False + freq = Frequencies.monthly + table = domain.get_table(freq, self.config.data_convention) + table_dir = os.path.join(member_path, table.name) + if not os.path.isdir(table_dir): + return False + count = self._check_var_presence(table_dir, count, startdate, member, domain, chunk, freq) + if count >= self.config.cmor.min_cmorized_vars: + return True + return False + + +class CMIP6Convention(Cmor3Convention): + pass + + +class PrimaveraConvention(Cmor3Convention): + pass + + +class MeteoFranceConvention(DataConvention): + + def get_file_name(self, startdate, member, domain, var, cmor_var, frequency, chunk, year, date_str, grid,): + if year is not None: + raise ValueError('Year not supported with MeteoFrance convention') + if date_str is not None: + raise ValueError('Date_str not supported with MeteoFrance convention') + if chunk is None: + raise ValueError('Chunk must be provided in MeteoFrance convention') + time_bound = self._get_chunk_time_bounds(startdate, chunk) + file_name = '{0}_{1}_{2}_{3}.nc'.format(var, frequency, time_bound, self.get_member_str(member)) + return file_name + + def get_cmor_folder_path(self, startdate, member, domain, var, frequency, grid, cmor_var): + folder_path = os.path.join(self.config.data_dir, self.experiment_name(startdate), + 'H{0}'.format(chr(64 + int(startdate[4:6]))), + startdate[0:4]) + return folder_path + + def get_member_str(self, member): + return '{0:02d}'.format(member) + + def _get_chunk_time_bounds(self, startdate, chunk): + start = parse_date(startdate) + chunk_start = chunk_start_date(start, chunk, self.config.experiment.chunk_size, 'month', + self.config.experiment.calendar) + time_bound = "{0:04}{1:02}".format(chunk_start.year, chunk_start.month) + return time_bound + + def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + pass + + def create_links(self, startdate, member=None): + pass + + def is_cmorized(self, startdate, member, chunk, domain): + return True diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index 490a4542dc0db1f61011036698c0aeb0013795ce..f0ec47872cf4e803cd1a2470cc05a52cb9f7b8e3 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -5,8 +5,10 @@ import os import shutil from datetime import datetime +import six import iris import iris.coords +import iris.exceptions import numpy as np from bscearth.utils.log import Log @@ -160,6 +162,7 @@ class DataFile(Publisher): ---------- diagnostic: Diagnostic + Returns ------- bool @@ -308,7 +311,7 @@ class DataFile(Publisher): var_handler = handler.variables[self.final_name] coords = set.intersection({'time', 'lev', self.lat_name, self.lon_name, 'leadtime', 'region', 'time_centered'}, set(handler.variables.keys())) - var_handler.coordinates = ' '.join(coords) + var_handler.coordinates = Utils.convert_to_ascii_if_possible(' '.join(coords)) if 'time_centered' in handler.variables: if hasattr(handler.variables['time_centered'], 'standard_name'): del handler.variables['time_centered'].standard_name @@ -615,6 +618,9 @@ class UnitConversion(object): else: return None, None + def check_is_in_storage(self): + return + class NetCDFFile(DataFile): """Implementation of DataFile for netCDF files""" @@ -648,6 +654,30 @@ class NetCDFFile(DataFile): Log.error('File {0} not available: {1}', self.remote_file, ex) self.local_status = LocalStatus.FAILED + def check_is_in_storage(self): + + if os.path.isfile(self.remote_file): + if self.region: + try: + cubes = iris.load(self.remote_file) + for cube in cubes: + try: + if isinstance(self.region, six.string_types): + regions = {self.region.name} + else: + regions = {basin.name for basin in self.region} + if regions.issubset(set(cube.coord('region').points)): + self.storage_status = StorageStatus.READY + except iris.exceptions.CoordinateNotFoundError: + pass + except iris.exceptions.TranslationError as ex: + # If the check goes wrong, we must execute everything + os.remove(self.remote_file) + except Exception as ex: + pass + else: + self.storage_status = StorageStatus.READY + def create_link(self): """Create a link from the original data in the _ folder""" try: diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 9bd4fb949d2e42af4ece538799621e7b9fc0f883..7e0b6e3158636339154d296f7f13a5f99bd00507 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -1,6 +1,5 @@ # coding: utf-8 """Base data manager for Earth diagnostics""" -import threading from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus, UnitConversion from earthdiagnostics.modelingrealm import ModelingRealms @@ -19,10 +18,8 @@ class DataManager(object): def __init__(self, config): self.config = config self.experiment = config.experiment - self._checked_vars = list() self.variable_list = config.var_manager UnitConversion.load_conversions() - self.lock = threading.Lock() self.requested_files = {} def _get_file_from_storage(self, filepath): @@ -55,42 +52,6 @@ class DataManager(object): var += box.get_lon_str() + box.get_lat_str() + box.get_depth_str() return var - def get_varfolder(self, domain, var, grid=None, frequency=None): - """Get variable folder name for _ folder""" - if grid: - var = '{0}-{1}'.format(var, grid) - - if domain in [ModelingRealms.ocean, ModelingRealms.seaIce, ModelingRealms.ocnBgchem]: - return DataManager._apply_fxh(var, self.experiment.ocean_timestep, frequency) - else: - return DataManager._apply_fxh(var, self.experiment.atmos_timestep, frequency) - - @staticmethod - def _apply_fxh(folder_name, timestep, frequency=None): - is_base_frequency = frequency is not None and frequency.frequency.endswith('hr') - if not is_base_frequency and timestep > 0: - return '{0}_f{1}h'.format(folder_name, timestep) - return folder_name - - def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): - """ - Create file link - - Must be implementd by the derived classes. If not, this method will have no effect - - Parameters - ---------- - domain: ModelingRealm - filepath: str - frequency: Frequency - var: str - grid: str - move_old: bool - vartype: VariableType - - """ - pass - def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 09c7efd746c7a24a0ddbbd12f3a2730d323d4752..86c07a62eeb33243bb867d16fbe67d48ee69e353 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -89,6 +89,7 @@ class Diagnostic(Publisher): if not self._generated_files: return False for file_generated in self._generated_files: + file_generated.check_is_in_storage() if file_generated.storage_status != StorageStatus.READY: return False if file_generated.has_modifiers(): @@ -206,8 +207,8 @@ class Diagnostic(Publisher): region = region.name generated_chunk = self.data_manager.declare_chunk(domain, var, startdate, member, chunk, grid, region, box, diagnostic=self, vartype=vartype, frequency=frequency) - if region is not None: - generated_chunk.add_modifier(self) + # if region is not None: + # generated_chunk.add_modifier(self) self._generated_files.append(generated_chunk) return generated_chunk diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 17262cd1500129d54f4007b0a361e1f60856eab7..23224c48dc7d5e7f2f4afe89b23498696a5f231b 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -157,12 +157,8 @@ class EarthDiags(object): self.dic_variables['x'] = 'i' self.dic_variables['y'] = 'j' self.dic_variables['z'] = 'lev' - if self.config.data_convention.lower() in ['primavera', 'cmip6']: - self.dic_variables['nav_lon'] = 'longitude' - self.dic_variables['nav_lat'] = 'latitude' - else: - self.dic_variables['nav_lon'] = 'lon' - self.dic_variables['nav_lat'] = 'lat' + self.dic_variables['nav_lon'] = self.config.data_convention.lon_name + self.dic_variables['nav_lat'] = self.config.data_convention.lat_name self.dic_variables['nav_lev'] = 'lev' self.dic_variables['time_counter'] = 'time' self.dic_variables['t'] = 'time' @@ -180,7 +176,6 @@ class EarthDiags(object): Log.debug('Using netCDF version {0}', netCDF4.getlibversion()) self._prepare_scratch_dir() - self._prepare_mesh_files() self._initialize_basins() diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 10a6dd0d5bc88d58073c6770ee705a8f2a62f33e..f4cb7b8282a449ccdb6c902f91a03624533c8937 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -63,4 +63,4 @@ class RelinkAll(Diagnostic): def compute(self): """Run the diagnostic""" - self.data_manager.create_links(self.startdate) + self.data_manager.convention.create_links(self.startdate) diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index af0856f44e7838765595b7c18ce7af2e7029934b..7072935803a83182926a5c532d089c487cdf214b 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -47,6 +47,20 @@ class ModelingRealm(object): def __repr__(self): return str(self) + def get_varfolder(self, var, ocean_timestep, atmos_timestep, grid=None, frequency=None): + """Get variable folder name for _ folder""" + if grid: + var = '{0}-{1}'.format(var, grid) + + if self in [ModelingRealms.ocean, ModelingRealms.seaIce, ModelingRealms.ocnBgchem]: + timestep = ocean_timestep + else: + timestep = atmos_timestep + is_base_frequency = frequency is not None and frequency.frequency.endswith('hr') + if not is_base_frequency and timestep > 0: + return '{0}_f{1}h'.format(var, timestep) + return var + def get_table_name(self, frequency, data_convention): """ Get table name for a domain-frequency pair diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py index 40c9869fffb843246d03e3ce577fcda177e217fa..892676e25b964f2a97131058fb583133d81ba7cc 100644 --- a/earthdiagnostics/obsreconmanager.py +++ b/earthdiagnostics/obsreconmanager.py @@ -71,7 +71,8 @@ class ObsReconManager(DataManager): return filepath def _get_folder_path(self, frequency, domain, variable, grid, vartype): - var_folder = self.get_varfolder(domain, variable, grid, frequency) + var_folder = domain.get_varfolder(variable, self.config.experiment.ocean_timestep, + self.config.experiment.atmos_timestep, grid=grid, frequency=frequency) folder_path = os.path.join(self.config.data_dir, self.config.data_type, self.experiment.institute.lower(), self.experiment.model.lower(), diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 8124ffc2e013b4736a948e71a242327cbfadc8ef..703301ebc9aba1744bbc43b2c29381acd1bebab0 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -42,7 +42,7 @@ class Siasiesiv(Diagnostic): e2t = None gphit = None - def __init__(self, data_manager, startdate, member, chunk, masks, var_manager, omit_vol): + def __init__(self, data_manager, startdate, member, chunk, masks, var_manager, data_convention, omit_vol): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -53,6 +53,7 @@ class Siasiesiv(Diagnostic): self.omit_volume = omit_vol self.sic_varname = self.var_manager.get_variable('sic').short_name self.sit_varname = self.var_manager.get_variable('sit').short_name + self.data_convention = data_convention self.results = {} for var in ('siarean', 'siareas', 'sivoln', 'sivols', 'siextentn', 'siextents'): @@ -89,7 +90,8 @@ class Siasiesiv(Diagnostic): job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(Siasiesiv(diags.data_manager, startdate, member, chunk, masks, - diags.config.var_manager, options['omit_volume'])) + diags.config.var_manager, diags.config.data_convention, + options['omit_volume'])) e1t = iris.load_cube('mesh_hgr.nc', 'e1t') e2t = iris.load_cube('mesh_hgr.nc', 'e2t') @@ -124,11 +126,12 @@ class Siasiesiv(Diagnostic): def compute(self): """Run the diagnostic""" + coordinates = ' '.join(('time', 'leadtime', 'time_centered', 'latitude', 'longitude')) handler = Utils.open_cdf(self.sic.local_file) - handler.variables[self.sic_varname].coordinates = 'time lat lon leadtime time_centered' + handler.variables[self.sic_varname].coordinates = coordinates handler.close() sic = iris.load_cube(self.sic.local_file) - if sic.units.origin == '%' and sic.data.max() < 2: + if sic.units.origin == '%' and sic.data.max < 2: sic.units = '1.0' sic.convert_units('1.0') sic *= Siasiesiv.area.data @@ -136,7 +139,7 @@ class Siasiesiv(Diagnostic): if not self.omit_volume: handler = Utils.open_cdf(self.sit.local_file) - handler.variables[self.sit_varname].coordinates = 'time lat lon leadtime time_centered' + handler.variables[self.sit_varname].coordinates = coordinates handler.close() sit = iris.load_cube(self.sic.local_file) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index be37da89bca569bbffa6cc115a76f07c3bb88571..b0b279566b4cd9ad24781f8a1f111cf81d16686c 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -113,7 +113,8 @@ class THREDDSManager(DataManager): def _get_folder_path(self, frequency, domain, variable, grid, vartype): if self.config.data_type == 'exp': - var_folder = self.get_varfolder(domain, variable, grid) + var_folder = domain.get_varfolder(variable, self.config.experiment.ocean_timestep, + self.config.experiment.atmos_timestep, grid=grid) else: var_folder = variable diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 36efe266db94b500b63533ed0f9a15defa50a1a3..597c274775dfc2c32910b45da1b5c45cb144d38c 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -283,7 +283,7 @@ class VariableManager(object): sheet = excel[sheet_name] if sheet.title == 'Primday': pass - if sheet['A1'].value != 'Priority': + if sheet['A1'].value not in ['Priority', 'rm']: continue self._load_xlsx_table(sheet, table_data) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 751947e244406426ce8d4a551cc3d38762334917..f6bd3ae96e7235a6781de027e913b0b13b8baffa 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -117,7 +117,7 @@ class WorkManager(object): self.downloader.start() printer_lock = threading.Lock() printer_lock.acquire() - printer = Thread(target=self._print_status, name="Printer", args=(printer_lock,) ) + printer = Thread(target=self._print_status, name="Printer", args=(printer_lock,)) printer.start() self.lock.acquire() diff --git a/test/integration/test_cmorizer.py b/test/integration/test_cmorizer.py index 0f5fb8ea7d8ddf3efa9dbfb0b22d532c12813403..e7b0d1291b65a8977fd542aa2e900f4fdb7dfae7 100644 --- a/test/integration/test_cmorizer.py +++ b/test/integration/test_cmorizer.py @@ -13,6 +13,7 @@ from iris.coords import DimCoord import tarfile import numpy as np import six +import calendar class TestCmorizer(TestCase): @@ -333,21 +334,21 @@ class TestCmorizer(TestCase): def _test_atmos_cmor(self): - if six.PY3: - - with self.assertLogs(log.Log.log) as cmd: - cmorizer = Cmorizer(self.data_manager, '19900101', 0) - cmorizer.cmorize_atmos() - - try: - self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) - self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) - except AssertionError: - print(cmd) - raise - else: + # if six.PY3: + # + # with self.assertLogs(log.Log.log) as cmd: + # cmorizer = Cmorizer(self.data_manager, '19900101', 0) + # cmorizer.cmorize_atmos() + # + # try: + # self.assertTrue([record for record in cmd.records if record.levelno == log.Log.RESULT]) + # self.assertFalse([record for record in cmd.records if record.levelno == log.Log.ERROR]) + # self.assertFalse([record for record in cmd.records if record.levelno == log.Log.CRITICAL]) + # self.assertFalse([record for record in cmd.records if record.levelno == log.Log.WARNING]) + # except AssertionError: + # print(cmd) + # raise + # else: cmorizer = Cmorizer(self.data_manager, '19900101', 0) cmorizer.cmorize_atmos() @@ -439,21 +440,22 @@ class TestCmorizer(TestCase): month) def _create_file_for_grib(self, coord_data, folder_path, filename, codes, month): - month -= 1 lat = DimCoord(coord_data, standard_name='latitude', long_name='latitude', var_name='lat', units='degrees_north') lon = DimCoord(coord_data, standard_name='longitude', long_name='longitude', var_name='lon', units='degrees_east') - time_data = np.array([0.25, 0.50], np.float) + month * 31 + month_days = calendar.monthrange(1990, month)[1] + month -= 1 + time_data = np.arange(0.25, month_days + 0.25, 0.25, np.float) + month * 31 time = DimCoord(time_data, standard_name='time', long_name='time', var_name='time', - units='days since 1990-01-01') + units='days since 1990-01-01 00:00:00') vars = [] for code in codes: - var = iris.cube.Cube(np.ones((2, 2, 2), np.float) * code, + var = iris.cube.Cube(np.ones((month_days * 4, 2, 2), np.float) * code, long_name='Variable {}'.format(code), var_name='var{}'.format(code)) - var.data[0, ...] += time_data[0] - var.data[1, ...] += time_data[1] + for x, data in enumerate(time_data): + var.data[x, ...] += data var.add_dim_coord(time, 0) var.add_dim_coord(lat, 1) var.add_dim_coord(lon, 2) @@ -489,11 +491,27 @@ class TestCmorizer(TestCase): } for var, code in six.iteritems(variables): self.assertTrue(os.path.isdir(os.path.join(self.tmp_dir, var))) - base_data = np.ones((2, 2, 2), np.float) * code - factor = 1 - month_offsets = np.array((0.375, 31.375)) - daily_offsets = np.array((0.375, 31.375)) - hourly_offsets = np.array((0.25, 0.5, 31.25, 31.5)) + base_data = np.ones((2, 2), np.float) * code + + if var in ('EWSS', 'TP', 'MN2T', 'MX2T', 'SSRD', 'TP'): + if var == 'MX2T': + month_offsets = np.array((16, 45.5)) + daily_offsets = np.arange(1.0, 60.0) + elif var == 'MN2T': + month_offsets = np.array((15.25, 44.75)) + daily_offsets = np.arange(0.25, 59.25) + else: + month_offsets = np.array((15.625, 45.125)) + daily_offsets = np.arange(0.625, 59.625) + + hourly_offsets = np.arange(0.25, 59.25, 0.25) + else: + month_offsets = np.array((15.5, 44.875)) + daily_offsets = np.arange(0.375, 59.375) + daily_offsets[0] = 0.5 + hourly_offsets = np.arange(0.25, 59, 0.25) + + factor = 1.0 if code == 129: factor = 9.81 elif code in (180, 169): @@ -504,12 +522,7 @@ class TestCmorizer(TestCase): daily_offsets *= 2 hourly_offsets *= 2 factor = 6 * 3600.0 / 1000 - if code == 201: - month_offsets = np.array((0.5, 31.5)) - daily_offsets = np.array((0.5, 31.5)) - elif code == 202: - month_offsets = np.array((0.25, 31.25)) - daily_offsets = np.array((0.25, 31.25)) + base_data /= factor month_offsets /= factor daily_offsets /= factor @@ -528,4 +541,4 @@ class TestCmorizer(TestCase): self.assertEqual(data.coord('time').shape, (len(offsets),)) for x, offset in enumerate(offsets): self.assertTrue(np.allclose(data.data[x, ...], base_data + offset), - '{} {} data wrong for {}'.format(freq, x, var)) + '{} {} data wrong for {}: {}'.format(freq, x, var, data.data[x, ...] - base_data)) diff --git a/test/unit/data_convention/__init__.py b/test/unit/data_convention/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/unit/data_convention/test_data_convention.py b/test/unit/data_convention/test_data_convention.py new file mode 100644 index 0000000000000000000000000000000000000000..4ce42ac0b5a802edf6b8fbf223a2c902193072d9 --- /dev/null +++ b/test/unit/data_convention/test_data_convention.py @@ -0,0 +1,188 @@ +import os +import shutil + +from unittest import TestCase +from mock import Mock +import mock +import tempfile + +from earthdiagnostics.data_convention import DataConvention +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestDataConvention(TestCase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, 'expid')) + + self.config = Mock() + self.config.experiment.experiment_name = 'expname' + self.config.data_dir = self.tmp_dir + self.config.experiment.expid = 'expid' + self.config.cmor.activity = 'activity' + + self.config.experiment.institute = 'institute' + self.config.experiment.model = 'model' + self.config.experiment.atmos_timestep = 3 + self.config.experiment.ocean_timestep = 6 + self.config.cmor.append_startdate = False + self.convention = DataConvention('name', self.config) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_get_filename(self): + with self.assertRaises(NotImplementedError): + self.convention.get_file_name(None, None, None, None, None, None, None, None, None, None) + + def test_get_startdate_path(self): + self.assertEqual(self.convention.get_startdate_path('19900101'), + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname')) + + def test_experiment_name(self): + self.assertEqual(self.convention.experiment_name('19900101'), + 'expname') + + def test_experiment_name_append(self): + self.config.cmor.append_startdate = True + self.assertEqual(self.convention.experiment_name('19900101'), + 'expnameS19900101') + + def test_get_cmor_folder_path(self): + with self.assertRaises(NotImplementedError): + self.convention.get_cmor_folder_path(None, None, None, None, None, None, None) + + def test_create_links(self): + with self.assertRaises(NotImplementedError): + self.convention.create_links(None, None) + + def test_get_member_str(self): + with self.assertRaises(NotImplementedError): + self.convention.get_member_str(None) + + def test_get_file_path_raise_incompatible_date_info(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + frequency = Mock() + frequency.frequency = 'monthly' + frequency.__str__ = Mock() + frequency.__str__.return_value = 'frequency' + + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency, year='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, year='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, 1, frequency, year='1998', date_str='1998') + self.assertRaises(ValueError, self.convention.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', + cmor_var, None, frequency) + + def test_create_link(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + filename = os.path.basename(path) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + + def test_create_link_no_source(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.remove(path) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + with self.assertRaises(ValueError): + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + + def test_create_link_exists(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + filename = os.path.basename(path) + + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + os.symlink(path, os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename)) + + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + + def test_create_link_default_is_link(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h')) + os.symlink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h'), + os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) + filename = os.path.basename(path) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + + def test_create_link_move_old(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + fd = open(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', 'var_20001101.nc'), 'w') + fd.close() + + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', None, True, None) + filename = os.path.basename(path) + self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) + self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'old_var_f6h', + 'var_20001101.nc'))) + + def test_create_link_with_grid(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) + filename = os.path.basename(path) + var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') + var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') + self.assertTrue(os.path.islink(var_mainfolder)) + self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) + self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) + + def test_create_link_with_grid_default_is_link(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h')) + os.symlink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h'), + os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) + filename = os.path.basename(path) + var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') + var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') + self.assertTrue(os.path.islink(var_mainfolder)) + self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) + self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) + + def test_create_link_with_grid_default_is_folder(self): + file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(file_descriptor) + os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) + + frequency = Mock() + frequency.folder_name.return_value = 'frequency_folder' + self.convention.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) + filename = os.path.basename(path) + var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') + var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') + self.assertTrue(os.path.islink(var_mainfolder)) + self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) + self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) diff --git a/test/unit/data_convention/test_meteofrance.py b/test/unit/data_convention/test_meteofrance.py new file mode 100644 index 0000000000000000000000000000000000000000..825977f28d854c7a10731d9483c0814445440ae8 --- /dev/null +++ b/test/unit/data_convention/test_meteofrance.py @@ -0,0 +1,167 @@ +import os +import shutil +import tempfile + +from unittest import TestCase +from mock import Mock +import mock + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.data_convention import MeteoFranceConvention + + +class TestMeteoFranceConvention(TestCase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, 'expid')) + self.config = Mock() + self.config.data_dir = self.tmp_dir + + self.config.experiment.experiment_name = 'experiment_name' + self.config.experiment.expid = 'expid' + self.config.experiment.institute = 'institute' + self.config.experiment.model = 'model' + self.config.experiment.member_count_start = 0 + self.config.experiment.chunk_size = 1 + self.config.experiment.calendar = 'standard' + + self.config.cmor.activity = 'activity' + self.config.cmor.append_startdate = False + self.config.cmor.initialization_number = 1 + self.config.cmor.version = 'version' + + self.convention = MeteoFranceConvention('name', self.config) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_get_startdate_path(self): + self.assertEqual(self.convention.get_startdate_path('19900101'), + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name')) + + def test_experiment_name(self): + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_name') + + def test_experiment_name_append(self): + self.config.cmor.append_startdate = True + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_nameS19900101') + + def test_get_cmor_folder_path(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'experiment_name/HA/1990')) + + def test_get_cmor_folder_path_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.atmos, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'experiment_name/HA/1990')) + + def test_get_cmor_folder_path_custom_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + 'grid', cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'experiment_name/HA/1990')) + + def test_get_cmor_folder_path_no_cmor(self): + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.frequency = 'mon' + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'experiment_name/HA/1990')) + + def test_get_file_path_no_version(self): + self.config.cmor.version = '' + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'experiment_name/HA/1990')) + + def test_get_filename(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_mon_199001_01.nc') + + def test_get_filename_daily(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.daily, 1, + None, None, None) + self.assertEqual(file_path, + 'var_day_199001_01.nc') + + def test_get_filename_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Amon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.atmos, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_mon_199001_01.nc') + + def test_get_filename_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, 'grid') + self.assertEqual(file_path, + 'var_mon_199001_01.nc') + + def test_get_filename_year(self): + with self.assertRaises(ValueError): + self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', None, + Frequencies.yearly, None, 1990, None, None) + + def test_get_filename_date_Str(self): + with self.assertRaises(ValueError): + self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', None, + Frequencies.yearly, None, None, 'date_str', None) + + def test_get_filename_no_date_info(self): + with self.assertRaises(ValueError): + self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', None, + Frequencies.monthly, None, None, None, None) + + def test_is_cmorized(self): + self.assertTrue(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_create_links_meteofrance(self): + self.convention.create_links(None, None) + + def test_create_link_meteofrance(self): + self.convention.create_link(None, None, None, None, None, None, None) diff --git a/test/unit/data_convention/test_preface.py b/test/unit/data_convention/test_preface.py new file mode 100644 index 0000000000000000000000000000000000000000..878428fe7ff0feaca914e6317994424536fe3aa9 --- /dev/null +++ b/test/unit/data_convention/test_preface.py @@ -0,0 +1,280 @@ +import os +import shutil +import tempfile + +from unittest import TestCase +from mock import Mock +import mock + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.data_convention import PrefaceConvention + + +class TestPrefaceConvention(TestCase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, 'expid')) + self.config = Mock() + self.config.data_dir = self.tmp_dir + + self.config.experiment.experiment_name = 'experiment_name' + self.config.experiment.expid = 'expid' + self.config.experiment.institute = 'institute' + self.config.experiment.model = 'model' + self.config.experiment.member_count_start = 0 + self.config.experiment.chunk_size = 1 + self.config.experiment.calendar = 'standard' + + self.config.cmor.activity = 'activity' + self.config.cmor.append_startdate = False + self.config.cmor.initialization_number = 1 + self.config.cmor.version = 'version' + + self.convention = PrefaceConvention('name', self.config) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_get_startdate_path(self): + self.assertEqual(self.convention.get_startdate_path('19900101'), + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S19900101')) + + def test_experiment_name(self): + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_name') + + def test_experiment_name_append(self): + self.config.cmor.append_startdate = True + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_nameS19900101') + + def test_get_cmor_folder_path(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S19900101/mon/' + 'ocean/var/r2i1p1/version')) + + def test_get_cmor_folder_path_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.atmos, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S19900101/mon/' + 'atmos/var/r2i1p1/version')) + + def test_get_cmor_folder_path_custom_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + 'grid', cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S19900101/mon/' + 'ocean/var/grid/r2i1p1/version')) + + def test_get_cmor_folder_path_no_cmor(self): + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.frequency = 'mon' + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S19900101/mon/' + 'ocean/var/r2i1p1/version')) + + def test_get_file_path_no_version(self): + self.config.cmor.version = '' + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S19900101/mon/' + 'ocean/var/r2i1p1')) + + def test_get_filename(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001_199001.nc') + + def test_get_filename_daily(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.daily, 1, + None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001_199001.nc') + + def test_get_filename_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Amon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.atmos, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Amon_model_experiment_name_S19900101_r2i1p1_199001_199001.nc') + + def test_get_filename_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, 'grid') + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001_199001.nc') + + def test_get_filename_year(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.yearly, None, 1990, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_1990.nc') + + def test_get_filename_date_Str(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, None, None, 'date_str', None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_date_str.nc') + + def test_get_filename_no_date_info(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + + with self.assertRaises(ValueError): + self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, None, None, None, None) + + @mock.patch('os.path.isfile') + def test_is_cmorized(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/experiment_name/S20000101/mon/' + 'ocean/var/r2i1p1/version')) + self.assertTrue(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_not_cmorized(self, mock_is_file): + mock_is_file.return_value = False + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/experiment_name/S20000101/mon/ocean/var/' + 'r2i1p1/version')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_false_not_member_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/experiment_name/S20000101/mon/ocean/var/')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_cmorized_not_enough_vars(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs( + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S20000101/mon/ocean/var/' + 'r2i1p1/version')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_not_domain_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S20000101/mon/' + 'r2i1p1/version')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_not_freq_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/experiment_name/S20000101')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('earthdiagnostics.data_convention.PrefaceConvention.create_link') + def test_create_links(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/experiment_name/S20010101/mon/ocean/var/r2i1p1') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.data_convention.PrefaceConvention.create_link') + def test_create_links_member_not_found(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/experiment_name/S20010101/mon/ocean/var/r1i1p1') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_not_called() + + @mock.patch('earthdiagnostics.data_convention.PrefaceConvention.create_link') + def test_create_links_with_grid(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/experiment_name/S20010101/mon/ocean/var/' + 'r2i1p1/grid') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_called() diff --git a/test/unit/data_convention/test_primavera.py b/test/unit/data_convention/test_primavera.py new file mode 100644 index 0000000000000000000000000000000000000000..271f8db2b2b98a7e2193af6f3f45928095a33a9c --- /dev/null +++ b/test/unit/data_convention/test_primavera.py @@ -0,0 +1,312 @@ +import os +import shutil +import tempfile + +from unittest import TestCase +from mock import Mock +import mock + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.data_convention import PrimaveraConvention + + +class TestPrimaveraConvention(TestCase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, 'expid')) + self.config = Mock() + self.config.data_dir = self.tmp_dir + + self.config.experiment.experiment_name = 'experiment_name' + self.config.experiment.expid = 'expid' + self.config.experiment.institute = 'institute' + self.config.experiment.model = 'model' + self.config.experiment.member_count_start = 0 + self.config.experiment.chunk_size = 1 + self.config.experiment.calendar = 'standard' + + self.config.cmor.activity = 'activity' + self.config.cmor.append_startdate = False + self.config.cmor.initialization_number = 1 + self.config.cmor.version = 'version' + self.config.cmor.default_ocean_grid = 'ocean_grid' + self.config.cmor.default_atmos_grid = 'atmos_grid' + + self.convention = PrimaveraConvention('name', self.config) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_get_startdate_path(self): + self.assertEqual(self.convention.get_startdate_path('19900101'), + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name')) + + def test_experiment_name(self): + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_name') + + def test_experiment_name_append(self): + self.config.cmor.append_startdate = True + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_nameS19900101') + + def test_get_cmor_folder_path(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' + 'r2i1p1f1/Omon/var/ocean_grid/version')) + + def test_get_cmor_folder_path_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.atmos, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' + 'r2i1p1f1/Omon/var/atmos_grid/version')) + + def test_get_cmor_folder_path_custom_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + 'grid', cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' + 'r2i1p1f1/Omon/var/grid/version')) + + def test_get_cmor_folder_path_no_cmor(self): + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.frequency = 'mon' + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name/' + 'r2i1p1f1/Omon/var/ocean_grid/version')) + + def test_get_file_path_no_version_primavera(self): + self.config.cmor.version = '' + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + with self.assertRaises(ValueError): + self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', 'grid', cmor_var) + + def test_get_filename(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc') + + def test_get_filename_daily(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.daily, 1, + None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_19900101-19900131.nc') + + def test_get_filename_6hourly(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.six_hourly, 1, + None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001010000-199001311800.nc') + + def test_get_filename_3hourly(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.three_hourly, 1, + None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001010000-199001312100.nc') + + def test_get_filename_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.atmos, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_atmos_grid_199001-199001.nc') + + def test_get_filename_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, 'grid') + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_grid_199001-199001.nc') + + def test_get_filename_year(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.yearly, None, 1990, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_1990.nc') + + def test_get_filename_date_Str(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, None, None, 'date_str', None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_date_str.nc') + + def test_get_filename_no_date_info(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + + with self.assertRaises(ValueError): + self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, None, None, None, None) + + @mock.patch('os.path.isfile') + def test_is_cmorized(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/var')) + self.assertTrue(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_not_cmorized(self, mock_is_file): + mock_is_file.return_value = False + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/var')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_false_not_member_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_false_not_table_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_cmorized_not_enough_vars(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S20000101/' + 'mon/ocean/var')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_not_domain_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') + def test_create_links_primavera(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/var/gn') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') + def test_create_links_with_version_primavera(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/' + 'var/gn/version') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') + def test_create_links_with_version_primavera_no_member(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/' + 'var/gn/version') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101') + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link') + def test_create_links_member_not_found_primavera(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/activity/institute/model/experiment_name/r1i1p1f1/Omon/var/gn') + os.makedirs(member_path) + self.config.var_manager.tables = {'Omon': Mock()} + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_not_called() diff --git a/test/unit/data_convention/test_specs.py b/test/unit/data_convention/test_specs.py new file mode 100644 index 0000000000000000000000000000000000000000..82e7e982ed722f21cb9fca23ccdb309bb0f019ac --- /dev/null +++ b/test/unit/data_convention/test_specs.py @@ -0,0 +1,280 @@ +import os +import shutil +import tempfile + +from unittest import TestCase +from mock import Mock +import mock + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.data_convention import SPECSConvention + + +class TestSpecsConvention(TestCase): + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, 'expid')) + self.config = Mock() + self.config.data_dir = self.tmp_dir + + self.config.experiment.experiment_name = 'experiment_name' + self.config.experiment.expid = 'expid' + self.config.experiment.institute = 'institute' + self.config.experiment.model = 'model' + self.config.experiment.member_count_start = 0 + self.config.experiment.chunk_size = 1 + self.config.experiment.calendar = 'standard' + + self.config.cmor.activity = 'activity' + self.config.cmor.append_startdate = False + self.config.cmor.initialization_number = 1 + self.config.cmor.version = 'version' + + self.convention = SPECSConvention('name', self.config) + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_get_startdate_path(self): + self.assertEqual(self.convention.get_startdate_path('19900101'), + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S19900101')) + + def test_experiment_name(self): + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_name') + + def test_experiment_name_append(self): + self.config.cmor.append_startdate = True + self.assertEqual(self.convention.experiment_name('19900101'), + 'experiment_nameS19900101') + + def test_get_cmor_folder_path(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S19900101/mon/' + 'ocean/var/r2i1p1/version')) + + def test_get_cmor_folder_path_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.atmos, 'var', 'mon', + None, cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S19900101/mon/' + 'atmos/var/r2i1p1/version')) + + def test_get_cmor_folder_path_custom_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', + 'grid', cmor_var) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S19900101/mon/' + 'ocean/var/grid/r2i1p1/version')) + + def test_get_cmor_folder_path_no_cmor(self): + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = 'mon' + frequency.frequency = 'mon' + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S19900101/mon/' + 'ocean/var/r2i1p1/version')) + + def test_get_file_path_no_version(self): + self.config.cmor.version = '' + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, None, None) + self.assertEqual(file_path, + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S19900101/mon/' + 'ocean/var/r2i1p1')) + + def test_get_filename(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + + def test_get_filename_daily(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.daily, 1, + None, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + + def test_get_filename_atmos(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Amon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.atmos, 'var', cmor_var, + Frequencies.monthly, 1, None, None, None) + self.assertEqual(file_path, + 'var_Amon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + + def test_get_filename_grid(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, 1, None, None, 'grid') + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_199001-199001.nc') + + def test_get_filename_year(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.yearly, None, 1990, None, None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_1990.nc') + + def test_get_filename_date_Str(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + file_path = self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, None, None, 'date_str', None) + self.assertEqual(file_path, + 'var_Omon_model_experiment_name_S19900101_r2i1p1_date_str.nc') + + def test_get_filename_no_date_info(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + + with self.assertRaises(ValueError): + self.convention.get_file_name('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, + Frequencies.monthly, None, None, None, None) + + @mock.patch('os.path.isfile') + def test_is_cmorized(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon/ocean/var/' + 'r2i1p1/version')) + self.assertTrue(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_not_cmorized(self, mock_is_file): + mock_is_file.return_value = False + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon/ocean/var/' + 'r2i1p1/version')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_false_not_member_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 1 + os.makedirs(os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon/ocean/var/')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('os.path.isfile') + def test_is_cmorized_not_enough_vars(self, mock_is_file): + mock_is_file.return_value = True + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs( + os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon/ocean/var/' + 'r2i1p1/version')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_not_domain_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon/' + 'r2i1p1/version')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + def test_is_cmorized_not_freq_folder(self): + cmor_var = Mock() + omon = Mock() + omon.name = 'Omon' + cmor_var.get_table.return_value = omon + self.config.var_manager.get_variable.return_value = cmor_var + self.config.cmor.min_cmorized_vars = 2 + os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S20000101')) + self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) + + @mock.patch('earthdiagnostics.data_convention.SPECSConvention.create_link') + def test_create_links(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name/S20010101/mon/ocean/var/r2i1p1') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_called() + + @mock.patch('earthdiagnostics.data_convention.SPECSConvention.create_link') + def test_create_links_member_not_found(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name/S20010101/mon/ocean/var/r1i1p1') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_not_called() + + @mock.patch('earthdiagnostics.data_convention.SPECSConvention.create_link') + def test_create_links_with_grid(self, mock_create_link): + member_path = os.path.join(self.tmp_dir, + 'expid/cmorfiles/institute/model/experiment_name/S20010101/mon/ocean/var/' + 'r2i1p1/grid') + os.makedirs(member_path) + tempfile.mkstemp(dir=member_path) + self.convention.create_links('20010101', 1) + mock_create_link.assert_called() diff --git a/test/unit/ocean/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py index 5ad93a2688c343c96973c5ab3fdb58091ffe0719..b1f6526ddc5511b183d7bf28ba6aa9c0973c6d9c 100644 --- a/test/unit/ocean/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -17,7 +17,9 @@ class TestSiasiesiv(TestCase): self.mask = Mock() self.var_manager = Mock() - self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, {Basins().Global: []}, self.var_manager, False) + self.convention = Mock() + self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, {Basins().Global: []}, self.var_manager, + self.convention, False) def test_str(self): self.assertEqual(str(self.psi), diff --git a/test/unit/test_cmormanager.py b/test/unit/test_cmormanager.py index 8e634029b42af500c671dd7a45fa950a207c7067..e4c4463c498ed3aad3e4fba48ca2a2f8cbb103a7 100644 --- a/test/unit/test_cmormanager.py +++ b/test/unit/test_cmormanager.py @@ -5,17 +5,32 @@ import tempfile from unittest import TestCase import mock -from mock import Mock +from mock import Mock, create_autospec from earthdiagnostics.cmormanager import CMORManager, MergeYear from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.data_convention import DataConvention class TestCMORManager(TestCase): def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + os.mkdir(os.path.join(self.tmp_dir, 'expid')) + + self.convention = create_autospec(DataConvention) + self.convention.name = 'specs' + self.convention.get_startdate_path.return_value = os.path.join(self.tmp_dir, 'expid', 'startdate') + self.convention.get_cmor_folder_path.return_value = os.path.join(self.tmp_dir, 'expid', 'startdate', 'cmor') + self.convention.get_file_name.return_value = 'filename' + self.convention.get_file_path.return_value = os.path.join(self.tmp_dir, 'expid', 'startdate', 'cmor', + 'filename') + self.convention.is_cmorized.return_value = True + self.config = Mock() - self.config.data_convention = 'specs' + self.config.data_dir = self.tmp_dir + self.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') + self.config.data_convention = self.convention self.config.frequency = '6hr' self.config.data_type = 'exp' self.config.experiment.expid = 'expid' @@ -42,11 +57,6 @@ class TestCMORManager(TestCase): self.config.cmor.skip_prepare = False self.config.cmor.append_startdate = False - self.tmp_dir = tempfile.mkdtemp() - os.mkdir(os.path.join(self.tmp_dir, self.config.experiment.expid)) - self.config.data_dir = self.tmp_dir - self.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch') - def tearDown(self): shutil.rmtree(self.tmp_dir) @@ -78,266 +88,6 @@ class TestCMORManager(TestCase): cmor_manager = CMORManager(self.config) self.assertEqual(cmor_manager.cmor_path, os.path.join(self.tmp_dir, 'exp', 'model', 'expid')) - def test_get_file_path_bad_convention(self): - self.config.cmor.version = 'version' - self.config.data_convention = 'bad_convention' - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - with self.assertRaises(ValueError): - cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, 'mon') - - def test_get_file_path_specs(self): - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' - 'ocean/var/r2i1p1/' - 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) - - def test_get_file_path_specs_non_cmor(self): - cmor_manager = CMORManager(self.config) - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' - 'ocean/var/r2i1p1/' - 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) - - def test_get_file_path_specs_empty_time_info(self): - cmor_manager = CMORManager(self.config) - with self.assertRaises(ValueError): - cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, None, - 'mon') - - def test_get_file_path_preface(self): - self._configure_preface() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/expname/S19900101/mon/' - 'ocean/var/r2i1p1/' - 'var_Omon_model_expname_S19900101_r2i1p1_198901_198912.nc')) - - def test_get_file_path_specs_version(self): - self.config.cmor.version = 'version' - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' - 'ocean/var/r2i1p1/version/' - 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) - - def test_get_file_path_specs_grid(self): - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - 'mon', 'grid') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' - 'ocean/var/grid/r2i1p1/' - 'var_Omon_model_expname_S19900101_r2i1p1_198901-198912.nc')) - - def test_get_file_path_specs_year(self): - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - 'year', year='1998') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/year/' - 'ocean/var/r2i1p1/' - 'var_Omon_model_expname_S19900101_r2i1p1_1998.nc')) - - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', - cmor_var, 1, 'mon', year='1998') - - def test_get_file_path_raise_incompatible_date_info(self): - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.frequency = 'monthly' - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' - - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, - None, frequency, year='1998') - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, - 1, frequency, year='1998') - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, - 1, frequency, date_str='1998') - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, - None, frequency, year='1998', date_str='1998') - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, - 1, frequency, year='1998', date_str='1998') - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, - None, frequency) - - def test_get_file_path_specs_date_str(self): - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - 'mon', date_str='date_str') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S19900101/mon/' - 'ocean/var/r2i1p1/' - 'var_Omon_model_expname_S19900101_r2i1p1_date_str.nc')) - - def test_get_file_path_primavera(self): - self._configure_primavera() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' - 'r2i1p1f1/Omon/var/ocean_grid/version/' - 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) - - def test_get_file_path_primavera_with_startdate(self): - self._configure_primavera() - self.config.cmor.append_startdate = True - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expnameS19900101/' - 'r2i1p1f1/Omon/var/ocean_grid/version/' - 'var_Omon_model_expnameS19900101_r2i1p1f1_ocean_grid_' - '198901-198912.nc')) - - def test_get_file_path_primavera_no_cmor(self): - self._configure_primavera() - cmor_manager = CMORManager(self.config) - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'mon' - frequency.frequency = 'mon' - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', None, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' - 'r2i1p1f1/Omon/var/ocean_grid/version/' - 'var_Omon_model_expname_r2i1p1f1_ocean_grid_198901-198912.nc')) - - def test_get_file_path_no_version_primavera(self): - self._configure_primavera() - self.config.cmor.version = '' - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = 'frequency' - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', 1, ModelingRealms.ocean, 'var', cmor_var, - 0, frequency) - - def _configure_primavera(self): - self.config.data_convention = 'primavera' - self.config.cmor.version = 'version' - - def _configure_meteofrance(self): - self.config.data_convention = 'meteofrance' - - def _configure_preface(self): - self.config.data_convention = 'preface' - - def test_get_file_path_primavera_grid(self): - self._configure_primavera() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, 0, - 'mon', 'grid') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' - 'Omon/var/grid/version/' - 'var_Omon_model_expname_r2i1p1f1_grid_198901-198912.nc')) - - def test_get_file_path_primavera_atmos(self): - self._configure_primavera() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.atmos, 'var', cmor_var, 0, - 'mon') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/' - 'r2i1p1f1/Omon/var/atmos_grid/version/' - 'var_Omon_model_expname_r2i1p1f1_atmos_grid_198901-198912.nc')) - - def test_get_file_path_primavera_year(self): - self._configure_primavera() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - 'year', year='1998') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/' - 'var/ocean_grid/version/' - 'var_Omon_model_expname_r2i1p1f1_ocean_grid_1998.nc')) - - self.assertRaises(ValueError, cmor_manager.get_file_path, '19900101', None, ModelingRealms.ocean, 'var', - cmor_var, 1, 'mon', year='1998') - - def test_get_file_path_primavera_date_str(self): - self._configure_primavera() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('19900101', 1, ModelingRealms.ocean, 'var', cmor_var, None, - 'mon', date_str='date_str') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/' - 'Omon/var/ocean_grid/version/' - 'var_Omon_model_expname_r2i1p1f1_ocean_grid_date_str.nc')) - def test_file_exists(self): with mock.patch('os.path.isfile') as isfile: cmor_manager = CMORManager(self.config) @@ -356,159 +106,6 @@ class TestCMORManager(TestCase): self.assertFalse(cmor_manager.file_exists(ModelingRealms.ocean, 'var', '20011101', 1, 1, possible_versions=('version1', 'version2'))) - def test_get_file_path_meteofrance(self): - self._configure_meteofrance() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, - 'day') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201101_01.nc')) - - file_path = cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 2, - 'day') - self.assertEqual(file_path, - os.path.join(self.tmp_dir, 'expname/HA/2011/soicecov_day_201201_01.nc')) - - def test_get_file_path_bad_convention(self): - self.config.data_convention = 'badconvention' - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - with self.assertRaises(ValueError): - cmor_manager.get_file_path('20110101', 1, ModelingRealms.ocean, 'soicecov', cmor_var, 1, - 'day') - - def test_create_link(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) - filename = os.path.basename(path) - self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) - - def test_create_link_no_source(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - os.remove(path) - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - with self.assertRaises(ValueError): - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) - - def test_create_link_exists(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - filename = os.path.basename(path) - - os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) - os.symlink(path, os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename)) - - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) - self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) - - def test_create_link_default_is_link(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h')) - os.symlink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h'), - os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, False, None) - filename = os.path.basename(path) - self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) - - def test_create_link_move_old(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) - fd = open(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', 'var_20001101.nc'), 'w') - fd.close() - - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', None, True, None) - filename = os.path.basename(path) - self.assertTrue(os.path.islink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h', filename))) - self.assertTrue(os.path.isfile(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'old_var_f6h', - 'var_20001101.nc'))) - - def test_create_link_with_grid(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) - filename = os.path.basename(path) - var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') - var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') - self.assertTrue(os.path.islink(var_mainfolder)) - self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) - self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) - - def test_create_link_with_grid_default_is_link(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h')) - os.symlink(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-original_f6h'), - os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) - - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) - filename = os.path.basename(path) - var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') - var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') - self.assertTrue(os.path.islink(var_mainfolder)) - self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) - self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) - - def test_create_link_with_grid_default_is_folder(self): - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - os.makedirs(os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h')) - - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) - filename = os.path.basename(path) - var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') - var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') - self.assertTrue(os.path.islink(var_mainfolder)) - self.assertTrue(os.path.islink(os.path.join(var_grid_folder, filename))) - self.assertTrue(os.readlink(var_mainfolder), var_grid_folder) - - def test_create_link_meteofrance(self): - self._configure_meteofrance() - cmor_manager = CMORManager(self.config) - file_descriptor, path = tempfile.mkstemp(dir=self.tmp_dir) - os.close(file_descriptor) - frequency = Mock() - frequency.folder_name.return_value = 'frequency_folder' - cmor_manager.create_link(ModelingRealms.ocean, path, frequency, 'var', 'grid', False, None) - filename = os.path.basename(path) - var_mainfolder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var_f6h') - var_grid_folder = os.path.join(self.tmp_dir, 'expid', 'frequency_folder', 'var-grid_f6h') - self.assertFalse(os.path.islink(var_mainfolder)) - self.assertFalse(os.path.islink(os.path.join(var_grid_folder, filename))) - @mock.patch('earthdiagnostics.cmormanager.Cmorizer', autospec=True) def test_prepare_cmorize(self, mock_cmor): mock_instance = mock_cmor.return_value @@ -529,10 +126,9 @@ class TestCMORManager(TestCase): mock_instance.cmorize_atmos.assert_called_once() @mock.patch('earthdiagnostics.cmormanager.CMORManager.is_cmorized') - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_links') @mock.patch('earthdiagnostics.utils.Utils.unzip') @mock.patch('earthdiagnostics.utils.Utils.untar') - def test_prepare_cmorize_force_untar(self, mock_untar, mock_unzip, mock_create_links, mock_cmorized): + def test_prepare_cmorize_force_untar(self, mock_untar, mock_unzip, mock_cmorized): original_cmor_path = os.path.join(self.config.data_dir, self.config.experiment.expid, 'original_files', 'cmorfiles') os.makedirs(original_cmor_path) @@ -546,7 +142,7 @@ class TestCMORManager(TestCase): cmor_manager = CMORManager(self.config) self.config.experiment.get_member_list.return_value = (('20000101', 2),) cmor_manager.prepare() - mock_create_links.assert_called_once() + self.convention.create_links.assert_called_once() mock_unzip.assert_called_once() mock_untar.assert_called_once() @@ -567,261 +163,58 @@ class TestCMORManager(TestCase): cmor_manager.prepare() def test_prepare_meteofrance(self): - self._configure_meteofrance() + self.convention.name = 'meteofrance' cmor_manager = CMORManager(self.config) cmor_manager.prepare() - def test_is_cmorized_false(self): - cmor_manager = CMORManager(self.config) - self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - - @mock.patch('os.path.isfile') - def test_is_cmorized_true(self, mock_is_file): - mock_is_file.return_value = True - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 1 - cmor_manager = CMORManager(self.config) - os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S20000101/mon/ocean/var')) - self.assertTrue(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - - @mock.patch('os.path.isfile') - def test_is_cmorized_not_enough_vars(self, mock_is_file): - mock_is_file.return_value = True - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 2 - cmor_manager = CMORManager(self.config) - os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S20000101/mon/ocean/var')) - self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - - def test_is_cmorized_not_domain_folder(self): - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 2 - cmor_manager = CMORManager(self.config) - os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/expname/S20000101/mon')) - self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - - @mock.patch('os.path.isfile') - def test_is_cmorized_true_primavera(self, mock_is_file): - self._configure_primavera() - mock_is_file.return_value = True - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 1 - cmor_manager = CMORManager(self.config) - os.makedirs(os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var')) - self.assertTrue(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - - @mock.patch('os.path.isfile') - def test_is_cmorized_false_primavera(self, mock_is_file): - self._configure_primavera() - mock_is_file.return_value = False - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 1 - cmor_manager = CMORManager(self.config) - os.makedirs(os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var')) - self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - - def test_is_cmorized_false_not_member_folder_primavera(self): - self._configure_primavera() - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 1 + def test_is_not_cmorized(self): + self.convention.is_cmorized.return_value = False cmor_manager = CMORManager(self.config) - os.makedirs(os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/')) self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - def test_is_cmorized_false_not_table_folder_primavera(self): - self._configure_primavera() - cmor_var = Mock() - omon = Mock() - omon.name = 'Omon' - cmor_var.get_table.return_value = omon - self.config.var_manager.get_variable.return_value = cmor_var - self.config.cmor.min_cmorized_vars = 1 - cmor_manager = CMORManager(self.config) - os.makedirs(os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1')) - self.assertFalse(cmor_manager.is_cmorized('20000101', 1, 1, ModelingRealms.ocean)) - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') - def test_link_file(self, mock_get_file_path, mock_create_link): - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - cmor_manager.link_file(ModelingRealms.ocean, 'var', cmor_var, '20010101', 1, 1) - mock_get_file_path.assert_called_once() - mock_create_link.assert_called_once() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') - def test_link_file(self, mock_get_file_path, mock_create_link): - frequency = Mock() - cmor_manager = CMORManager(self.config) - cmor_var = Mock() - cmor_manager.link_file(ModelingRealms.ocean, 'var', cmor_var, '20010101', 1, 1, frequency=frequency) - mock_get_file_path.assert_called_once() - mock_create_link.assert_called_once() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') - def test_link_file_default_freq(self, mock_get_file_path, mock_create_link): + def test_link_file(self): cmor_manager = CMORManager(self.config) cmor_var = Mock() cmor_manager.link_file(ModelingRealms.ocean, 'var', cmor_var, '20010101', 1, 1) - mock_get_file_path.assert_called_once() - mock_create_link.assert_called_once() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - def test_create_links_specs(self, mock_create_link): - member_path = os.path.join(self.tmp_dir, - 'expid/cmorfiles/institute/model/expname/S20010101/mon/ocean/var/r2i1p1') - os.makedirs(member_path) - tempfile.mkstemp(dir=member_path) - cmor_manager = CMORManager(self.config) - cmor_manager.create_links('20010101', 1) - mock_create_link.assert_called() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - def test_create_links_specs_member_not_found(self, mock_create_link): - member_path = os.path.join(self.tmp_dir, - 'expid/cmorfiles/institute/model/expname/S20010101/mon/ocean/var/r1i1p1') - os.makedirs(member_path) - tempfile.mkstemp(dir=member_path) - cmor_manager = CMORManager(self.config) - cmor_manager.create_links('20010101', 1) - mock_create_link.assert_not_called() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - def test_create_links_specs_with_grid(self, mock_create_link): - member_path = os.path.join(self.tmp_dir, - 'expid/cmorfiles/institute/model/expname/S20010101/mon/ocean/var/r2i1p1/grid') - os.makedirs(member_path) - tempfile.mkstemp(dir=member_path) - cmor_manager = CMORManager(self.config) - cmor_manager.create_links('20010101', 1) - mock_create_link.assert_called() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - def test_create_links_primavera(self, mock_create_link): - self._configure_primavera() - member_path = os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var/gn') - os.makedirs(member_path) - self.config.var_manager.tables = {'Omon': Mock()} - tempfile.mkstemp(dir=member_path) - cmor_manager = CMORManager(self.config) - cmor_manager.create_links('20010101', 1) - mock_create_link.assert_called() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - def test_create_links_with_version_primavera(self, mock_create_link): - self._configure_primavera() - member_path = os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var/gn/version') - os.makedirs(member_path) - self.config.var_manager.tables = {'Omon': Mock()} - tempfile.mkstemp(dir=member_path) - cmor_manager = CMORManager(self.config) - cmor_manager.create_links('20010101', 1) - mock_create_link.assert_called() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - def test_create_links_with_version_primavera_no_member(self, mock_create_link): - self._configure_primavera() - member_path = os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/r2i1p1f1/Omon/var/gn/version') - os.makedirs(member_path) - self.config.var_manager.tables = {'Omon': Mock()} - tempfile.mkstemp(dir=member_path) - cmor_manager = CMORManager(self.config) - cmor_manager.create_links('20010101') - mock_create_link.assert_called() - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.create_link') - def test_create_links_member_not_found_primavera(self, mock_create_link): - self._configure_primavera() - member_path = os.path.join(self.tmp_dir, - 'expid/cmorfiles/activity/institute/model/expname/r1i1p1f1/Omon/var/gn') - os.makedirs(member_path) - self.config.var_manager.tables = {'Omon': Mock()} - tempfile.mkstemp(dir=member_path) - cmor_manager = CMORManager(self.config) - cmor_manager.create_links('20010101', 1) - mock_create_link.assert_not_called() + self.convention.create_link.assert_called_once() - def test_create_links_meteofrance(self): - self._configure_meteofrance() - cmor_manager = CMORManager(self.config) - with self.assertRaises(ValueError): - cmor_manager.create_links('20010101', 1) - - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') - def test_request_chunk(self, mock_get_file_path): - mock_get_file_path.return_value = '/path/to/file' + def test_request_chunk(self): cmor_manager = CMORManager(self.config) datafile = cmor_manager.request_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) - self.assertEqual(datafile.remote_file, '/path/to/file') + self.assertEqual(datafile.remote_file, self._mock_filepath()) + + def _mock_filepath(self): + return os.path.join(self.tmp_dir, 'expid/startdate/cmor/filename') - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') - def test_declare_chunk(self, mock_get_file_path): - mock_get_file_path.return_value = '/path/to/file' + def test_declare_chunk(self): cmor_manager = CMORManager(self.config) datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) - self.assertEqual(datafile.remote_file, '/path/to/file') + self.assertEqual(datafile.remote_file, self._mock_filepath()) mock_frequency = Mock() datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1, frequency=mock_frequency) - self.assertEqual(datafile.remote_file, '/path/to/file') + self.assertEqual(datafile.remote_file, self._mock_filepath()) self.config.var_manager.get_variable.return_value = None datafile = cmor_manager.declare_chunk(ModelingRealms.ocean, 'var', '20010101', 1, 1) - self.assertEqual(datafile.remote_file, '/path/to/file') + self.assertEqual(datafile.remote_file, self._mock_filepath()) - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') - def test_request_year(self, mock_get_file_path): + def test_request_year(self): self.config.experiment.get_year_chunks.return_value = (1, 2) - mock_get_file_path.return_value = '/path/to/file' mock_diagnostic = Mock() cmor_manager = CMORManager(self.config) datafile = cmor_manager.request_year(mock_diagnostic, ModelingRealms.ocean, 'var', '20010101', 1, 2000) - self.assertEqual(datafile.remote_file, '/path/to/file') + self.assertEqual(datafile.remote_file, self._mock_filepath()) - @mock.patch('earthdiagnostics.cmormanager.CMORManager.get_file_path') - def test_declare_year(self, mock_get_file_path): - mock_get_file_path.return_value = '/path/to/file' + def test_declare_year(self): cmor_manager = CMORManager(self.config) datafile = cmor_manager.declare_year(ModelingRealms.ocean, 'var', '20010101', 1, 2001) - self.assertEqual(datafile.remote_file, '/path/to/file') + self.assertEqual(datafile.remote_file, self._mock_filepath()) self.config.var_manager.get_variable.return_value = None datafile = cmor_manager.declare_year(ModelingRealms.ocean, 'var', '20010101', 1, 2001) - self.assertEqual(datafile.remote_file, '/path/to/file') + self.assertEqual(datafile.remote_file, self._mock_filepath()) class TestMergeYear(TestCase): diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 8f7a610dce1f6c04915ca4620177528e51f7394e..b588881a1ba89e0e7f518523f157a74f2140e716 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -7,6 +7,7 @@ import os from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig, Config from earthdiagnostics.frequency import Frequencies from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.data_convention import SPECSConvention, PrimaveraConvention class VariableMock(object): @@ -525,7 +526,7 @@ class TestConfig(TestCase): config = Config() self.mock_parser.add_value('DIAGNOSTICS', 'DATA_CONVENTION', 'primavera') self._parse(config) - self.assertEqual(config.data_convention, 'primavera') + self.assertIsInstance(config.data_convention, PrimaveraConvention) self.assertEqual(config.scratch_masks, '/scratch/Earth/ocean_masks/primavera') namelist = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'earthdiagnostics/CDFTOOLS_primavera.namlist')) diff --git a/test/unit/test_data_manager.py b/test/unit/test_data_manager.py index cb4c7af87f9f957182847c9bba20e06bf34693ef..3d5a3d60ce7054b09e327ac6af064d8e2122fcb5 100644 --- a/test/unit/test_data_manager.py +++ b/test/unit/test_data_manager.py @@ -20,9 +20,6 @@ class TestDataManager(TestCase): def test_prepare(self): self.data_manager.prepare() - def test_create_link(self): - self.data_manager.create_link(None, '', None, '', '', False, Mock()) - def test_link_file(self): self.data_manager.link_file(None, '', None, '', 0, 0) @@ -46,18 +43,6 @@ class TestDataManager(TestCase): with self.assertRaises(NotImplementedError): self.data_manager.declare_year(None, '', '', 0, 0) - def test_get_varfolder(self): - self.assertEqual(self.data_manager.get_varfolder(ModelingRealms.ocean, 'var'), - 'var_f6h') - self.assertEqual(self.data_manager.get_varfolder(ModelingRealms.atmos, 'var'), - 'var_f3h') - frequency = Mock() - frequency.__str__ = Mock() - frequency.__str__.return_value = '3hr' - frequency.frequency = '3hr' - self.assertEqual(self.data_manager.get_varfolder(ModelingRealms.atmos, 'var', frequency=frequency), - 'var') - def test_get_final_varname(self): self.assertEqual(self.data_manager._get_final_var_name(None, 'var'), 'var') box = Mock() diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py index c1f3a0ccf120168fa1ac1187a45c3d13c8f63b38..4c7bfe89c3236142d442f1db75b2f058e4f699df 100644 --- a/test/unit/test_earthdiags.py +++ b/test/unit/test_earthdiags.py @@ -42,7 +42,9 @@ class TestEarthDiags(TestCase): self.earthdiags.config = Mock() self.earthdiags.config.scratch_dir = 'scratch_path' self.earthdiags.config.cdftools_path = 'cdftools_path' - self.earthdiags.config.data_convention = 'SPECS' + self.earthdiags.config.data_convention = Mock() + self.earthdiags.config.data_convention.lat_name = 'lat' + self.earthdiags.config.data_convention.lon_name = 'lon' self.earthdiags.read_config('config/path') @@ -64,7 +66,9 @@ class TestEarthDiags(TestCase): self.earthdiags.config = Mock() self.earthdiags.config.scratch_dir = 'scratch_path' self.earthdiags.config.cdftools_path = 'cdftools_path' - self.earthdiags.config.data_convention = 'PRIMAVERA' + self.earthdiags.config.data_convention = Mock() + self.earthdiags.config.data_convention.lat_name = 'latitude' + self.earthdiags.config.data_convention.lon_name = 'longitude' self.earthdiags.read_config('config/path') diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 103bcdc387f20a47a40f1870d24fe9f17568dd39..82baa52929fb213b6a0c530d9c31aa9414d30af7 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -1,5 +1,6 @@ # coding=utf-8 from unittest import TestCase +from mock import Mock from earthdiagnostics.frequency import Frequencies from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms @@ -49,3 +50,13 @@ class TestModellingRealm(TestCase): def test_get_6hrplev(self): self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev') + + def test_get_varfolder(self): + self.assertEqual(ModelingRealm('ocean').get_varfolder('var', 6, 3), 'var_f6h') + self.assertEqual(ModelingRealm('atmos').get_varfolder('var', 6, 3), 'var_f3h') + frequency = Mock() + frequency.__str__ = Mock() + frequency.__str__.return_value = '3hr' + frequency.frequency = '3hr' + self.assertEqual(ModelingRealm('atmos').get_varfolder('var', 6, 3, frequency=frequency), + 'var') diff --git a/test/unit/test_workmanager.py b/test/unit/test_workmanager.py index 53417fd7df6bf366500d140c1fade8d398ef0fe6..119d3c6fa6908eadda707f7085a30431785b78eb 100644 --- a/test/unit/test_workmanager.py +++ b/test/unit/test_workmanager.py @@ -189,7 +189,7 @@ class TestDownloader(TestCase): class MockFile(DataFile): def __init__(self): - super().__init__() + super(MockFile, self).__init__() self.upload_fails = False def download(self): @@ -204,6 +204,9 @@ class MockFile(DataFile): def clean_local(self): pass + def check_is_in_storage(self): + return False + class TestWorkManager(TestCase):