diff --git a/VERSION b/VERSION index 88a9ea52787d83e15abc044ce88e3e1a71de7ea7..50a63251d3bf9e6bf83e6e37865fbcd0737b054a 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0b32 +3.0.0b33 diff --git a/diags.conf b/diags.conf index 5ee2717319f8ddafc21e2f8a73e7123c3dac2999..0b3c4f4f0aa47705a3948471c42343d93f1dbe62 100644 --- a/diags.conf +++ b/diags.conf @@ -1,22 +1,125 @@ [DIAGNOSTICS] +# Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) +DATA_ADAPTOR = CMOR +# Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER +# Root path for the cmorized data to use DATA_DIR = /esnas:/esarchive -CON_FILES = /esnas/autosubmit/con_files/ -DIAGS = moc ohc,glob,0,0,0 -FREQUENCY = mon -MAX_CORES = 4 +# Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) +DATA_TYPE = exp +# CMORization type to use. Important also for THREDDS as it affects variable name conventions. +# Options: SPECS (default), PRIMAVERA, CMIP6 DATA_CONVENTION = CMIP6 +# Path to NEMO's mask and grid files needed for CDFTools +CON_FILES = /esnas/autosubmit/con_files/ +# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or +# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it +# empty +DIAGS = +# DIAGS = OHC +# Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores +# its results at monthly frequency (obvious) and has a parameter to specify input's frequency. +FREQUENCY = 6hr +# Path to CDFTOOLS binaries +CDFTOOLS_PATH = +# If true, copies the mesh files regardless of presence in scratch dir +RESTORE_MESHES = False +# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available) +MAX_CORES = 1 + +[CMOR] +# If true, recreates CMOR files regardless of presence. Default = False +FORCE = True +# If true, CMORizes ocean files. Default = True +OCEAN_FILES = True +FILTER_FILES = _PISC _dia +# If true, CMORizes atmosphere files. Default = True +ATMOSPHERE_FILES = False +# You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var +VARIABLE_LIST = + +# Variables to be CMORized from the grib atmospheric files, separated by comma. +# You can also specify the levels to extract using the following syntax +# VARIABLE_CODE, VARIABLE_CODE:LEVEL, VARIABLE_CODE:LEVEL1-LEVEL2, VARIABLE_CODE:MIN_LEVEL:MAX_LEVEL:STEP +# Examples: +# Variable with code 129 at level 30000: 129:30000 +# Variable with code 129 at levels 30000, 40000 and 60000: 129:30000-40000-60000 +# Variable with code 129 at levels between 30000 and 600000 with 10000 intervals: +# 129:30000:60000:10000 equivalent to 129:30000-40000-50000-60000 + +# Hourly vars +ATMOS_HOURLY_VARS = 129:30000:90000:5000, 130, 131:30000:90000:5000, 132:30000:90000:5000, 151, 167, 168, 164, 165, 166 +# Daily vars +ATMOS_DAILY_VARS = 167, 165, 166, 151, 164, 168, 169, 177, 179, 228, 201, 202, 130:85000 +# Monthly vars +ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, 147, 176, 169, 177, 175, 212, 141, 180, 181, 179, 168, 243, 129:5000-20000-50000-85000, 130:5000-20000-50000-85000, 131:5000-20000-50000-85000, 132:5000-20000-50000-85000, 133:5000-20000-50000-85000 + +# The next bunch of parameters are used to provide metadata for the CMOR files +# ASSOCIATED_EXPERIMENT = +# INITIALIZATION_METHOD = 1 +# INITIALIZATION_DESCRIPTION = ocean: ECMWF system4, ice: DFS4.3 , atmosphere: +# PHYSICS_VERSION = 1 +# PHYSICS_DESCRIPTION = +# ASSOCIATED_MODEL = +# SOURCE = 'EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 + +[THREDDS] +SERVER_URL = https://earth.bsc.es/thredds + [EXPERIMENT] +# Experiments parameters as defined in CMOR standard INSTITUTE = BSC -MODEL = EC-EARTH3 +MODEL = EC-EARTH +# Model version: Available versions MODEL_VERSION =Ec3.2_O1L75 -EXPID = t014 +# Atmospheric output timestep in hours +ATMOS_TIMESTEP = 6 +# Ocean output timestep in hours +OCEAN_TIMESTEP = 6 + +# For those who use Autosubmit, this will be easy +# EXPID is the unique identifier of the experiment. +# STARTDATES is the list of start dates +# MEMBERS is the list of members of your experiment (only the numbers, the fc will be added by the tool) +# MEMBER_DIGITS is the minimum number of digits to use for the member name: if 1 the name for member 0 will be fc0, +# if 2, fc00 +# CHUNK_SIZE is the size of each data file, given in months +# CHUNKS is the number of chunks. You can specify less chunks than present on the experiment +EXPID = a0c2 STARTDATES = 19900101 MEMBERS = 0 -CHUNK_SIZE = 1 +MEMBER_DIGITS = 1 +CHUNK_SIZE = 12 CHUNKS = 2 +# CHUNKS = 1 + + +# This ALIAS section is a bit different +# Inside this, you can provide alias for frequent diagnostics calls. +# By default, there are some of the diagnostics available at the previous version. +# You can define an alias for one or moraa90a1ee diagnostic calls +[ALIAS] +MAX_MOC = mocmax,38,50,500,2000 mocmax,40,40,0,10000 +AREA_MOC = mocarea,40,55,1000,2000,atl mocarea,30,40,1000,2000,atl +STC = mocarea,0,25,0,200,Pac mocarea,-25,0,0,200,Pac mocarea,0,25,0,200,Atl mocarea,-25,0,0,200,Atl +HEAT_SAL_MXL = mlotstsc mlotsthc +LMSALC = vertmeanmeters,so,300,5400 +USALC = vertmeanmeters,so,0,300 +OHC = ohc,glob,0,1,10 +XOHC = ohc,glob,1,0,0 +LOHC = ohc,glob,0,23,46 +MOHC = ohc,glob,0,18,22 +UOHC = ohc,glob,0,1,17 +OHC_SPECIFIED_LAYER = ohclayer,0,300 ohclayer,300,800 +3DTEMP = interp,thetao +3DSAL = interp,so +TSEC_AVE190-220E =avgsection,thetao,190,220,-90,90 +SSEC_AVE190-220E =avgsection,so,190,220,-90,90 +VERT_SSECTIONS = cutsection,so,Z,0 cutsection,so,Z,45 cutsection,so,Z,-45 cutsection,so,M,-30 cutsection,so,M,180 cutsection,so,M,80 +VERT_TSECTIONS = cutsection,thetao,Z,0 cutsection,thetao,Z,45 cutsection,thetao,Z,-45 cutsection,thetao,M,-30 cutsection,thetao,M,180 cutsection,thetao,M,80 +SIASIESIV = siasiesiv,glob diff --git a/doc/source/conf.py b/doc/source/conf.py index 3af007a7073cb9c54a7155b4976a3dc948de768b..26fbbb19ba8da4560cb9e7c1f2f1af4c887988cf 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b32' +release = '3.0.0b33' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index 370a218159927804c6e9ee2d95c5a6ec24eb0693..eee3b4ab80665f05e2af6d8636d4271228dac4ca 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cmor_tables/default.csv b/earthdiagnostics/cmor_tables/default.csv index fc2742efab4bc0e096eccfd8683b659f2f838646..26c06435bce63a944bc4d508d529143314868c2a 100644 --- a/earthdiagnostics/cmor_tables/default.csv +++ b/earthdiagnostics/cmor_tables/default.csv @@ -295,5 +295,46 @@ rsdo,rsds,downwelling_shortwave_flux_in_sea_water,Downwelling Shortwave Radiatio wo,wo,sea_water_upward_velocity,Sea Water Upward Velocity ,ocean,,,,,, w2o,wosq,square_of_sea_water_upward_velocity,Square of Sea Water Upward Velocity ,ocean,,,,,, difvho,difvho,ocean_vertical_heat_diffusivity,Ocean Vertical Heat Diffusivity,ocean,,,,,, -vovematr,wmo,upward_ocean_mass_transport,Upward Ocean Mass Transport ,ocean,,,,,, +vovematr,wmo,upward_ocean_mass_transport,Upward Ocean Mass Transport,ocean,,,,,, qtr_ice,qtr,shortwave_flux_transmitted_through_ice,Shortwave Flux Transmitted Through The Ice,seaIce,,,,,, +poc,poc,small_organic_carbon_concentration,Small organic carbon Concentration,ocnBgchem,,,,,, +nanophy,nanophy,nanopthyoplankton_concentration,(Nano)Phytoplankton Concentration,ocnBgchem,,,,,, +dsi,dsi,diatoms_silicate_concentration,Diatoms Silicate Concentration,ocnBgchem,,,,,, +goc,goc,big_organic_carbon_concentration,Big organic carbon Concentration,ocnBgchem,,,,,, +sfe,sfe,small_iron_particles_concentration,Small iron particles Concentration,ocnBgchem,,,,,, +nfe,nfe,nano_iron_concentration,Nano iron Concentration,ocnBgchem,,,,,, +nchl,nchl,nano_chlorophyl_concentration,Nano chlorophyl Concentration,ocnBgchem,,,,,, +pno3tot,pno3tot,global_mean_nitrate_concentration,Global mean nitrate concentration,ocnBgchem,,,,,, +psiltot,psiltot,global_mean_silicate_concentration,Global mean silicate concentration,ocnBgchem,,,,,, +palktot,palktot,global_mean_alkalinity_concentration,Global mean alkalinity concentration,ocnBgchem,,,,,, +pfertot,pfertot,global_mean_iron_concentration,Global mean iron concentration,ocnBgchem,,,,,, +tcflx,tcflx,total_flux_carbon_out_of_the_ocean,total Flux of Carbon out of the ocean,ocnBgchem,,,,,, +tcflxcum,tcflxcum,cumulative_total_flux_of_carbon_out_of_the_ocean,cumulative total Flux of Carbon out of the ocean,ocnBgchem,,,,,, +c-export,c-export,total_carbon_export_at_100m,total Carbon export at 100m,ocnBgchem,,,,,, +tintpp,tintpp,global_total_integrated_primary_production,global total integrated primary production,ocnBgchem,,,,,, +tnfix,tnfix,global_total_nitrogen_fixation,global total nitrogen fixation,ocnBgchem,,,,,, +tdenit,tdenit,total_denitrification,Total denitrification,ocnBgchem,,,,,, +inttpp,inttpp,total_primary_production_of_phyto,Total Primary production of phyto,ocnBgchem,,,,,, +inttppnew,inttppnew,new_primary_production_of_phyto,New Primary production of phyto,ocnBgchem,,,,,, +intppphy,intppphy,vertically_integrated_primary_production_by_nanophy,Vertically integrated primary production by nanophy,ocnBgchem,,,,,, +ppphy,ppphy,primary_production_of_nanooplakton,Primary production of nanooplakton,ocnBgchem,,,,,, +intpbcal,intpbcal,vertically_integrated_of_calcite_productdic_fluxion,Vertically integrated of calcite productDIC fluxion,ocnBgchem,,,,,, +cflx,cflx,dic_flux,DIC flux,ocnBgchem,,,,,, +remin,remin,oxic_remineralization_of_om,Oxic remineralization of OM,ocnBgchem,,,,,, +denit,denit,anoxic_remineralization_of_om,Anoxic remineralization of OM,ocnBgchem,,,,,, +nfix,nfix,nitrogen_fixation,Nitrogen fixation,ocnBgchem,,,,,, +sdenit,sdenit,nitrate_reduction_in_the_sediments,Nitrate reduction in the sediments,ocnBgchem,,,,,, +par,par,photosynthetically_available_radiation,photosynthetically Available Radiation,ocnBgchem,,,,,, +lnnut,lnnut,nutrient_limitation_term_in_nanophyto,Nutrient limitation term in Nanophyto,ocnBgchem,,,,,, +ldnut,ldnut,nutrient_limitation_term_in_diatoms,Nutrient limitation term in Diatoms,ocnBgchem,,,,,, +lnfe,lnfe,iron_limitation_term_in_nanophyoto,Iron limitation term in Nanophyoto,ocnBgchem,,,,,, +lnlight,lnlight,light_limitation_term_in_nanophyto,Light limitation term in Nanophyto,ocnBgchem,,,,,, +ldlight,ldlight,light_limitation_term_in_diatoms,Light limitation term in Diatoms,ocnBgchem,,,,,, +graz1,graz1,grazing_by_microzooplankton,Grazing by microzooplankton,ocnBgchem,,,,,, +graz2,graz2,grazing_by_mesozooplankto_,Grazing by mesozooplankton,ocnBgchem,,,,,, +mumax,mumax,maximum_growth_rate,Maximum growth rate,ocnBgchem,,,,,, +mun,mun,realized_growth_rate_for_nanophyto,Realized growth rate for nanophyto,ocnBgchem,,,,,, +mud,mud,realized_growth_rate_for_diatomes,Realized growth rate for diatomes,ocnBgchem,,,,,, +ppnewn,ppnewn,new_primary_production_of_nanophyto,New Primary production of nanophyto,ocnBgchem,,,,,, +ppnewd,ppnewd,new_primary_production_of_diatoms,New Primary production of diatoms,ocnBgchem,,,,,, +dic,dic,disolved_inorganic_carbon,Disolved Inorganic Carbon,ocnBgchem,,,,,, diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 59db9d432c99ae8cb93f71c2849db83ed37949e8..dad631850eb71c1ce659cd828158377663375f4a 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -13,6 +13,7 @@ from autosubmit.date.chunk_date_lib import parse_date, chunk_end_date, previous_ from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import TempFile, Utils +from earthdiagnostics.variable import VariableManager class Cmorizer(object): @@ -70,6 +71,7 @@ class Cmorizer(object): tar_folder = os.path.join(self.original_files_path, '{0}*'.format(prefix)) tar_files = glob.glob(tar_folder) tar_files.sort() + count = 1 for tarfile in tar_files: if not self.cmorization_required(self.get_chunk(os.path.basename(tarfile)), ModelingRealms.ocean): @@ -86,9 +88,25 @@ class Cmorizer(object): Log.error('Could not CMORize oceanic file {0}: {1}', count, ex) count += 1 + def _filter_files(self, file_list): + if not self.cmor.filter_files: + return file_list + filtered = list() + filters = self.cmor.filter_files.split(' ') + for filename in file_list: + if any(f in filename for f in filters): + filtered.append(filename) + else: + os.remove(filename) + if len(filtered) == 0: + Log.warning('Filters {0} do not match any of the files', filters) + return filtered + def _cmorize_nc_files(self): - for filename in glob.glob(os.path.join(self.cmor_scratch, '*.nc')): + nc_files = glob.glob(os.path.join(self.cmor_scratch, '*.nc')) + for filename in nc_files: self._cmorize_nc_file(filename) + self._clean_cmor_scratch() def _correct_fluxes(self): fluxes_vars = ("prsn", "rss", "rls", "rsscs", "rsds", "rlds", "hfss", 'hfls') @@ -102,11 +120,19 @@ class Cmorizer(object): handler.close() def _unpack_tar_file(self, tarfile): - if os.path.exists(self.cmor_scratch): - shutil.rmtree(self.cmor_scratch) + self._clean_cmor_scratch() os.makedirs(self.cmor_scratch) Utils.untar((tarfile,), self.cmor_scratch) - Utils.unzip(glob.glob(os.path.join(self.cmor_scratch, '*.gz'))) + zip_files = glob.glob(os.path.join(self.cmor_scratch, '*.gz')) + for zip_file in self._filter_files(zip_files): + try: + Utils.unzip(zip_file) + except Utils.UnzipException as ex: + Log.error('File {0} could not be unzipped: {1}', tarfile, ex) + + def _clean_cmor_scratch(self): + if os.path.exists(self.cmor_scratch): + shutil.rmtree(self.cmor_scratch) def _merge_mma_files(self, tarfile): temp = TempFile.get() @@ -200,26 +226,9 @@ class Cmorizer(object): Utils.copy_file(original_gribfile, gribfile) self._obtain_atmos_timestep(gribfile) + full_file = self._get_monthly_grib(current_date, gribfile, grid) + self._unpack_grib(full_file, gribfile, grid) - prev_gribfile = self.get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) - if os.path.exists(prev_gribfile): - self._merge_grib_files(current_date, prev_gribfile, gribfile) - full_file = 'ICM' - else: - full_file = gribfile - - Log.info('Unpacking... ') - # remap on regular Gauss grid - if grid == 'SH': - Utils.cdo.splitparam(input='-sp2gpl {0}'.format(full_file), output=gribfile + '_', - options='-f nc4') - else: - Utils.cdo.splitparam(input=full_file, output=gribfile + '_', options='-R -f nc4') - # total precipitation (remove negative values) - Utils.cdo.setcode(228, input='-setmisstoc,0 -setvrange,0,Inf -add ' - '{0}_{{142,143}}.128.nc'.format(gribfile), - output='{0}_228.128.nc'.format(gribfile)) - Utils.remove_file('ICM') next_gribfile = self.get_original_grib_path(add_months(current_date, 1, self.experiment.calendar), grid) if not os.path.exists(next_gribfile): @@ -241,6 +250,29 @@ class Cmorizer(object): self._merge_and_cmorize_atmos(chunk_start, chunk_end, grid, '{0}hr'.format(self.atmos_timestep)) + def _unpack_grib(self, full_file, gribfile, grid): + Log.info('Unpacking... ') + # remap on regular Gauss grid + if grid == 'SH': + Utils.cdo.splitparam(input='-sp2gpl {0}'.format(full_file), output=gribfile + '_', + options='-f nc4') + else: + Utils.cdo.splitparam(input=full_file, output=gribfile + '_', options='-R -f nc4') + # total precipitation (remove negative values) + Utils.cdo.setcode(228, input='-setmisstoc,0 -setvrange,0,Inf -add ' + '{0}_{{142,143}}.128.nc'.format(gribfile), + output='{0}_228.128.nc'.format(gribfile)) + Utils.remove_file('ICM') + + def _get_monthly_grib(self, current_date, gribfile, grid): + prev_gribfile = self.get_scratch_grib_path(add_months(current_date, -1, self.experiment.calendar), grid) + if os.path.exists(prev_gribfile): + self._merge_grib_files(current_date, prev_gribfile, gribfile) + full_file = 'ICM' + else: + full_file = gribfile + return full_file + def get_scratch_grib_path(self, current_date, grid): return os.path.join(self.config.scratch_dir, self._get_grib_filename(grid, current_date)) @@ -334,13 +366,13 @@ class Cmorizer(object): :param variable: variable's name :type variable: str """ - temp = TempFile.get() - alias, var_cmor = self.data_manager.variable_list.get_variable_and_alias(variable) + alias, var_cmor = VariableManager().get_variable_and_alias(variable) if var_cmor is None: return if not self.cmor.cmorize(var_cmor): return - frequency = Frequency.parse(frequency) + + temp = TempFile.get() Utils.nco.ncks(input=file_path, output=temp, options='-v {0}'.format(variable)) self._rename_level_variables(temp, var_cmor) @@ -360,7 +392,7 @@ class Cmorizer(object): self.data_manager.send_file(temp, var_cmor.domain, var_cmor.short_name, self.startdate, self.member, frequency=frequency, rename_var=variable, date_str=date_str, region=region, - move_old=True, grid=alias.grid, cmorized=True) + move_old=True, grid=var_cmor.grid, cmorized=True) def get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') @@ -409,12 +441,6 @@ class Cmorizer(object): if var_cmor.domain == ModelingRealms.atmos: Utils.rename_variables(temp, {'depth': 'plev'}, False, True) - @staticmethod - def translate_frequency(frequency): - if frequency == 'h': - frequency = '6hr' - return Frequency(frequency) - @staticmethod def _merge_grib_files(current_month, prev_gribfile, gribfile): Log.info('Merging data from different files...') @@ -583,7 +609,11 @@ class Cmorizer(object): return len(gribfiles) > 0 def cmorization_required(self, chunk, domain): - return self.config.cmor.force or not self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain) + if not self.config.cmor.chunk_cmorization_requested(chunk): + return False + if self.config.cmor.force: + return True + return not self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain) class CMORException(Exception): diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index e894b7537b4e88142a96f4d95b317b61890acf89..2c6bc9d91f55a3c18fead6296dc8a912a3b49195 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -32,10 +32,14 @@ class CMORManager(DataManager): if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): self.config.data_dir = data_folder break + test_folder = os.path.join(data_folder, self.experiment.model.lower().replace('-', '')) + if os.path.isdir(os.path.join(test_folder, self.experiment.expid)): + self.config.data_dir = test_folder + break - data_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) - if os.path.isdir(os.path.join(data_folder, self.experiment.expid)): - self.config.data_dir = data_folder + test_folder = os.path.join(data_folder, self.config.data_type, experiment_folder) + if os.path.isdir(os.path.join(test_folder, self.experiment.expid)): + self.config.data_dir = test_folder break if not self.config.data_dir: @@ -184,6 +188,7 @@ class CMORManager(DataManager): def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): + """ Creates the link of a given file from the CMOR repository. @@ -275,7 +280,8 @@ class CMORManager(DataManager): if not frequency: frequency = self.config.frequency - filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, chunk, frequency, grid, year, date_str) + filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, chunk, frequency, grid, year, + date_str) netcdf_file = NetCDFFile(filepath, filetosend, domain, final_name, cmor_var, self.config.data_convention, region) netcdf_file.frequency = frequency @@ -406,18 +412,22 @@ class CMORManager(DataManager): filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar.gz') if len(filepaths) > 0: - Log.info('Unzipping cmorized data for {0} {1} {2}...', startdate, member, chunk) - Utils.unzip(filepaths, True) + if self.config.cmor.chunk_cmorization_requested(chunk): + Log.info('Unzipping cmorized data for {0} {1} {2}...', startdate, member, chunk) + Utils.unzip(filepaths, True) + else: + return True if not os.path.exists(self.cmor_path): os.mkdir(self.cmor_path) filepaths = self._get_transferred_cmor_data_filepaths(startdate, member, chunk, 'tar') if len(filepaths) > 0: - Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) - Utils.untar(filepaths, self.cmor_path) - self._correct_paths(startdate) - self.create_links(startdate) + if self.config.cmor.chunk_cmorization_requested(chunk): + Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) + Utils.untar(filepaths, self.cmor_path) + self._correct_paths(startdate) + self.create_links(startdate) return True return False diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index afec437548becf98a931ebbd19c63d9d57f51405..db28a892ebceb7bd0f9c189d13a22bb8639dda73 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -92,9 +92,11 @@ class CMORConfig(object): def __init__(self, parser): self.force = parser.get_bool_option('CMOR', 'FORCE', False) self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) + self.filter_files = parser.get_option('CMOR', 'FILTER_FILES') self.ocean = parser.get_bool_option('CMOR', 'OCEAN_FILES', True) self.atmosphere = parser.get_bool_option('CMOR', 'ATMOSPHERE_FILES', True) self.use_grib = parser.get_bool_option('CMOR', 'USE_GRIB', True) + self._chunks = [int(chunk) for chunk in parser.get_list_option('CMOR', 'CHUNKS')] self.associated_experiment = parser.get_option('CMOR', 'ASSOCIATED_EXPERIMENT', 'to be filled') self.associated_model = parser.get_option('CMOR', 'ASSOCIATED_MODEL', 'to be filled') self.initialization_description = parser.get_option('CMOR', 'INITIALIZATION_DESCRIPTION', 'to be filled') @@ -138,6 +140,11 @@ class CMORConfig(object): return False + def chunk_cmorization_requested(self, chunk): + if len(self._chunks) == 0: + return True + return chunk in self._chunks + @staticmethod def _parse_variables(raw_string): variables = dict() diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 73994fba2092d2568c3b7b6a0aeb491b8c18299c..e9e775b6f98f031d80552c9f5cf715fff729002a 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -223,7 +223,7 @@ class DataManager(object): self.lock.release() # Overridable methods (not mandatory) - def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, box=None, + def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -244,8 +244,6 @@ class DataManager(object): :type chunk: int :param grid: file's grid (only needed if it is not the original) :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str :param vartype: Variable type (mean, statistic) diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 86787556c32cc91a34b34b2e39436305c737b7f9..83b1e44e001f8b538883b7d8ba90841ed944adb0 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -2,7 +2,7 @@ from earthdiagnostics.constants import Basins from earthdiagnostics.frequency import Frequency from earthdiagnostics.variable_type import VariableType -from earthdiagnostics.modelingrealm import ModelingRealm +from earthdiagnostics.modelingrealm import ModelingRealms class Diagnostic(object): @@ -174,16 +174,29 @@ class DiagnosticIntOption(DiagnosticOption): class DiagnosticListIntOption(DiagnosticOption): + + def __init__(self, name, default_value=None, min_limit=None, max_limit=None): + super(DiagnosticListIntOption, self).__init__(name, default_value) + self.min_limit = min_limit + self.max_limit = max_limit + def parse(self, option_value): option_value = self.check_default(option_value) if isinstance(option_value, tuple) or isinstance(option_value, list): return option_value - return [int(i) for i in option_value.split('-')] + values = [int(i) for i in option_value.split('-')] + for value in values: + if self.min_limit is not None and value < self.min_limit: + raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) + if self.max_limit is not None and value > self.max_limit: + raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) + + return values class DiagnosticDomainOption(DiagnosticOption): def parse(self, option_value): - return ModelingRealm.parse(self.check_default(option_value)) + return ModelingRealms.parse(self.check_default(option_value)) class DiagnosticFrequencyOption(DiagnosticOption): diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index 55adabe9f70ae4c3195b309d5ba57d4918dadb66..b0d6011e51733e16c7463292b733826db44ebe60 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -101,7 +101,7 @@ class EarthDiags(object): if Log.console_handler.level <= Log.DEBUG: Utils.cdo.debug = True - Utils.nco.debug = False # This is due to a bug in nco. Must change when it's solved + Utils.nco.debug = False # This is due to a bug in nco. Must change when it's solved if args.logfilepath: Log.set_file(Utils.expand_path(args.logfilepath)) diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py index 451017264dfe023a1e940d0e27ba0d1b8d2ca9d9..da40398f3ea8be065187533823eb1fc967735bd5 100644 --- a/earthdiagnostics/frequency.py +++ b/earthdiagnostics/frequency.py @@ -20,7 +20,7 @@ class Frequency(object): try: self.frequency = Frequency._recognized[freq] except KeyError: - raise Exception('Frequency {0} not supported'.format(freq)) + raise ValueError('Frequency {0} not supported'.format(freq)) def __eq__(self, other): return self.frequency == other.frequency diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index f65809a9831e399c2f793a271a0dc45b9bfe701a..6ee2d226a763d3fb63260130ddb2b3c6d72363af 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -1,6 +1,5 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticBoolOption -from earthdiagnostics.modelingrealm import ModelingRealm +from earthdiagnostics.diagnostic import Diagnostic class RelinkAll(Diagnostic): diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index ffe252ccc79ece6616c5e5af0c634dfde5f98a51..acc5646c9c4b8b999dd200725f83e1e4bbf7ac90 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -17,7 +17,7 @@ class ModelingRealm(object): elif domain_name in ['ocean', 'atmos', 'land', 'aerosol']: self.name = domain_name else: - raise ValueError('Domain {0} not recognized!'.format(domain_name)) + raise ValueError('Modelling realm {0} not recognized!'.format(domain_name)) def __eq__(self, other): return other.__class__ == ModelingRealm and self.name == other.name @@ -90,5 +90,5 @@ class ModelingRealms(object): if isinstance(value, ModelingRealm): if modelling_realm.lower() in [value.name.lower()]: return value - return None + raise ValueError('Modelling realm {0} not recognized!'.format(modelling_realm)) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index ee070c19e9bb6b06af376a650700649aa79a7cf8..edf5711fb24321ab09396a38ab1c08becb33bba2 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -91,7 +91,7 @@ class Moc(Diagnostic): ('time', 'lev', 'i', 'j', 'basin'), fill_value=example._FillValue) - moc.units = example.units + moc.units = Utils.convert_to_ASCII_if_possible(example.units) moc.add_offset = example.add_offset moc.scale_factor = example.scale_factor diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index 072bfb875b0db305a08263a2405d71e85aa9d50d..2c7ff0462772f63b1f3e82116a8977111bcce889 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -25,7 +25,7 @@ class Psi(Diagnostic): :type chunk: int """ - alias = 'psi' + alias = 'diagnostic' "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk): diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 33fe83d967c36795be04cb9ec7d2c2786480f12b..8831bca6eeb5cb91526c9b0eb6830e4f510bab25 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -51,8 +51,8 @@ class ClimatologicalPercentile(Diagnostic): return self.domain == other.domain and self.variable == other.variable and self.leadtimes == other.leadtimes def __str__(self): - return 'Climatological percentile Variable: {0}:{1} Leadtimes: {2}'.format(self.domain, self.variable, - self.leadtimes) + return 'Climatological percentile Variable: {0}:{1} Leadtimes: {2} ' \ + 'Bins: {3}'.format(self.domain, self.variable, self.leadtimes, self.num_bins) @classmethod def generate_jobs(cls, diags, options): diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 45b7652c16da2689003c8e71dd8caa378a2ee37a..f609d2d59e82bc3e005ef65d72702a24f5d18fac 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -3,7 +3,7 @@ import shutil from autosubmit.config.log import Log -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticIntOption +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption from earthdiagnostics.frequency import Frequencies from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.variable_type import VariableType @@ -29,23 +29,24 @@ class MonthlyPercentile(Diagnostic): alias = 'monpercent' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, startdate, member, chunk, variable, domain, percentile): + def __init__(self, data_manager, startdate, member, chunk, domain, variable, percentiles): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member self.chunk = chunk self.variable = variable self.domain = domain - self.percentile = percentile + self.percentiles = percentiles def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and self.percentile == other.percentile + self.domain == other.domain and self.variable == other.variable and self.percentiles == other.percentiles def __str__(self): - return 'Monthly percentile {0} Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Percentile: {5}'.format(self.startdate, self.member, self.chunk, - self.domain, self.variable, self.percentile) + return 'Monthly percentile Startdate: {0} Member: {1} Chunk: {2} ' \ + 'Variable: {3}:{4} Percentiles: {5}'.format(self.startdate, self.member, self.chunk, + self.domain, self.variable, + ', '.join(str(i) for i in self.percentiles)) @classmethod def generate_jobs(cls, diags, options): @@ -60,13 +61,13 @@ class MonthlyPercentile(Diagnostic): """ options_available = (DiagnosticOption('domain'), DiagnosticDomainOption('variable'), - DiagnosticIntOption('percentile', None, 0, 100)) + DiagnosticListIntOption('percentiles', None, 0, 100)) options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(MonthlyPercentile(diags.data_manager, startdate, member, chunk, - options['variable'], options['domain'], options['percentile'])) + options['variable'], options['domain'], options['percentiles'])) return job_list def compute(self): @@ -108,12 +109,13 @@ class MonthlyPercentile(Diagnostic): monmax_file = TempFile.get() Utils.cdo.monmax(input=variable_file, output=monmax_file) - Log.debug('Computing percentile') - Utils.cdo.monpctl(str(self.percentile), input=[variable_file, monmin_file, monmax_file], output=temp) - Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, - self.member, self.chunk, frequency=Frequencies.monthly, rename_var=self.variable, - vartype=VariableType.STATISTIC) + for percentile in self.percentiles: + Log.debug('Computing percentile {0}', percentile) + Utils.cdo.monpctl(str(percentile), input=[variable_file, monmin_file, monmax_file], output=temp) + Utils.rename_variable(temp, 'lev', 'ensemble', False, True) + self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, percentile), self.startdate, + self.member, self.chunk, frequency=Frequencies.monthly, rename_var=self.variable, + vartype=VariableType.STATISTIC) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index ba0dce80577d956ef20439c781a88ece6c0cdbeb..f2395a905950576fc88244cccf7c91b69ab6e934 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -79,7 +79,8 @@ class THREDDSManager(DataManager): """ aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) - start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', + self.experiment.calendar) end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) @@ -113,7 +114,8 @@ class THREDDSManager(DataManager): """ aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) - start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', + self.experiment.calendar) end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) @@ -288,7 +290,7 @@ class THREDDSManager(DataManager): else: return '{0}.nc'.format(var) - def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, + def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): """ Creates the link of a given file from the CMOR repository. @@ -309,8 +311,6 @@ class THREDDSManager(DataManager): :type chunk: int :param grid: file's grid (only needed if it is not the original) :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: str :param vartype: Variable type (mean, statistic) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 4c5d6317c5f4d1cab0629c55e1bf283c4190b580..308ec030940b1acbcdaf38af24aeb96d746e90a0 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -546,6 +546,7 @@ class Utils(object): :type path: str """ if not os.path.exists(path): + # noinspection PyBroadException try: os.makedirs(path) except: @@ -580,10 +581,12 @@ class Utils(object): """ Unzip a list of files :param files: files to unzip - :type files: list + :type files: list | str :param force: if True, it will overwrite unzipped files :type force: bool """ + if isinstance(files, basestring): + files = [files] for filepath in files: Log.debug('Unzipping {0}', filepath) if force: diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index 5b33488658f53ab789194a7c98fa50e20b2992fa..1507fa5fa953fb842ab5f3e116043724cb246bd3 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -8,7 +8,7 @@ from autosubmit.config.log import Log from earthdiagnostics.constants import Basins from earthdiagnostics.frequency import Frequency -from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms +from earthdiagnostics.modelingrealm import ModelingRealms class VariableJsonException(Exception): @@ -144,6 +144,7 @@ class VariableManager(object): def _load_json_variables(self, json_data, table): for short_name in json_data.keys(): + short_name = str.strip(str(short_name)) if short_name.lower() in self._dict_variables: self._dict_variables[short_name.lower()].tables.append(table) continue @@ -176,6 +177,7 @@ class VariableManager(object): cmor_vars = [] for alias in aliases: + alias = str.strip(alias) if alias.lower() in self._dict_variables: cmor_vars.append(self._dict_variables[alias.lower()]) if len(cmor_vars) == 0: @@ -295,18 +297,18 @@ class Variable(object): def parse_json(self, json_var, key): if 'out_name' in json_var: - self.short_name = json_var['out_name'] + self.short_name = json_var['out_name'].strip() else: raise VariableJsonException('Variable has no out name defined'.format(key)) - self.standard_name = json_var['standard_name'] - self.long_name = json_var['long_name'] + self.standard_name = json_var['standard_name'].strip() + self.long_name = json_var['long_name'].strip() domain = json_var['modeling_realm'].split(' ') self.domain = self.get_modelling_realm(domain) - self.valid_min = json_var['valid_min'] - self.valid_max = json_var['valid_max'] - self.units = json_var['units'] + self.valid_min = json_var['valid_min'].strip() + self.valid_max = json_var['valid_max'].strip() + self.units = json_var['units'].strip() def get_modelling_realm(self, domains): if len(domains) > 1: @@ -368,9 +370,6 @@ class Variable(object): return None - - - class VariableAlias(object): """ Class to characterize a CMOR variable. It also contains the static method to make the match between thje original diff --git a/earthdiagnostics/variable_alias/cmip6.csv b/earthdiagnostics/variable_alias/cmip6.csv index 6aee91f502a827957a7411b8657e52e8d0344f37..e996633ec04d5e21a7fdd0f7001c210caaecce47 100644 --- a/earthdiagnostics/variable_alias/cmip6.csv +++ b/earthdiagnostics/variable_alias/cmip6.csv @@ -1,3 +1,73 @@ Aliases,Shortname,Basin,Grid iiceconc:soicecov:ileadfra,siconc,, ci,siconc,,ifs +alk,talk,, +oxygen,o2,, +calcite,calc,, +po4,po4,, +poc,poc,, +silicate,si,, +nanophy,nanophy,, +microzoo,zmicro,, +doc,dissoc,, +diaphy,phydiat,, +mesozoo,zmeso,, +dsi,dsi,, +dissfe,dfe,, +bfe,bfe,, +goc,goc,, +sfe,sfe,, +dfe,dfe,, +micrzoo,zmicro,, +nfe,nfe,, +nchl,nchl,, +dchl,chldiat,, +nitrate,no3,, +ammonium,nh4,, +pno3tot,pno3tot,, +psiltot,psiltot,, +palktot,palktot,, +pfertot,pfertot,, +tcflx,tcflx,, +tcflxcum,tcflxcum,, +c-export,c-export,, +tintpp,tintpp,, +tnfix,tnfix,, +tdenit,tdenit,, +inttpp,inttpp,, +inttppnew,inttppnew,, +inttpbfe,pbfe,, +intdic,intdic,, +o2min,o2min,, +zo2min,zo2min,, +intnfix,intpn2,, +intppphy,intppphy,, +intppphy2,intppdiat,, +ppphy ,ppphy ,, +ppphy2 ,pdi,, +intpp,intpp,, +intpbfe,intpbfe,, +intpbsi,intpbsi,, +intpbcal,intpbcal,, +cflx,cflx,, +remin,remin,, +denit,denit,, +nfix,nfix,, +sdenit,sdenit,, +dpco2,dpco2,, +epc100,epc100,, +expc,expc,, +par,par,, +lnnut,lnnut,, +ldnut,ldnut,, +lnfe,lnfe,, +ldfe,limfediat,, +lnlight,lnlight,, +ldlight,ldlight,, +graz1,graz1,, +graz2,graz2,, +mumax,mumax,, +mun,mun,, +mud,mud,, +ppnewn,ppnewn,, +ppnewd,ppnewd,, diff --git a/earthdiagnostics/variable_alias/specs.csv b/earthdiagnostics/variable_alias/specs.csv index 4b0687598bcae99cd861f2d4ce93289d3ca03975..993ebfa5c4cb1dba85d7400e8cba3d03d0349847 100644 --- a/earthdiagnostics/variable_alias/specs.csv +++ b/earthdiagnostics/variable_alias/specs.csv @@ -1,3 +1,3 @@ Aliases,Shortname,Basin,Grid -iiceconc:siconc:soicecov,sic,, +iiceconc:siconc:soicecov:ileadfra,sic,, ci,sic,,ifs \ No newline at end of file diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 3c07e90196c414109ffe592d381dccdc45d463ff..caa995421069086fb1ec536dd3868931ac30e48e 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -11,6 +11,7 @@ from test_areamoc import TestAreaMoc from test_averagesection import TestAverageSection from test_cutsection import TestCutSection from test_convectionsites import TestConvectionSites +from test_frequency import TestFrequency from test_gyres import TestGyres from test_heatcontent import TestHeatContent from test_heatcontentlayer import TestHeatContentLayer @@ -19,8 +20,13 @@ from test_maxmoc import TestMaxMoc from test_mixedlayerheatcontent import TestMixedLayerHeatContent from test_mixedlayersaltcontent import TestMixedLayerSaltContent from test_moc import TestMoc +from test_modelling_realm import TestModellingRealms, TestModellingRealm from test_siasiesiv import TestSiasiesiv from test_verticalmean import TestVerticalMean from test_verticalmeanmeters import TestVerticalMeanMeters from test_monthlymean import TestMonthlyMean from test_rewrite import TestRewrite +from test_variable_type import TestVariableType +from test_monthlypercentile import TestMonthlyPercentile +from test_climatologicalpercentile import TestClimatologicalPercentile +from test_variable import TestCMORTable, TestVariableAlias diff --git a/test/unit/test_areamoc.py b/test/unit/test_areamoc.py index f4ddf699dc690767ed161511060f67c535ec0aa3..fb23ff38dca894d3822aa7db651c88418c8c2a2b 100644 --- a/test/unit/test_areamoc.py +++ b/test/unit/test_areamoc.py @@ -23,20 +23,20 @@ class TestAreaMoc(TestCase): self.psi = AreaMoc(self.data_manager, '20000101', 1, 1, Basins.Antarctic, self.box) def test_generate_jobs(self): - jobs = AreaMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0']) + jobs = AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, Basins.Global, self.box)) self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, Basins.Global, self.box)) - jobs = AreaMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', 'atl']) + jobs = AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', 'atl']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, Basins.Atlantic, self.box)) self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, Basins.Atlantic, self.box)) with self.assertRaises(Exception): - AreaMoc.generate_jobs(self.diags, ['psi']) + AreaMoc.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - AreaMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0']) + AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.psi), 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0') diff --git a/test/unit/test_averagesection.py b/test/unit/test_averagesection.py index 78554e5e495ab6b28a7d5c90e7d7537628abf1b7..7a454c4a97c2b13ccfe099169a9dfc12d2548b79 100644 --- a/test/unit/test_averagesection.py +++ b/test/unit/test_averagesection.py @@ -24,14 +24,14 @@ class TestAverageSection(TestCase): self.psi = AverageSection(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', self.box) def test_generate_jobs(self): - jobs = AverageSection.generate_jobs(self.diags, ['psi', 'var', '0', '0', '0', '0']) + jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'var', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', self.box)) self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', self.box)) - jobs = AverageSection.generate_jobs(self.diags, ['psi', 'var', '0', '0', '0', '0', 'ocean']) + jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'var', '0', '0', '0', '0', 'ocean']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', self.box)) @@ -39,9 +39,9 @@ class TestAverageSection(TestCase): self.box)) with self.assertRaises(Exception): - AverageSection.generate_jobs(self.diags, ['psi']) + AverageSection.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - AverageSection.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + AverageSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.psi), 'Average section Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0E ' diff --git a/test/unit/test_climatologicalpercentile.py b/test/unit/test_climatologicalpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..95afc38bb081bf3cdee76a6f64521c688817158d --- /dev/null +++ b/test/unit/test_climatologicalpercentile.py @@ -0,0 +1,35 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile +from mock import Mock + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestClimatologicalPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.data_manager.variable_list.get_variable.return_value = None + + self.diags = Mock() + self.diags.data_manager = self.data_manager + + self.diagnostic = ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', + [10, 90], 1000, self.diags.config.experiment) + + def test_generate_jobs(self): + jobs = ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', 'ocean', 'var', '1-2', '1000']) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0], ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', [1, 2], + 1000, self.diags.config.experiment)) + + with self.assertRaises(Exception): + ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent']) + with self.assertRaises(Exception): + ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.diagnostic), 'Climatological percentile Variable: ocean:var Leadtimes: [10, 90] ' + 'Bins: 1000') diff --git a/test/unit/test_constants.py b/test/unit/test_constants.py index f8010d53a4dbbe42a506dde27c501ed5f0e076f2..720478187ac7ca5c8d7cb60b918f6e13498db779 100644 --- a/test/unit/test_constants.py +++ b/test/unit/test_constants.py @@ -29,3 +29,6 @@ class TestBasin(TestCase): self.assertFalse(Basin('bas', 'OtherBasin') == self.basin) self.assertFalse(Basin('otbas', 'Basin') == self.basin) self.assertFalse(Basin('otbas', 'OtherBasin') == self.basin) + + def test__str__(self): + self.assertEquals(str(self.basin), 'Basin') diff --git a/test/unit/test_convectionsites.py b/test/unit/test_convectionsites.py index b6a7e543ba60da2d741734a6d1eeaeff95e1bad6..63710d054e6a4f937cb9c22e8e78d659c80f3c73 100644 --- a/test/unit/test_convectionsites.py +++ b/test/unit/test_convectionsites.py @@ -16,13 +16,13 @@ class TestConvectionSites(TestCase): self.psi = ConvectionSites(self.data_manager, '20000101', 1, 1, 'model_version') def test_generate_jobs(self): - jobs = ConvectionSites.generate_jobs(self.diags, ['psi']) + jobs = ConvectionSites.generate_jobs(self.diags, ['diagnostic']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], ConvectionSites(self.data_manager, '20010101', 0, 0, 'model_version')) self.assertEqual(jobs[1], ConvectionSites(self.data_manager, '20010101', 0, 1, 'model_version')) with self.assertRaises(Exception): - ConvectionSites.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + ConvectionSites.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.psi), 'Convection sites Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/test_cutsection.py b/test/unit/test_cutsection.py index 8cfb3cdb4e4f6ffd56e58e064522d01c7784d7ec..170d04265c6a55a8816deb494416f04d3237abaa 100644 --- a/test/unit/test_cutsection.py +++ b/test/unit/test_cutsection.py @@ -24,14 +24,14 @@ class TestCutSection(TestCase): self.psi = CutSection(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', True, 0) def test_generate_jobs(self): - jobs = CutSection.generate_jobs(self.diags, ['psi', 'var', 'true', '10']) + jobs = CutSection.generate_jobs(self.diags, ['diagnostic', 'var', 'true', '10']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', True, 10)) self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', True, 10)) - jobs = CutSection.generate_jobs(self.diags, ['psi', 'var', 'false', '0', 'atmos']) + jobs = CutSection.generate_jobs(self.diags, ['diagnostic', 'var', 'false', '0', 'atmos']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], CutSection(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', False, 0)) @@ -39,9 +39,9 @@ class TestCutSection(TestCase): False, 0)) with self.assertRaises(Exception): - CutSection.generate_jobs(self.diags, ['psi']) + CutSection.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - CutSection.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + CutSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.psi), 'Cut section Startdate: 20000101 Member: 1 Chunk: 1 Variable: atmos:var ' diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py new file mode 100644 index 0000000000000000000000000000000000000000..155c37f9fd4e732814c1d70ec479393507406baf --- /dev/null +++ b/test/unit/test_earthdiags.py @@ -0,0 +1,10 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.earthdiags import EarthDiags + + +class TestEarthDiags(TestCase): + pass + + diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py new file mode 100644 index 0000000000000000000000000000000000000000..28b5bd9117ca91329f701ad99aee440c9ed51601 --- /dev/null +++ b/test/unit/test_frequency.py @@ -0,0 +1,32 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.frequency import Frequencies, Frequency +from earthdiagnostics.variable_type import VariableType + + +class TestFrequency(TestCase): + + def test_not_supported(self): + with self.assertRaises(ValueError): + Frequency('badfreq') + + def test_get_monthly_mean(self): + self.assertEqual(Frequency('m').folder_name(VariableType.MEAN), 'monthly_mean') + + def test_get_monthly_stats(self): + self.assertEqual(Frequency('m').folder_name(VariableType.STATISTIC), 'monthly_statistics') + + def test_get_daily_mean(self): + self.assertEqual(Frequency('d').folder_name(VariableType.MEAN), 'daily_mean') + + def test_get_daily_stats(self): + self.assertEqual(Frequency('d').folder_name(VariableType.STATISTIC), 'daily_statistics') + + def test_get_6hourlymean(self): + self.assertEqual(Frequency('6hr').folder_name(VariableType.STATISTIC), '6hourly') + + def test_get_climatology(self): + self.assertEqual(Frequency('clim').folder_name(VariableType.STATISTIC), 'clim') + self.assertEqual(Frequency('clim').folder_name(VariableType.MEAN), 'clim') + diff --git a/test/unit/test_gyres.py b/test/unit/test_gyres.py index 77f3987258eca4cee0a494197090db777a2d7061..becc4e7d331754aff23baae244d82154252eaa6e 100644 --- a/test/unit/test_gyres.py +++ b/test/unit/test_gyres.py @@ -17,13 +17,13 @@ class TestGyres(TestCase): self.gyres = Gyres(self.data_manager, '20000101', 1, 1, 'model_version') def test_generate_jobs(self): - jobs = Gyres.generate_jobs(self.diags, ['psi']) + jobs = Gyres.generate_jobs(self.diags, ['diagnostic']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Gyres(self.data_manager, '20010101', 0, 0, 'model_version')) self.assertEqual(jobs[1], Gyres(self.data_manager, '20010101', 0, 1, 'model_version')) with self.assertRaises(Exception): - Gyres.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + Gyres.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.gyres), 'Gyres Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/test_heatcontent.py b/test/unit/test_heatcontent.py index b2c28f8cac30f3b418989ff8a496a88d4a628af0..8452eb598d1cf4e7d62a5e7c0f3a2f1922d505fc 100644 --- a/test/unit/test_heatcontent.py +++ b/test/unit/test_heatcontent.py @@ -23,16 +23,16 @@ class TestHeatContent(TestCase): self.heat_content = HeatContent(self.data_manager, '20000101', 1, 1, Basins.Global, 1, self.box) def test_generate_jobs(self): - jobs = HeatContent.generate_jobs(self.diags, ['psi', 'atl', '-1', '0', '100']) + jobs = HeatContent.generate_jobs(self.diags, ['diagnostic', 'atl', '-1', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], HeatContent(self.data_manager, '20010101', 0, 0, Basins.Atlantic, -1, self.box)) self.assertEqual(jobs[1], HeatContent(self.data_manager, '20010101', 0, 1, Basins.Atlantic, -1, self.box)) with self.assertRaises(Exception): - HeatContent.generate_jobs(self.diags, ['psi']) + HeatContent.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - HeatContent.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + HeatContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.heat_content), 'Heat content Startdate: 20000101 Member: 1 Chunk: 1 Mixed layer: 1 ' diff --git a/test/unit/test_maxmoc.py b/test/unit/test_maxmoc.py index 99c2fca6da18126d818ec305728a900ef5d4bb1b..35117a673907aa68ea13978ae39111e75bdba608 100644 --- a/test/unit/test_maxmoc.py +++ b/test/unit/test_maxmoc.py @@ -27,25 +27,25 @@ class TestMaxMoc(TestCase): self.diags.config.experiment.members = (0,) self.diags.config.experiment.get_full_years.return_value = (2000, 2001) - jobs = MaxMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0']) + jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, Basins.Global, self.box)) self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, Basins.Global, self.box)) - jobs = MaxMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', 'atl']) + jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', 'atl']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, Basins.Atlantic, self.box)) self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, Basins.Atlantic, self.box)) self.diags.config.experiment.get_full_years.return_value = list() - jobs = MaxMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0']) + jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) self.assertEqual(len(jobs), 0) with self.assertRaises(Exception): - MaxMoc.generate_jobs(self.diags, ['psi']) + MaxMoc.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - MaxMoc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.maxmoc), 'Max moc Startdate: 20000101 Member: 1 Year: 2000 ' diff --git a/test/unit/test_mixedlayerheatcontent.py b/test/unit/test_mixedlayerheatcontent.py index bf7bff95ff38b46ee998d78d4ce005217a9085d9..b7cf564bc9cdfabe924327ea0602ea3bc6d84478 100644 --- a/test/unit/test_mixedlayerheatcontent.py +++ b/test/unit/test_mixedlayerheatcontent.py @@ -17,13 +17,13 @@ class TestMixedLayerHeatContent(TestCase): self.mixed = MixedLayerHeatContent(self.data_manager, '20000101', 1, 1) def test_generate_jobs(self): - jobs = MixedLayerHeatContent.generate_jobs(self.diags, ['psi']) + jobs = MixedLayerHeatContent.generate_jobs(self.diags, ['diagnostic']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MixedLayerHeatContent(self.data_manager, '20010101', 0, 0)) self.assertEqual(jobs[1], MixedLayerHeatContent(self.data_manager, '20010101', 0, 1)) with self.assertRaises(Exception): - MixedLayerHeatContent.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + MixedLayerHeatContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Mixed layer heat content Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/test_mixedlayersaltcontent.py b/test/unit/test_mixedlayersaltcontent.py index 38074f0f481675b968dd331f8e5b65a08a4c5794..7aa42f6691eff2d80c4290b1fa5d1505794543d2 100644 --- a/test/unit/test_mixedlayersaltcontent.py +++ b/test/unit/test_mixedlayersaltcontent.py @@ -17,13 +17,13 @@ class TestMixedLayerSaltContent(TestCase): self.mixed = MixedLayerSaltContent(self.data_manager, '20000101', 1, 1) def test_generate_jobs(self): - jobs = MixedLayerSaltContent.generate_jobs(self.diags, ['psi']) + jobs = MixedLayerSaltContent.generate_jobs(self.diags, ['diagnostic']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MixedLayerSaltContent(self.data_manager, '20010101', 0, 0)) self.assertEqual(jobs[1], MixedLayerSaltContent(self.data_manager, '20010101', 0, 1)) with self.assertRaises(Exception): - MixedLayerSaltContent.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + MixedLayerSaltContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Mixed layer salt content Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/test_moc.py b/test/unit/test_moc.py index 1a143035edfc3e841d3b9907bfa21ddd2214bf90..0c05f8c4fe7dd3632cc5b662aacda78c95dbfe95 100644 --- a/test/unit/test_moc.py +++ b/test/unit/test_moc.py @@ -17,13 +17,13 @@ class TestMoc(TestCase): self.mixed = Moc(self.data_manager, '20000101', 1, 1) def test_generate_jobs(self): - jobs = Moc.generate_jobs(self.diags, ['psi']) + jobs = Moc.generate_jobs(self.diags, ['diagnostic']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Moc(self.data_manager, '20010101', 0, 0)) self.assertEqual(jobs[1], Moc(self.data_manager, '20010101', 0, 1)) with self.assertRaises(Exception): - Moc.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + Moc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'MOC Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py new file mode 100644 index 0000000000000000000000000000000000000000..2d44e6a6ddb03a17835afd1f3b5bb1bc881623f0 --- /dev/null +++ b/test/unit/test_modelling_realm.py @@ -0,0 +1,52 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms + + +class TestModellingRealms(TestCase): + + def test_parse(self): + self.assertEquals(ModelingRealms.parse('atmos'), ModelingRealms.atmos) + self.assertEquals(ModelingRealms.parse('atmoschem'), ModelingRealms.atmosChem) + self.assertEquals(ModelingRealms.parse('atmoSChem'), ModelingRealms.atmosChem) + with self.assertRaises(ValueError): + ModelingRealms.parse('badrealm') + + +class TestModellingRealm(TestCase): + + def setUp(self): + self.basin = ModelingRealm('ocean') + + def test_constructor_fail_on_bad_realm(self): + with self.assertRaises(ValueError): + ModelingRealm('badrealm') + + def test_comparison(self): + self.assertEqual(ModelingRealm('ocean'), self.basin) + self.assertNotEqual(ModelingRealm('OCEAN'), self.basin) + self.assertNotEqual(ModelingRealm('atmos'), self.basin) + + def test_get_omon(self): + self.assertEqual(self.basin.get_table_name(Frequencies.monthly, 'specs'), 'Omon') + + def test_get_oimon(self): + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'specs'), 'OImon') + + def test_get_simon(self): + self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'cmip6'), 'SImon') + + def test_get_limon(self): + self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, 'specs'), 'LImon') + + def test_get_day(self): + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, 'specs'), 'day') + + def test_get_6hrplev(self): + self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev') + + + + diff --git a/test/unit/test_monthlymean.py b/test/unit/test_monthlymean.py index 91e42a5387c9cf2df305f0f31321aa52caa3cd5d..e2165f5dee2f7bb65259926f1c946ea3801af080 100644 --- a/test/unit/test_monthlymean.py +++ b/test/unit/test_monthlymean.py @@ -26,21 +26,21 @@ class TestMonthlyMean(TestCase): def test_generate_jobs(self): - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'ocean']) + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'var', 'ocean']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', Frequencies.daily, '')) self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', Frequencies.daily, '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'atmos', 'monthly']) + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'var', 'atmos', 'monthly']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', Frequencies.monthly, '')) self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', Frequencies.monthly, '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['psi', 'var', 'seaice', 'mon', 'grid']) + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'var', 'seaice', 'mon', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', Frequencies.monthly, 'grid')) @@ -48,10 +48,10 @@ class TestMonthlyMean(TestCase): Frequencies.monthly, 'grid')) with self.assertRaises(Exception): - MonthlyMean.generate_jobs(self.diags, ['psi']) + MonthlyMean.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - MonthlyMean.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + MonthlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Calculate monthly mean Startdate: 20000101 Member: 1 Chunk: 1 ' diff --git a/test/unit/test_monthlypercentile.py b/test/unit/test_monthlypercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..4b9bbfb09ed3b03bb0aa2b4cd68a373972c844a7 --- /dev/null +++ b/test/unit/test_monthlypercentile.py @@ -0,0 +1,41 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.box import Box +from earthdiagnostics.statistics.monthlypercentile import MonthlyPercentile +from mock import Mock + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestMonthlyPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + + self.box = Box() + self.box.min_lat = 0 + self.box.max_lat = 0 + self.box.min_lon = 0 + self.box.max_lon = 0 + + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diagnostic = MonthlyPercentile(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', [10, 90]) + + def test_generate_jobs(self): + jobs = MonthlyPercentile.generate_jobs(self.diags, ['monpercent', 'var', 'ocean', '10-90']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MonthlyPercentile(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + [10, 90])) + self.assertEqual(jobs[1], MonthlyPercentile(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + [10, 90])) + + with self.assertRaises(Exception): + MonthlyPercentile.generate_jobs(self.diags, ['monpercent']) + with self.assertRaises(Exception): + MonthlyPercentile.generate_jobs(self.diags, ['monpercent', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.diagnostic), 'Monthly percentile Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Percentiles: 10, 90') diff --git a/test/unit/test_psi.py b/test/unit/test_psi.py index 3099fa82bd3178f7dec2327623e59ec06e7b326e..019e9339ca281274d59ba15b07bf190ee9b457da 100644 --- a/test/unit/test_psi.py +++ b/test/unit/test_psi.py @@ -13,13 +13,13 @@ class TestPsi(TestCase): self.psi = Psi(self.data_manager, '20000101', 1, 1) def test_generate_jobs(self): - jobs = Psi.generate_jobs(self.diags, ['psi']) + jobs = Psi.generate_jobs(self.diags, ['diagnostic']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Psi(self.data_manager, '20010101', 0, 0)) self.assertEqual(jobs[1], Psi(self.data_manager, '20010101', 0, 1)) with self.assertRaises(Exception): - Psi.generate_jobs(self.diags, ['psi', 'badoption']) + Psi.generate_jobs(self.diags, ['diagnostic', 'badoption']) def test_str(self): self.assertEquals(str(self.psi), 'PSI Startdate: 20000101 Member: 1 Chunk: 1') diff --git a/test/unit/test_rewrite.py b/test/unit/test_rewrite.py index f125947f4c2f82c3c15eb5d0b0cb5e8360419f56..25380fcd81a752a85e679ff93787aa2ec4d0bf77 100644 --- a/test/unit/test_rewrite.py +++ b/test/unit/test_rewrite.py @@ -25,21 +25,21 @@ class TestRewrite(TestCase): def test_generate_jobs(self): - jobs = Rewrite.generate_jobs(self.diags, ['psi', 'var', 'atmos']) + jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'var', 'atmos']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'original')) self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'original')) - jobs = Rewrite.generate_jobs(self.diags, ['psi', 'var', 'ocean', 'grid']) + jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'var', 'ocean', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid')) self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid')) with self.assertRaises(Exception): - Rewrite.generate_jobs(self.diags, ['psi']) + Rewrite.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - Rewrite.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + Rewrite.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Rewrite output Startdate: 20000101 Member: 1 Chunk: 1 ' diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 7f3e0b8472157c518f517af25cbafdb1a4812bf5..07199ab822962e8d8c43a6685dddc30ddb4ff96f 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -1,29 +1,27 @@ # coding=utf-8 -# from unittest import TestCase -# -# from earthdiagnostics.variable import Variable -# from earthdiagnostics.modelingrealm import ModelingRealms - - -# class TestVariable(TestCase): -# -# def test__init__(self): -# variable = Variable('alias:alias2,name,standard_name,long_name,ocean,basin,units,' -# 'valid_min,valid_max,grid'.split(',')) -# self.assertEqual(variable.short_name, 'name') -# self.assertEqual(variable.standard_name, 'standard_name') -# self.assertEqual(variable.long_name, 'long_name') -# self.assertEqual(variable.domain, Domains.ocean) -# self.assertEqual(variable.basin, None) -# self.assertEqual(variable.units, 'units') -# self.assertEqual(variable.valid_min, 'valid_min') -# self.assertEqual(variable.valid_max, 'valid_max') -# self.assertEqual(variable.grid, 'grid') -# -# def test_get_variable(self): -# Variable._dict_variables = dict() -# variable = Variable('alias:alias2,name,standard_name,long_name,atmos,basin,units,valid_min,' -# 'valid_max,grid'.split(',')) -# Variable._dict_variables['var'] = variable -# self.assertIs(Variable.get_variable('var'), variable) -# self.assertIsNone(Variable.get_variable('novar')) + +from mock import Mock +from unittest import TestCase +from earthdiagnostics.variable import CMORTable, VariableAlias + + +class TestCMORTable(TestCase): + + def setUp(self): + self.frequency = Mock() + + def test_str(self): + self.assertEquals(str(CMORTable('name', 'm', 'Month YEAR')), 'name') + + +class TestVariableAlias(TestCase): + + def test_str(self): + alias = VariableAlias('alias') + self.assertEquals(str(alias), 'alias') + alias.basin = 'basin' + self.assertEquals(str(alias), 'alias Basin: basin') + alias.grid = 'grid' + self.assertEquals(str(alias), 'alias Basin: basin Grid: grid') + + diff --git a/test/unit/test_variable_type.py b/test/unit/test_variable_type.py new file mode 100644 index 0000000000000000000000000000000000000000..28dd44f56c34599ce1c9d293d6a1a430e71709d7 --- /dev/null +++ b/test/unit/test_variable_type.py @@ -0,0 +1,20 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.variable_type import VariableType + + +class TestVariableType(TestCase): + + def test_mean(self): + self.assertEqual(VariableType.to_str(VariableType.MEAN), 'mean') + + def test_statistics(self): + self.assertEqual(VariableType.to_str(VariableType.STATISTIC), 'statistics') + + def test_bad_one(self): + with self.assertRaises(ValueError): + VariableType.to_str('bad type') + + + diff --git a/test/unit/test_verticalmean.py b/test/unit/test_verticalmean.py index dc2d32ae8d45f5160f453acdf7263412c53eda90..59d0fb501a35cabcba02d01cdc189d3dc2a302be 100644 --- a/test/unit/test_verticalmean.py +++ b/test/unit/test_verticalmean.py @@ -22,28 +22,28 @@ class TestVerticalMean(TestCase): self.mixed = VerticalMean(self.data_manager, '20000101', 1, 1, 'var', self.box) def test_generate_jobs(self): - jobs = VerticalMean.generate_jobs(self.diags, ['psi', 'var', '0', '100']) + jobs = VerticalMean.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMean(self.data_manager, '20010101', 0, 0, 'var', self.box)) self.assertEqual(jobs[1], VerticalMean(self.data_manager, '20010101', 0, 1, 'var', self.box)) - jobs = VerticalMean.generate_jobs(self.diags, ['psi', 'var', '0']) + jobs = VerticalMean.generate_jobs(self.diags, ['diagnostic', 'var', '0']) box = Box() box.min_depth = 0 self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMean(self.data_manager, '20010101', 0, 0, 'var', box)) self.assertEqual(jobs[1], VerticalMean(self.data_manager, '20010101', 0, 1, 'var', box)) - jobs = VerticalMean.generate_jobs(self.diags, ['psi', 'var']) + jobs = VerticalMean.generate_jobs(self.diags, ['diagnostic', 'var']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMean(self.data_manager, '20010101', 0, 0, 'var', Box())) self.assertEqual(jobs[1], VerticalMean(self.data_manager, '20010101', 0, 1, 'var', Box())) with self.assertRaises(Exception): - VerticalMean.generate_jobs(self.diags, ['psi']) + VerticalMean.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - VerticalMean.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + VerticalMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' diff --git a/test/unit/test_verticalmeanmeters.py b/test/unit/test_verticalmeanmeters.py index 20599cd56fe64c3f4e817a60c396cf167164fc59..2a70f1edeb3fd32b8725948a89e1474b8833d9cd 100644 --- a/test/unit/test_verticalmeanmeters.py +++ b/test/unit/test_verticalmeanmeters.py @@ -22,28 +22,28 @@ class TestVerticalMeanMeters(TestCase): self.mixed = VerticalMeanMeters(self.data_manager, '20000101', 1, 1, 'var', self.box) def test_generate_jobs(self): - jobs = VerticalMeanMeters.generate_jobs(self.diags, ['psi', 'var', '0', '100']) + jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, 'var', self.box)) self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, 'var', self.box)) - jobs = VerticalMeanMeters.generate_jobs(self.diags, ['psi', 'var', '0']) + jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var', '0']) box = Box(True) box.min_depth = 0 self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, 'var', box)) self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, 'var', box)) - jobs = VerticalMeanMeters.generate_jobs(self.diags, ['psi', 'var']) + jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, 'var', Box(True))) self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, 'var', Box(True))) with self.assertRaises(Exception): - VerticalMeanMeters.generate_jobs(self.diags, ['psi']) + VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - VerticalMeanMeters.generate_jobs(self.diags, ['psi', '0', '0', '0', '0', '0', '0', '0']) + VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 Variable: var '