diff --git a/VERSION b/VERSION index f335f1fb6f68c35f86caa0e79ec4e3943b0719f2..c4dd8a8adc11e0beee6be4c8c5bab7389b9c2c55 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -3.0.0b57 +3.0.0rc1 diff --git a/diags.conf b/diags.conf index b9d3207df956532412ffbb559ecc1c265c78a999..7c063c694accc339a9709b8419eb4e0eb36c4eab 100644 --- a/diags.conf +++ b/diags.conf @@ -1,32 +1,33 @@ [DIAGNOSTICS] # Data adaptor type: CMOR (for our experiments), THREDDS (for other experiments) -DATA_ADAPTOR = CMOR +DATA_ADAPTOR = OBSRECON # Path to the folder where you want to create the temporary files SCRATCH_DIR = /scratch/Earth/$USER # Root path for the cmorized data to use -DATA_DIR = /esnas:/esarchive:/esarchive/exp/PREFACE +DATA_DIR = /esnas:/esarchive # Specify if your data is from an experiment (exp), observation (obs) or reconstructions (recon) -DATA_TYPE = exp +DATA_TYPE = recon # CMORization type to use. Important also for THREDDS as it affects variable name conventions. # Options: SPECS (default), PRIMAVERA, CMIP6 -DATA_CONVENTION = PREFACE +DATA_CONVENTION = SPECS # Path to NEMO's mask and grid files needed for CDFTools CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it # empty -DIAGS = interpcdo,tos,global_1,,,True,,True -# DIAGS = OHC +#DIAGS = discretize,atmos,sfcWind,,0,40 +DIAGS = climpercent,atmos,sfcWind,2010,2012,11 daysover,atmos,sfcWind,2010,2012,11 +#DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. -FREQUENCY = mon +FREQUENCY = 6hr # Path to CDFTOOLS binaries CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin # If true, copies the mesh files regardless of presence in scratch dir RESTORE_MESHES = False -# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available) -MAX_CORES = 1 +# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available)z +MAX_CORES = 2 [CMOR] # If true, recreates CMOR files regardless of presence. Default = False @@ -62,19 +63,17 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, # PHYSICS_VERSION = 1 # PHYSICS_DESCRIPTION = # ASSOCIATED_MODEL = -# SOURCE = EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 -VERSION = v20161031 +# SOURCE = 'EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 [THREDDS] SERVER_URL = https://earth.bsc.es/thredds [EXPERIMENT] # Experiments parameters as defined in CMOR standard -INSTITUTE = Cerfacs -MODEL = CNRM-CM-HR -NAME = TAU30 +INSTITUTE = ecmwf +MODEL = erainterim # Model version: Available versions -MODEL_VERSION = Ec3.1_O25L75 +MODEL_VERSION = Ec3.2_O1L75 # Atmospheric output timestep in hours ATMOS_TIMESTEP = 6 # Ocean output timestep in hours @@ -88,12 +87,12 @@ OCEAN_TIMESTEP = 6 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = ctrl -STARTDATES = 20000501 20020201 20030501 20050201 20060501 20080201 20090501 20010201 20020501 20040201 20050501 20070201 20080501 20000201 20010501 20030201 20040501 20060201 20070501 20090201 -MEMBERS = 0 1 2 +EXPID = testing_erainterim +STARTDATES = 20101101 20111101 20121101 +# STARTDATES = 19840101 19850101 +MEMBERS = 0 MEMBER_DIGITS = 1 -# CHUNK_SIZE = 1 -CHUNK_SIZE = 6 +CHUNK_SIZE = 1 CHUNKS = 1 # CHUNKS = 1 @@ -110,17 +109,17 @@ STC = mocarea,0,25,0,200,Pac mocarea,-25,0,0,200,Pac mocarea,0,25,0,200,Atl moca HEAT_SAL_MXL = mlotstsc mlotsthc LMSALC = vertmeanmeters,so,300,5400 USALC = vertmeanmeters,so,0,300 -OHC = ohc,glob,0,0,2000 +OHC = ohc,glob,0,1,10 XOHC = ohc,glob,1,0,0 -LOHC = ohc,glob,0,700,2000 -MOHC = ohc,glob,0,300,700 -UOHC = ohc,glob,0,0,300 +LOHC = ohc,glob,0,23,46 +MOHC = ohc,glob,0,18,22 +UOHC = ohc,glob,0,1,17 OHC_SPECIFIED_LAYER = ohclayer,0,300 ohclayer,300,800 3DTEMP = interp,thetao 3DSAL = interp,so -TSEC_AVE190-220E =avgsection,thetao,190,220,-90,90 -SSEC_AVE190-220E =avgsection,so,190,220,-90,90 -VERT_SSECTIONS = cutsection,so,Z,0 cutsection,so,Z,45 cutsection,so,Z,-45 cutsection,so,M,-30 cutsection,so,M,180 cutsection,so,M,80 +TSEC_AVE190-220E =avgsection,ocean,thetao,190,220,-90,90,regular +SSEC_AVE190-220E =avgsection,ocean,so,190,220,-90,90,regular +VERT_SSECTIONS = cutsection,so,Z,0 cutsection,so,Z,45 cutsection,so,Z,-45 cutsection,so,M,-30 cutsection,so,M,80 VERT_TSECTIONS = cutsection,thetao,Z,0 cutsection,thetao,Z,45 cutsection,thetao,Z,-45 cutsection,thetao,M,-30 cutsection,thetao,M,180 cutsection,thetao,M,80 SIASIESIV = siasiesiv,glob diff --git a/doc/source/config_file.rst b/doc/source/config_file.rst index dde7685aa3d0f930fd55a310416f42075cca43bf..efe2c79bb060589f84c17434f970265fbaaf9be5 100644 --- a/doc/source/config_file.rst +++ b/doc/source/config_file.rst @@ -31,7 +31,7 @@ Mandatory configurations ignore it completely. * DIAGS: - List of diagnostic to run, in the order you want them to run + List of diagnostic to run. No specific order is needed: data dependencies will be enforced. Optional configurations @@ -53,8 +53,8 @@ Optional configurations Type of the dataset to use. It can be exp, obs or recon. Default is exp. * DATA_CONVENTION - Convention to use for file paths and names and variable naming among other things. Can be SPECS, PRIMAVERA or CMIP6. - Default is SPECS. + Convention to use for file paths and names and variable naming among other things. Can be SPECS, PREFACE, + PRIMAVERA or CMIP6. Default is SPECS. * CDFTOOLS_PATH Path to the folder containing CDFTOOLS executables. By default is empty, so CDFTOOLS binaries must be added to the @@ -65,6 +65,28 @@ Optional configurations necessary when launching through a scheduler, as Earthdiagnostics can detect how many cores the scheduler has allocated to it. +* AUTO_CLEAN + If True, EarthDiagnostics removes the temporary folder just after finsihing. If RAM_DISK is set to True, this value + is ignored and always Default is True + +* RAM_DISK + If set to True, the temporary files is created at the /dev/shm partition. This partition is not mounted from a disk. + Instead, all files are created in the RAM memory, so hopefully this will improve performance at the cost of a much + higher RAM consumption. Default is False. + +* MESH_MASK + Custom file to use instead of the corresponding mesh mask file. + +* NEW_MASK_GLO + Custom file to use instead of the corresponding new mask glo file + +* MASK_REGIONS + Custom file to use instead of the corresponding 2D regions file + +* MASK_REGIONS_3D + Custom file to use instead of the corresponding 3D regions file + + EXPERIMENT ---------- @@ -117,6 +139,9 @@ This sections contains options related to the experiment's definition or configu * CHUNKS Number of chunks to run +* CHUNK_LIST + List of chunks to run. If empty, all diagnostics will be applied to all chunks + * CALENDAR Calendar to use for date calculation. All calendars supported by Autosubmit are available. Default is 'standard' @@ -213,6 +238,17 @@ cmorized files. * SOURCE Default value is 'to be filled' +* VERSION + Dataset version to use (not present in all conventions) + +* DEFAULT_OCEAN_GRID + Name of the default ocean grid for those conventions that require it (CMIP6 and PRIMAVERA). Default is gn. + +* DEFAULT_ATMOS_GRID + Name of the default atmos grid for those conventions that require it (CMIP6 and PRIMAVERA). Default is gr. + +* ACTIVITY + Name of the activity. Default is CMIP THREDDS ------- diff --git a/doc/source/tips.rst b/doc/source/tips.rst index b9f886619a5d7e32063ea7f951a7d3aa4ff8ea72..2ef743657bb699f882fd0862e416a8a3f8e46299 100644 --- a/doc/source/tips.rst +++ b/doc/source/tips.rst @@ -16,15 +16,16 @@ system, the diagnostics will always use the number of cores that you reserved. I system, the diagnostics will try to use all the cores on the machine. To avoid this, add the MAX_CORES parameter to the DIAGNOSTICS section inside the diags.conf file that you are using. -NEMO files ----------- +Cleaning temp file +------------------ + +By default, EarthDiagnostics removes the temporary directory after execution. This behaviour can be avoided be setting +ra -Unlike the bash version of the ocean diagnostics, this program keeps the NEMO files in the scratch folder so you can -launch different configurations for the same experiment with reduced start time. You will need to remove the experiment's -folder in the scratch directory at the end of the experiment to avoid wasting resources. To do this, just use +By default .. code-block:: bash earthdiags -f PATH_TO_CONF --clean -If you plan to run the earthdiagnostics only once, you can add this line after the execution \ No newline at end of file + diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index c2b99900d72bea39a2fe0f65ac5a2ccf4811c0f6..52d071c54ead176430a7a1b0aa6cbf3c6acbe62f 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -37,8 +37,8 @@ Creating a config file ---------------------- Go to the folder where you installed the EarthDiagnostics. You will see a folder called earthdiagnostics, -and, inside it, a diags.conf file that can be used as a model for your config file. Create a copy of it wherever it -suites you. +and, inside it, the model_diags.conf file that can be used as a template for your config file. Create a copy of it +wherever it suites you. Now open your brand new copy with your preferred text editor. The file contains commentaries explaining each one of its options, so read it carefully and edit whatever you need. Don't worry about DIAGS option, we will @@ -53,21 +53,20 @@ whichever suits you better. From now on, we will assume that you are going to ru .. hint:: For old Ocean Diagnostics users: you can use most of the old names as aliases to launch one or multiple diagnostics. - Check the ALIAS section on the diags.conf to see which ones are available. + Check the ALIAS section on the model_diags.conf to see which ones are available. -First, choose a variable that has daily data. Then replace the DIAGS option with the next one where $VARIABLE represents the -variable's name and $DOMAIN its domain (atmos, ocean, seaice, landice...) +First, choose a variable that has daily data. Then replace the DIAGS option with the next one where $VARIABLE represents +the variable's name and $DOMAIN its domain (atmos, ocean, seaice, landice...) .. code-block:: sh - DIAGS = monmean,$VARIABLE,$DOMAIN + DIAGS = monmean,$DOMAIN,$VARIABLE Prepare the run script ---------------------- -Once you have configured your experiment you can execute any diagnostic with the provided launch_diags.sh script. -Create a copy and change the variables PATH_TO_CONF_FILE and PATH_TO_DIAGNOSTICS so they point to your conf file and -installation folder. +Once you have configured your experiment you can execute any diagnostic with the provided model_launch_diags.sh script. +Create a copy and change the variable PATH_TO_CONF_FILE so it points to your conf file . Now, execute the script (or submit it to bsceslogin01, it has the correct header) and... that's it! You will find your results directly on the storage and a folder for the temp files in the scratch named after the EXPID. diff --git a/earthdiagnostics/box.py b/earthdiagnostics/box.py index 68b24bca4beeb0f9826fecadfb7e0b5cf0c5d4c3..f3dc76693e3130c51c453294f593561cc28cbbec 100644 --- a/earthdiagnostics/box.py +++ b/earthdiagnostics/box.py @@ -153,10 +153,10 @@ class Box(object): else: suffix = '' - string = str(abs(self.min_depth)) + suffix + string = str(abs(self.max_depth)) + suffix if self.min_depth != self.max_depth: - string += '-' + str(abs(self.max_depth)) + suffix + string = '{0}-{1}'.format(str(abs(self.min_depth)), string) return string diff --git a/earthdiagnostics/cdftools.py b/earthdiagnostics/cdftools.py index df2b4bf5176071d61e2e4d29a37ed59982ad5649..36fffdaacce1d61d790900f21e8c75d626c7ebf9 100644 --- a/earthdiagnostics/cdftools.py +++ b/earthdiagnostics/cdftools.py @@ -28,7 +28,7 @@ class CDFTools(object): :param output: output file. Not all tools support this parameter :type options: str :param options: options for the tool. - :type options: str | list[str] | Tuple[str] + :type options: str | [str] | Tuple[str] | NoneType :param log_level: log level at which the output of the cdftool command will be added :type log_level: int :param input_option: option to add before input file @@ -60,7 +60,7 @@ class CDFTools(object): @staticmethod def _check_output_was_created(line, output): if output: - if not os.path.exists(output): + if not os.path.isfile(output): raise Exception('Error executing {0}\n Output file not created', ' '.join(line)) # noinspection PyShadowingBuiltins @@ -69,12 +69,12 @@ class CDFTools(object): if input: if isinstance(input, six.string_types): line.append(input) - if not os.path.exists(input): + if not os.path.isfile(input): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, input) else: for element in input: line.append(element) - if not os.path.exists(element): + if not os.path.isfile(element): raise ValueError('Error executing {0}\n Input file {1} file does not exist', command, element) # noinspection PyMethodMayBeStatic @@ -91,4 +91,4 @@ class CDFTools(object): exe_file = os.path.join(path, command) if self.is_exe(exe_file): return - raise ValueError('Error executing {0}\n Command does not exist in {1}', command, self.path) + raise ValueError('Error executing {0}\n Command does not exist in {1}'.format(command, self.path)) diff --git a/earthdiagnostics/cmorizer.py b/earthdiagnostics/cmorizer.py index 593180f780eb8038b7dff575f0254da0fb92f0b3..7cf8c5c354410fc4024f7d73ea2f605f1ecb00f9 100644 --- a/earthdiagnostics/cmorizer.py +++ b/earthdiagnostics/cmorizer.py @@ -1,19 +1,18 @@ # coding=utf-8 import glob +import os +import pygrib import shutil import uuid - -import os from datetime import datetime -import pygrib -from bscearth.utils.log import Log from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months +from bscearth.utils.log import Log +from earthdiagnostics.datafile import NetCDFFile from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import TempFile, Utils -from earthdiagnostics.variable import VariableManager class Cmorizer(object): @@ -277,13 +276,12 @@ class Cmorizer(object): Log.info('Unpacking... ') # remap on regular Gauss grid if grid == 'SH': - Utils.cdo.splitparam(input='-sp2gpl {0}'.format(full_file), output=gribfile + '_', - options='-f nc4') + Utils.cdo.splitparam(input='-sp2gpl {0}'.format(full_file), output=gribfile + '_', options='-f nc4') else: Utils.cdo.splitparam(input=full_file, output=gribfile + '_', options='-R -f nc4') # total precipitation (remove negative values) - Utils.cdo.setcode(228, input='-setmisstoc,0 -setvrange,0,Inf -add ' - '{0}_{{142,143}}.128.nc'.format(gribfile), + Utils.cdo.setcode(228, + input='-setmisstoc,0 -setvrange,0,Inf -add {0}_{{142,143}}.128.nc'.format(gribfile), output='{0}_228.128.nc'.format(gribfile)) Utils.remove_file('ICM') @@ -397,7 +395,7 @@ class Cmorizer(object): :param variable: variable's name :type variable: str """ - alias, var_cmor = VariableManager().get_variable_and_alias(variable) + alias, var_cmor = self.config.var_manager.get_variable_and_alias(variable) if var_cmor is None: return @@ -422,9 +420,26 @@ class Cmorizer(object): raise CMORException('Variable {0}:{1} can not be cmorized. Original filename does not match a recognized ' 'pattern'.format(var_cmor.domain, var_cmor.short_name)) - self.data_manager.send_file(temp, var_cmor.domain, var_cmor.short_name, self.startdate, self.member, - frequency=frequency, rename_var=variable, date_str=date_str, region=region, - move_old=True, grid=alias.grid, cmorized=True) + netcdf_file = NetCDFFile() + netcdf_file.data_manager = self.data_manager + netcdf_file.local_file = temp + netcdf_file.remote_file = self.data_manager.get_file_path(self.startdate, self.member, + var_cmor.domain, var_cmor.short_name, var_cmor, + None, frequency, + grid=alias.grid, year=None, date_str=date_str) + + netcdf_file.data_convention = self.config.data_convention + netcdf_file.region = region + + netcdf_file.frequency = frequency + netcdf_file.domain = var_cmor.domain + netcdf_file.var = var_cmor.short_name + netcdf_file.final_name = var_cmor.short_name + + netcdf_file.prepare_to_upload(rename_var=variable) + netcdf_file.add_cmorization_history() + netcdf_file.upload() + if region: region_str = ' (Region {})'.format(region) else: @@ -433,7 +448,7 @@ class Cmorizer(object): def get_date_str(self, file_path): file_parts = os.path.basename(file_path).split('_') - if file_parts[0] in (self.experiment.expid, 'MMA', 'MMASH', 'MMAGG', 'MMO') or file_parts[0].startswith('ORCA'): + if file_parts[0] in (self.experiment.expid, 't00o', 'MMA', 'MMASH', 'MMAGG', 'MMO') or file_parts[0].startswith('ORCA'): # Model output if file_parts[-1].endswith('.tar'): file_parts = file_parts[-1][0:-4].split('-') @@ -645,12 +660,15 @@ class Cmorizer(object): gribfiles = glob.glob(grb_path) return len(gribfiles) > 0 - def cmorization_required(self, chunk, domain): + def cmorization_required(self, chunk, domains): if not self.config.cmor.chunk_cmorization_requested(chunk): return False if self.config.cmor.force: return True - return not self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain) + for domain in domains: + if self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain): + return False + return True class CMORException(Exception): diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index 2aef7baaef986ec821c2e6e2655b2b207684506c..58ce7e2307b2ff418d670df3c1c0bf721e1be1d6 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -1,13 +1,15 @@ # coding=utf-8 import glob +import os from datetime import datetime -import os -from bscearth.utils.log import Log from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, previous_day +from bscearth.utils.log import Log +from datafile import StorageStatus +from diagnostic import Diagnostic from earthdiagnostics.cmorizer import Cmorizer -from earthdiagnostics.datamanager import DataManager, NetCDFFile +from earthdiagnostics.datamanager import DataManager from earthdiagnostics.frequency import Frequencies, Frequency from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.utils import TempFile, Utils @@ -45,6 +47,7 @@ class CMORManager(DataManager): raise Exception('Can not find model data') self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') + # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN, possible_versions=None): cmor_var = self.variable_list.get_variable(var) @@ -67,11 +70,11 @@ class CMORManager(DataManager): pass return False - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VariableType.MEAN): + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + :param vartype: :param domain: CMOR domain :type domain: Domain :param var: variable name @@ -88,8 +91,6 @@ class CMORManager(DataManager): :type box: Box :param frequency: file's frequency (only needed if it is different from the default) :type frequency: Frequency|NoneType - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType :return: path to the copy created on the scratch folder :rtype: str """ @@ -97,14 +98,124 @@ class CMORManager(DataManager): var = self._get_final_var_name(box, var) filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid, None, None) - temp_path = TempFile.get() - Utils.copy_file(filepath, temp_path) - return temp_path + return self._get_file_from_storage(filepath) + + def request_year(self, diagnostic, domain, var, startdate, member, year, grid=None, box=None, frequency=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param year: + :param diagnostic: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :rtype: str + """ + + job = MergeYear(self, domain, var, startdate, member, year, grid, box, frequency) + job.request_data() + job.declare_data_generated() + if not job.year_file.job_added: + diagnostic.subjobs.append(job) + job.year_file.job_added = True + return job.year_file + + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param region: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + if not frequency: + frequency = self.config.frequency + original_name = var + cmor_var = self.variable_list.get_variable(var) + if cmor_var: + var = cmor_var.short_name + final_name = self._get_final_var_name(box, var) + + filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, chunk, frequency, grid) + netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, + region, diagnostic, grid, vartype, original_name) + netcdf_file.frequency = frequency + return netcdf_file + + def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param year: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + original_name = var + cmor_var = self.variable_list.get_variable(var) + if cmor_var: + var = cmor_var.short_name + final_name = self._get_final_var_name(box, var) + + filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, None, Frequencies.yearly, grid, + year=year) + netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, + None, diagnostic, grid, vartype, original_name) + netcdf_file.frequency = Frequencies.yearly + return netcdf_file def get_file_path(self, startdate, member, domain, var, cmor_var, chunk, frequency, grid=None, year=None, date_str=None): """ Returns the path to a concrete file + :param cmor_var: :param startdate: file's startdate :type startdate: str :param member: file's member @@ -114,7 +225,7 @@ class CMORManager(DataManager): :param var: file's var :type var: var :param chunk: file's chunk - :type chunk: int + :type chunk: int|NoneType :param frequency: file's frequency :type frequency: Frequency :param grid: file's grid @@ -216,6 +327,7 @@ class CMORManager(DataManager): """ Creates the link of a given file from the CMOR repository. + :param cmor_var: :param move_old: :param date_str: :param year: if frequency is yearly, this parameter is used to give the corresponding year @@ -246,147 +358,7 @@ class CMORManager(DataManager): frequency = self.config.frequency filepath = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency, grid=grid, year=str(year), date_str=date_str) - self._create_link(domain, filepath, frequency, var, grid, move_old, vartype) - - def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, - box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False, vartype=VariableType.MEAN): - """ - Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge - with already existing ones as needed - - :param move_old: if true, moves files following older conventions that may be found on the links folder - :type move_old: bool - :param date_str: exact date_str to use in the cmorized file - :type: str - :param year: if frequency is yearly, this parameter is used to give the corresponding year - :type year: int - :param rename_var: if exists, the given variable will be renamed to the one given by var - :type rename_var: str - :param filetosend: path to the file to send to the CMOR repository - :type filetosend: str - :param region: specifies the region represented by the file. If it is defined, the data will be appended to the - CMOR repository as a new region in the file or will overwrite if region was already present - :type region: str - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param diagnostic: diagnostic used to generate the file - :type diagnostic: Diagnostic - :param cmorized: flag to indicate if file was generated in cmorization process - :type cmorized: bool - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - """ - - if rename_var: - original_name = rename_var - else: - original_name = var - - cmor_var = self.variable_list.get_variable(var) - final_name = self._get_final_var_name(box, var) - - if final_name != original_name: - Utils.rename_variable(filetosend, original_name, final_name) - - if not frequency: - frequency = self.config.frequency - - filepath = self.get_file_path(startdate, member, domain, final_name, cmor_var, chunk, frequency, grid, year, - date_str) - netcdf_file = NetCDFFile(filepath, filetosend, domain, final_name, cmor_var, self.config.data_convention, - region) - netcdf_file.frequency = frequency - if diagnostic: - netcdf_file.add_diagnostic_history(diagnostic) - elif cmorized: - netcdf_file.add_cmorization_history() - else: - raise ValueError('You must provide a diagnostic or set cmorized to true to store data ' - 'using the CMORManager') - netcdf_file.send() - - self._create_link(domain, filepath, frequency, final_name, grid, move_old, vartype) - - def get_year(self, domain, var, startdate, member, year, grid=None, box=None): - """ - Ge a file containing all the data for one year for one variable - :param domain: variable's domain - :type domain: str - :param var: variable's name - :type var: str - :param startdate: startdate to retrieve - :type startdate: str - :param member: member to retrieve - :type member: int - :param year: year to retrieve - :type year: int - :param grid: variable's grid - :type grid: str - :param box: variable's box - :type box: Box - :return: - """ - - chunk_files = list() - for chunk in self.experiment.get_year_chunks(startdate, year): - chunk_files.append(self.get_file(domain, var, startdate, member, chunk, grid=grid, box=box)) - - if len(chunk_files) > 1: - temp = self._merge_chunk_files(chunk_files) - else: - temp = chunk_files[0] - temp2 = self._select_data_of_given_year(temp, year) - os.remove(temp) - return temp2 - - @staticmethod - def _select_data_of_given_year(data_file, year): - temp2 = TempFile.get() - handler = Utils.openCdf(data_file) - times = Utils.get_datetime_from_netcdf(handler) - x = 0 - first_index = None - last_index = None - while x < times.size: - if times[x].year == year: - first_index = x - break - else: - x += 1 - - while x < times.size: - if times[x].year != year: - last_index = x - break - else: - x += 1 - if last_index is None: - last_index = times.size - Utils.nco.ncks(input=data_file, output=temp2, options=['-d time,{0},{1}'.format(first_index, last_index - 1)]) - return temp2 - - @staticmethod - def _merge_chunk_files(chunk_files): - temp = TempFile.get() - Utils.nco.ncrcat(input=' '.join(chunk_files), output=temp) - for chunk_file in chunk_files: - os.remove(chunk_file) - return temp + self.create_link(domain, filepath, frequency, var, grid, move_old, vartype) # noinspection PyPep8Naming def prepare(self): @@ -406,46 +378,53 @@ class CMORManager(DataManager): if not self._unpack_cmor_files(startdate, member): self._cmorize_member(startdate, member) - def is_cmorized(self, startdate, member, chunk): + def is_cmorized(self, startdate, member, chunk, domain): identifier = (startdate, member, chunk) if identifier not in self._dic_cmorized: - self._dic_cmorized[identifier] = self._is_cmorized(startdate, member, chunk) - if self._dic_cmorized[identifier]: - return True - return False + self._dic_cmorized[identifier] = {} + self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain) + elif domain not in self._dic_cmorized[identifier]: + self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain) + return self._dic_cmorized[identifier][domain] - def _is_cmorized(self, startdate, member, chunk): + def _is_cmorized(self, startdate, member, chunk, domain): startdate_path = self._get_startdate_path(startdate) if not os.path.isdir(startdate_path): return False + count = 0 if self.config.data_convention == 'specs': for freq in os.listdir(startdate_path): - for domain in (ModelingRealms.ocean, ModelingRealms.ocnBgchem, ModelingRealms.ocnBgchem, - ModelingRealms.atmos): - domain_path = os.path.join(startdate_path, freq, - domain.name) - if os.path.isdir(domain_path): - for var in os.listdir(domain_path): - cmor_var = self.variable_list.get_variable(var, True) - var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, - Frequency(freq)) - if os.path.isfile(var_path): + domain_path = os.path.join(startdate_path, freq, + domain.name) + if os.path.isdir(domain_path): + for var in os.listdir(domain_path): + cmor_var = self.variable_list.get_variable(var, True) + var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, + Frequency(freq)) + if os.path.isfile(var_path): + count += 1 + if count >= self.config.cmor.min_cmorized_vars: return True + else: + continue else: member_path = os.path.join(startdate_path, self._get_member_str(member)) if not os.path.isdir(member_path): return False - for table, domain, freq in (('Amon', ModelingRealms.atmos, Frequencies.monthly), - ('Omon', ModelingRealms.ocean, Frequencies.monthly), - ('SImon', ModelingRealms.seaIce, Frequencies.monthly)): - table_dir = os.path.join(member_path, table) - if not os.path.isdir(table_dir): - continue - for var in os.listdir(table_dir): - cmor_var = self.variable_list.get_variable(var, True) - var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq) - if os.path.isfile(var_path): + freq = Frequencies.monthly + table = domain.get_table(freq, self.config.data_convention) + table_dir = os.path.join(member_path, table.name) + if not os.path.isdir(table_dir): + return False + for var in os.listdir(table_dir): + cmor_var = self.variable_list.get_variable(var, True) + var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq) + if os.path.isfile(var_path): + count += 1 + if count >= self.config.cmor.min_cmorized_vars: return True + else: + continue return False def _cmorize_member(self, startdate, member): @@ -465,7 +444,8 @@ class CMORManager(DataManager): cmorized = False if not self.config.cmor.force_untar: - while self.is_cmorized(startdate, member, chunk): + while self.is_cmorized(startdate, member, chunk, ModelingRealms.atmos) or \ + self.is_cmorized(startdate, member, chunk, ModelingRealms.ocean): chunk += 1 while self._unpack_chunk(startdate, member, chunk): @@ -496,7 +476,7 @@ class CMORManager(DataManager): if self.config.cmor.chunk_cmorization_requested(chunk): Log.info('Unpacking cmorized data for {0} {1} {2}...', startdate, member, chunk) Utils.untar(filepaths, self.cmor_path) - self._correct_paths(startdate, member) + self._correct_paths(startdate) self.create_links(startdate, member) return True return False @@ -515,25 +495,25 @@ class CMORManager(DataManager): filepaths += glob.glob(os.path.join(tar_original_files, 'outputs', file_name)) return filepaths - def _correct_paths(self, startdate, member): + def _correct_paths(self, startdate): self._remove_extra_output_folder() - self._fix_model_as_experiment_error(startdate, member) + self._fix_model_as_experiment_error(startdate) - def _fix_model_as_experiment_error(self, startdate, member): + def _fix_model_as_experiment_error(self, startdate): if self.experiment.experiment_name != self.experiment.model: bad_path = os.path.join(self.cmor_path, self.experiment.institute, self.experiment.model, self.experiment.model) Log.debug('Correcting double model appearance') for (dirpath, dirnames, filenames) in os.walk(bad_path, False): - for filename in filenames: if '_S{0}_'.format(startdate) in filename: continue filepath = os.path.join(dirpath, filename) - good = filepath.replace('_{0}_output_'.format(self.experiment.model), - '_{0}_{1}_S{2}_'.format(self.experiment.model, - self.experiment.experiment_name, - startdate)) + good = filepath + good = good.replace('_{0}_output_'.format(self.experiment.model), + '_{0}_{1}_S{2}_'.format(self.experiment.model, + self.experiment.experiment_name, + startdate)) good = good.replace('/{0}/{0}'.format(self.experiment.model), '/{0}/{1}'.format(self.experiment.model, @@ -568,12 +548,12 @@ class CMORManager(DataManager): for name in os.listdir(os.path.join(path, freq, domain, var, member)): filepath = os.path.join(path, freq, domain, var, member, name) if os.path.isfile(filepath): - self._create_link(domain, filepath, frequency, var, "", False, - vartype=VariableType.MEAN) + self.create_link(domain, filepath, frequency, var, "", False, + vartype=VariableType.MEAN) else: for filename in os.listdir(filepath): - self._create_link(domain, os.path.join(filepath, filename), frequency, var, "", - False, vartype=VariableType.MEAN) + self.create_link(domain, os.path.join(filepath, filename), frequency, var, "", + False, vartype=VariableType.MEAN) Log.debug('Links ready') def _get_startdate_path(self, startdate): @@ -604,3 +584,81 @@ class CMORManager(DataManager): return template.format(member + 1 - self.experiment.member_count_start, self.config.cmor.initialization_number) + +class MergeYear(Diagnostic): + @classmethod + def generate_jobs(cls, diags, options): + pass + + def __init__(self, data_manager, domain, var, startdate, member, year, grid=None, box=None, frequency=None): + super(MergeYear, self).__init__(data_manager) + self.chunk_files = [] + self.experiment = self.data_manager.experiment + self.domain = domain + self.var = var + self.startdate = startdate + self.member = member + self.year = year + self.grid = grid + self.box = box + self.frequency = frequency + + def request_data(self): + for chunk in self.experiment.get_year_chunks(self.startdate, self.year): + self.chunk_files.append(self.request_chunk(self.domain, self.var, self.startdate, self.member, chunk, + grid=self.grid, box=self.box, frequency=self.frequency)) + + def declare_data_generated(self): + self.year_file = self.declare_year(self.domain, self.var, self.startdate, self.member, self.year, + grid=self.grid, box=self.box) + self.year_file.storage_status = StorageStatus.NO_STORE + + def compute(self): + temp = self._merge_chunk_files() + temp2 = self._select_data_of_given_year(temp) + self.year_file.set_local_file(temp2) + + def _select_data_of_given_year(self, data_file): + temp2 = TempFile.get() + handler = Utils.openCdf(data_file) + times = Utils.get_datetime_from_netcdf(handler) + x = 0 + first_index = None + last_index = None + while x < times.size: + if times[x].year == self.year: + first_index = x + break + else: + x += 1 + + while x < times.size: + if times[x].year != self.year: + last_index = x + break + else: + x += 1 + if last_index is None: + last_index = times.size + Utils.nco.ncks(input=data_file, output=temp2, options=['-d time,{0},{1}'.format(first_index, last_index - 1)]) + return temp2 + + def _merge_chunk_files(self): + temp = TempFile.get() + if len(self.chunk_files) == 1: + Utils.copy_file(self.chunk_files[0].local_file, temp) + return temp + + Utils.nco.ncrcat(input=' '.join(self.chunk_files), output=temp) + for chunk_file in self.chunk_files: + os.remove(chunk_file) + return temp + + def __str__(self): + return 'Create year CMOR file Startdate: {0.startdate} Member: {0.member} Year: {0.year} ' \ + 'Variable: {0.domain}:{0.var} Grid: {0.grid} Box: {0.box}'.format(self) + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.year == other.year and\ + self.domain == other.domain and self.var == other.var and self.grid == other.grid and \ + self.box == other.box diff --git a/earthdiagnostics/config.py b/earthdiagnostics/config.py index 1926bc81c5abc1104c6a8b35420daf4391f692a0..133bd6daa4945a79fd7a8c97c8f738cd05e2cfd7 100644 --- a/earthdiagnostics/config.py +++ b/earthdiagnostics/config.py @@ -2,15 +2,19 @@ import os import six -from bscearth.utils.log import Log -from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str from bscearth.utils.config_parser import ConfigParser +from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, date2str +from bscearth.utils.log import Log from earthdiagnostics.frequency import Frequency, Frequencies from earthdiagnostics.variable import VariableManager from modelingrealm import ModelingRealm +class ConfigException(Exception): + pass + + class Config(object): """ Class to read and manage the configuration @@ -20,12 +24,14 @@ class Config(object): """ def __init__(self, path): + parser = ConfigParser() parser.optionxform = str parser.read(path) # Read diags config - self.data_adaptor = parser.get_choice_option('DIAGNOSTICS', 'DATA_ADAPTOR', ('CMOR', 'THREDDS'), 'CMOR') + self.data_adaptor = parser.get_choice_option('DIAGNOSTICS', 'DATA_ADAPTOR', ('CMOR', 'THREDDS', 'OBSRECON'), + 'CMOR') "Scratch folder path" self.scratch_dir = parser.get_path_option('DIAGNOSTICS', 'SCRATCH_DIR') "Scratch folder path" @@ -57,7 +63,8 @@ class Config(object): self.data_convention = parser.get_choice_option('DIAGNOSTICS', 'DATA_CONVENTION', ('specs', 'primavera', 'cmip6', 'preface'), 'specs', ignore_case=True) - VariableManager().load_variables(self.data_convention) + self.var_manager = VariableManager() + self.var_manager.load_variables(self.data_convention) self._diags = parser.get_option('DIAGNOSTICS', 'DIAGS') self.frequency = Frequency(parser.get_option('DIAGNOSTICS', 'FREQUENCY')) "Default data frequency to be used by the diagnostics" @@ -66,6 +73,10 @@ class Config(object): "Path to CDFTOOLS executables" self.max_cores = parser.get_int_option('DIAGNOSTICS', 'MAX_CORES', 0) "Maximum number of cores to use" + self.parallel_downloads = parser.get_int_option('DIAGNOSTICS', 'PARALLEL_DOWNLOADS', 1) + "Maximum number of simultaneous downloads" + self.parallel_uploads = parser.get_int_option('DIAGNOSTICS', 'PARALLEL_UPLOADS', 1) + "Maximum number of simultaneous uploads" self.restore_meshes = parser.get_bool_option('DIAGNOSTICS', 'RESTORE_MESHES', False) "If True, forces the tool to copy all the mesh and mask files for the model, regardless of existence" @@ -96,7 +107,7 @@ class Config(object): self.scratch_dir = os.path.join(self.scratch_dir, 'diags', self.experiment.expid) - self.cmor = CMORConfig(parser) + self.cmor = CMORConfig(parser, self.var_manager) self.thredds = THREDDSConfig(parser) self.report = ReportConfig(parser) @@ -111,7 +122,7 @@ class Config(object): class CMORConfig(object): - def __init__(self, parser): + def __init__(self, parser, var_manager): self.force = parser.get_bool_option('CMOR', 'FORCE', False) self.force_untar = parser.get_bool_option('CMOR', 'FORCE_UNTAR', False) self.filter_files = parser.get_option('CMOR', 'FILTER_FILES', '') @@ -131,23 +142,27 @@ class CMORConfig(object): self.default_ocean_grid = parser.get_option('CMOR', 'DEFAULT_OCEAN_GRID', 'gn') self.default_atmos_grid = parser.get_option('CMOR', 'DEFAULT_ATMOS_GRID', 'gr') self.activity = parser.get_option('CMOR', 'ACTIVITY', 'CMIP') + self.min_cmorized_vars = parser.get_int_option('CMOR', 'MIN_CMORIZED_VARS', 10) vars_string = parser.get_option('CMOR', 'VARIABLE_LIST', '') - var_manager = VariableManager() + self.var_manager = var_manager if vars_string: self._variable_list = list() for domain_var in vars_string.split(' '): if domain_var.startswith('#'): break splitted = domain_var.split(':') - cmor_var = var_manager.get_variable(splitted[1], silent=True) + cmor_var = self.var_manager.get_variable(splitted[1], silent=True) if not cmor_var: + Log.warning('Variable {0} not recognized. It will not be cmorized', domain_var) continue if ModelingRealm(splitted[0]) != cmor_var.domain: Log.warning('Domain {0} for variable {1} is not correct: is {2}', splitted[0], cmor_var.short_name, cmor_var.domain) - return + continue self._variable_list.append('{0.domain}:{0.short_name}'.format(cmor_var)) + if len(self._variable_list) == 0: + raise ConfigException('Variable list value is specified, but no variables were found') else: self._variable_list = None @@ -173,7 +188,7 @@ class CMORConfig(object): if self._variable_list is None: return True for var in variables: - if self.cmorize(VariableManager().get_variable(var, silent=True)): + if self.cmorize(self.var_manager.get_variable(var, silent=True)): return True return False @@ -216,7 +231,7 @@ class CMORConfig(object): return self._var_daily elif frequency == Frequencies.monthly: return self._var_monthly - raise Exception('Frequency not recognized: {0}'.format(frequency)) + raise ValueError('Frequency not recognized: {0}'.format(frequency)) def get_levels(self, frequency, variable): return self.get_variables(frequency)[variable] @@ -260,7 +275,14 @@ class ExperimentConfig(object): members.append(int(mem)) self.members = members - self.startdates = parser.get_option('EXPERIMENT', 'STARTDATES').split() + startdates = parser.get_list_option('EXPERIMENT', 'STARTDATES') + + import exrex + self.startdates = [] + for startdate_pattern in startdates: + for startdate in exrex.generate(startdate_pattern): + self.startdates.append(startdate) + self.chunk_size = parser.get_int_option('EXPERIMENT', 'CHUNK_SIZE') self.num_chunks = parser.get_int_option('EXPERIMENT', 'CHUNKS') self.chunk_list = parser.get_int_list_option('EXPERIMENT', 'CHUNK_LIST', []) @@ -323,6 +345,7 @@ class ExperimentConfig(object): return chunks def get_chunk_start(self, startdate, chunk): + # noinspection PyTypeChecker if isinstance(startdate, six.string_types): startdate = parse_date(startdate) return chunk_start_date(startdate, chunk, self.chunk_size, 'month', self.calendar) @@ -356,7 +379,7 @@ class ExperimentConfig(object): first_january += 1 years = list() - for chunk in range(first_january, self.num_chunks - chunks_per_year, chunks_per_year): + for chunk in range(first_january, chunks_per_year, self.num_chunks): years.append(first_year) first_year += 1 return years diff --git a/earthdiagnostics/constants.py b/earthdiagnostics/constants.py index 88da0afafacf839ba7fa6a676c87a51d5256b352..c1e51013879b1417bbb0960b3963bc949a2d691d 100644 --- a/earthdiagnostics/constants.py +++ b/earthdiagnostics/constants.py @@ -189,6 +189,8 @@ class Models(object): """ EC-Earth 3.1 ORCA0.25 L75 """ ECEARTH_3_2_O1L75 = 'Ec3.2_O1L75' """ EC-Earth 3.2 ORCA1 L75 """ + ECEARTH_3_2_O25L75 = 'Ec3.2_O25L75' + """ EC-Earth 3.2 ORCA0.25 L75 """ NEMO_3_2_O1L42 = 'N3.2_O1L42' """ NEMO 3.2 ORCA1 L42 """ diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py new file mode 100644 index 0000000000000000000000000000000000000000..ddaf1b102e54e16dc8bce3e3faaf5cbd3f4adb50 --- /dev/null +++ b/earthdiagnostics/datafile.py @@ -0,0 +1,475 @@ +# coding: utf-8 +import csv +import os +import shutil +from datetime import datetime + +import numpy as np +from bscearth.utils.log import Log + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile +from publisher import Publisher +from variable_type import VariableType + + +class LocalStatus(object): + PENDING = 0 + DOWNLOADING = 1 + READY = 2 + FAILED = 3 + NOT_REQUESTED = 4 + COMPUTING = 5 + + +class StorageStatus(object): + PENDING = 0 + UPLOADING = 1 + READY = 2 + FAILED = 3 + NO_STORE = 4 + + +class DataFile(Publisher): + + def __init__(self): + super(DataFile, self).__init__() + self.remote_file = None + self.local_file = None + self.domain = None + self.var = None + self.cmor_var = None + self.region = None + self.frequency = None + self.data_convention = None + self.diagnostic = None + self.grid = None + self.data_manager = None + self.final_name = None + self.var_type = VariableType.MEAN + self._local_status = LocalStatus.NOT_REQUESTED + self._storage_status = StorageStatus.READY + self.job_added = False + self._modifiers = [] + + def __str__(self): + return 'Data file for {0}'.format(self.remote_file) + + def unsubscribe(self, who): + super(DataFile, self).unsubscribe(who) + self._clean_local() + + @property + def size(self): + if self.local_status == LocalStatus.READY: + os.path.getsize(self.local_file) + return None + + def _clean_local(self): + if self.local_status != LocalStatus.READY or len(self.suscribers) > 0 or self.upload_required(): + return + Log.debug('File {0} no longer needed. Deleting from scratch...'.format(self.remote_file)) + os.remove(self.local_file) + Log.debug('File {0} deleted from scratch'.format(self.remote_file)) + self.local_file = None + self.local_status = LocalStatus.PENDING + + def upload_required(self): + return self.local_status == LocalStatus.READY and self.storage_status == StorageStatus.PENDING + + def download_required(self): + if not self.local_status == LocalStatus.PENDING: + return False + + if self.storage_status == StorageStatus.READY: + return True + + if self.has_modifiers(): + return True + + def add_modifier(self, diagnostic): + self._modifiers.append(diagnostic) + + def has_modifiers(self): + return len(self._modifiers) > 0 + + def ready_to_run(self, diagnostic): + if not self.local_status == LocalStatus.READY: + return False + if len(self._modifiers) == 0: + return True + return self._modifiers[0] is diagnostic + + @property + def local_status(self): + return self._local_status + + @local_status.setter + def local_status(self, value): + if self._local_status == value: + return + self._local_status = value + self.dispatch(self) + + @property + def storage_status(self): + return self._storage_status + + @storage_status.setter + def storage_status(self, value): + if self._storage_status == value: + return + self._storage_status = value + self.dispatch(self) + + @classmethod + def from_storage(cls, filepath): + file_object = cls() + file_object.remote_file = filepath + file_object.local_status = LocalStatus.PENDING + return file_object + + @classmethod + def to_storage(cls, remote_file): + new_object = cls() + new_object.remote_file = remote_file + new_object.storage_status = StorageStatus.PENDING + return new_object + + def download(self): + raise NotImplementedError('Class must implement the download method') + + def prepare_to_upload(self, rename_var): + Utils.convert2netcdf4(self.local_file) + if rename_var: + original_name = rename_var + else: + original_name = self.var + if self.final_name != original_name: + Utils.rename_variable(self.local_file, original_name, self.final_name) + self._rename_coordinate_variables() + self._correct_metadata() + self._prepare_region() + + self.add_diagnostic_history() + + def upload(self): + self.storage_status = StorageStatus.UPLOADING + try: + Utils.copy_file(self.local_file, self.remote_file, save_hash=True) + except Exception as ex: + Log.error('File {0} can not be uploaded: {1}', self.remote_file, ex) + self.storage_status = StorageStatus.FAILED + return + + Log.info('File {0} uploaded!', self.remote_file) + try: + self.create_link() + except Exception as ex: + Log.warning('Link for file {0} can not be created: {1}', self.remote_file, ex) + self.storage_status = StorageStatus.READY + self._clean_local() + + def set_local_file(self, local_file, diagnostic=None, rename_var=''): + if diagnostic in self._modifiers: + self._modifiers.remove(diagnostic) + self.local_file = local_file + self.prepare_to_upload(rename_var) + self.local_status = LocalStatus.READY + + def create_link(self): + pass + + def _correct_metadata(self): + handler = Utils.openCdf(self.local_file) + var_handler = handler.variables[self.final_name] + coords = set.intersection({'time', 'lev', 'lat', 'lon'}, set(handler.variables.keys())) + var_handler.coordinates = ' '.join(coords) + if not self.cmor_var: + handler.close() + return + + self._fix_variable_name(var_handler) + handler.modeling_realm = self.cmor_var.domain.name + table = self.cmor_var.get_table(self.frequency, self.data_convention) + handler.table_id = 'Table {0} ({1})'.format(table.name, table.date) + if self.cmor_var.units: + self._fix_units(var_handler) + handler.sync() + self._fix_coordinate_variables_metadata(handler) + var_type = var_handler.dtype + handler.close() + self._fix_values_metadata(var_type) + + def _fix_variable_name(self, var_handler): + var_handler.standard_name = self.cmor_var.standard_name + var_handler.long_name = self.cmor_var.long_name + # var_handler.short_name = self.cmor_var.short_name + + def _fix_values_metadata(self, var_type): + if self.cmor_var.valid_min != '': + valid_min = '-a valid_min,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, self.cmor_var.valid_min) + else: + valid_min = '' + if self.cmor_var.valid_max != '': + valid_max = '-a valid_max,{0},o,{1},"{2}" '.format(self.final_name, var_type.char, self.cmor_var.valid_max) + else: + valid_max = '' + Utils.nco.ncatted(input=self.local_file, output=self.local_file, + options=('-O -a _FillValue,{0},o,{1},"1.e20" ' + '-a missingValue,{0},o,{1},"1.e20" {2}{3}'.format(self.final_name, var_type.char, + valid_min, valid_max),)) + + def _fix_coordinate_variables_metadata(self, handler): + if 'lev' in handler.variables: + handler.variables['lev'].short_name = 'lev' + if self.domain == ModelingRealms.ocean: + handler.variables['lev'].standard_name = 'depth' + if 'lon' in handler.variables: + handler.variables['lon'].short_name = 'lon' + handler.variables['lon'].standard_name = 'longitude' + if 'lat' in handler.variables: + handler.variables['lat'].short_name = 'lat' + handler.variables['lat'].standard_name = 'latitude' + + def _fix_units(self, var_handler): + if 'units' not in var_handler.ncattrs(): + return + if var_handler.units == '-': + var_handler.units = '1.0' + if var_handler.units == 'PSU': + var_handler.units = 'psu' + if var_handler.units == 'C' and self.cmor_var.units == 'K': + var_handler.units = 'deg_C' + if self.cmor_var.units != var_handler.units: + self._convert_units(var_handler) + var_handler.units = self.cmor_var.units + + def _convert_units(self, var_handler): + try: + Utils.convert_units(var_handler, self.cmor_var.units) + except ValueError as ex: + Log.warning('Can not convert {3} from {0} to {1}: {2}', var_handler.units, self.cmor_var.units, ex, + self.cmor_var.short_name) + factor, offset = UnitConversion.get_conversion_factor_offset(var_handler.units, + self.cmor_var.units) + + var_handler[:] = var_handler[:] * factor + offset + if 'valid_min' in var_handler.ncattrs(): + var_handler.valid_min = float(var_handler.valid_min) * factor + offset + if 'valid_max' in var_handler.ncattrs(): + var_handler.valid_max = float(var_handler.valid_max) * factor + offset + + def _prepare_region(self): + if not self.region: + return + if not os.path.exists(self.remote_file): + self._add_region_dimension_to_var() + else: + self._update_var_with_region_data() + self._correct_metadata() + Utils.nco.ncks(input=self.local_file, output=self.local_file, options=['--fix_rec_dmn region']) + + def _update_var_with_region_data(self): + temp = TempFile.get() + shutil.copyfile(self.remote_file, temp) + Utils.nco.ncks(input=temp, output=temp, options=['--mk_rec_dmn region']) + handler = Utils.openCdf(temp) + handler_send = Utils.openCdf(self.local_file) + value = handler_send.variables[self.final_name][:] + var_region = handler.variables['region'] + basin_index = np.where(var_region[:] == self.region) + if len(basin_index[0]) == 0: + var_region[var_region.shape[0]] = self.region + basin_index = var_region.shape[0] - 1 + + else: + basin_index = basin_index[0][0] + handler.variables[self.final_name][..., basin_index] = value + handler.close() + handler_send.close() + Utils.move_file(temp, self.local_file) + + def _add_region_dimension_to_var(self): + handler = Utils.openCdf(self.local_file) + handler.createDimension('region') + var_region = handler.createVariable('region', str, 'region') + var_region[0] = self.region + original_var = handler.variables[self.final_name] + new_var = handler.createVariable('new_var', original_var.datatype, + original_var.dimensions + ('region',)) + new_var.setncatts({k: original_var.getncattr(k) for k in original_var.ncattrs()}) + value = original_var[:] + new_var[..., 0] = value + handler.close() + Utils.nco.ncks(input=self.local_file, output=self.local_file, options='-O -x -v {0}'.format(self.final_name)) + Utils.rename_variable(self.local_file, 'new_var', self.final_name) + + def _rename_coordinate_variables(self): + variables = dict() + variables['x'] = 'i' + variables['y'] = 'j' + variables['nav_lat_grid_V'] = 'lat' + variables['nav_lon_grid_V'] = 'lon' + variables['nav_lat_grid_U'] = 'lat' + variables['nav_lon_grid_U'] = 'lon' + variables['nav_lat_grid_T'] = 'lat' + variables['nav_lon_grid_T'] = 'lon' + Utils.rename_variables(self.local_file, variables, False, True) + + def add_diagnostic_history(self): + if not self.diagnostic: + return + from earthdiagnostics.earthdiags import EarthDiags + history_line = 'Diagnostic {1} calculated with EarthDiagnostics version {0}'.format(EarthDiags.version, + self.diagnostic) + self._add_history_line(history_line) + + def add_cmorization_history(self): + from earthdiagnostics.earthdiags import EarthDiags + history_line = 'CMORized with Earthdiagnostics version {0}'.format(EarthDiags.version) + self._add_history_line(history_line) + + def _add_history_line(self, history_line): + utc_datetime = 'UTC ' + datetime.utcnow().isoformat() + history_line = '{0}: {1};'.format(utc_datetime, history_line) + + handler = Utils.openCdf(self.local_file) + try: + history_line = history_line + handler.history + except AttributeError: + history_line = history_line + handler.history = Utils.convert_to_ASCII_if_possible(history_line) + handler.close() + + +class UnitConversion(object): + """ + Class to manage unit conversions + """ + _dict_conversions = None + + @classmethod + def load_conversions(cls): + """ + Load conversions from the configuration file + """ + cls._dict_conversions = dict() + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conversions.csv'), 'rb') as csvfile: + reader = csv.reader(csvfile, dialect='excel') + for line in reader: + if line[0] == 'original': + continue + cls.add_conversion(UnitConversion(line[0], line[1], line[2], line[3])) + + @classmethod + def add_conversion(cls, conversion): + """ + Adds a conversion to the dictionary + + :param conversion: conversion to add + :type conversion: UnitConversion + """ + cls._dict_conversions[(conversion.source, conversion.destiny)] = conversion + + def __init__(self, source, destiny, factor, offset): + self.source = source + self.destiny = destiny + self.factor = float(factor) + self.offset = float(offset) + + @classmethod + def get_conversion_factor_offset(cls, input_units, output_units): + """ + Gets the conversion factor and offset for two units . The conversion has to be done in the following way: + converted = original * factor + offset + + :param input_units: original units + :type input_units: str + :param output_units: destiny units + :type output_units: str + :return: factor and offset + :rtype: [float, float] + """ + units = input_units.split() + if len(units) == 1: + scale_unit = 1 + unit = units[0] + else: + if '^' in units[0]: + values = units[0].split('^') + scale_unit = pow(int(values[0]), int(values[1])) + else: + scale_unit = float(units[0]) + unit = units[1] + + units = output_units.split() + if len(units) == 1: + scale_new_unit = 1 + new_unit = units[0] + else: + if '^' in units[0]: + values = units[0].split('^') + scale_new_unit = pow(int(values[0]), int(values[1])) + else: + scale_new_unit = float(units[0]) + new_unit = units[1] + + factor, offset = UnitConversion._get_factor(new_unit, unit) + if factor is None: + return None, None + factor = factor * scale_unit / float(scale_new_unit) + offset /= float(scale_new_unit) + + return factor, offset + + @classmethod + def _get_factor(cls, new_unit, unit): + # Add only the conversions with a factor greater than 1 + if unit == new_unit: + return 1, 0 + elif (unit, new_unit) in cls._dict_conversions: + conversion = cls._dict_conversions[(unit, new_unit)] + return conversion.factor, conversion.offset + elif (new_unit, unit) in cls._dict_conversions: + conversion = cls._dict_conversions[(new_unit, unit)] + return 1 / conversion.factor, -conversion.offset + else: + return None, None + + +class NetCDFFile(DataFile): + + def download(self): + try: + self.local_status = LocalStatus.DOWNLOADING + Log.debug('Downloading file {0}...', self.remote_file) + if not self.local_file: + self.local_file = TempFile.get() + Utils.get_file_hash(self.remote_file, use_stored=True, save=True) + Utils.copy_file(self.remote_file, self.local_file) + Log.info('File {0} ready!', self.remote_file) + self.local_status = LocalStatus.READY + + except Exception as ex: + if os.path.isfile(self.local_file): + os.remove(self.local_file) + Log.error('File {0} not available: {1}', self.remote_file, ex) + self.local_status = LocalStatus.FAILED + + def create_link(self): + try: + self.data_manager.create_link(self.domain, self.remote_file, self.frequency, self.final_name, + self.grid, True, self.var_type) + except Exception as ex: + Log.error('Can not create link to {1}: {0}'.format(ex, self.remote_file)) + + @property + def size(self): + if self.local_status == LocalStatus.READY: + return os.path.getsize(self.local_file) + if self.storage_status == StorageStatus.READY: + return os.path.getsize(self.remote_file) + return None + + diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 6eb172779dc473b9c2aa496fab88f96d9655d1eb..e3799d31ef13dc1ac1aaf7396ee3cbb1e8914f6d 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -1,18 +1,14 @@ # coding: utf-8 import csv -import shutil -import threading -from datetime import datetime - -import numpy as np import os import re -from bscearth.utils.log import Log +import shutil +import threading -from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Variable, VariableManager -from earthdiagnostics.variable_type import VariableType +from earthdiagnostics.datafile import NetCDFFile as NCfile, StorageStatus, LocalStatus from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils +from earthdiagnostics.variable_type import VariableType class DataManager(object): @@ -26,131 +22,35 @@ class DataManager(object): self.config = config self.experiment = config.experiment self._checked_vars = list() - self.variable_list = VariableManager() + self.variable_list = config.var_manager UnitConversion.load_conversions() self.lock = threading.Lock() - - def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VariableType.MEAN): - """ - Checks if a given file exists - - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str - """ - raise NotImplementedError() - - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VariableType.MEAN): - """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str - """ - raise NotImplementedError() - - def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, - box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False, vartype=VariableType.MEAN): - """ - Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge - with already existing ones as needed - - :param move_old: if true, moves files following older conventions that may be found on the links folder - :type move_old: bool - :param date_str: exact date_str to use in the cmorized file - :type: str - :param year: if frequency is yearly, this parameter is used to give the corresponding year - :type year: int - :param rename_var: if exists, the given variable will be renamed to the one given by var - :type rename_var: str - :param filetosend: path to the file to send to the CMOR repository - :type filetosend: str - :param region: specifies the region represented by the file. If it is defined, the data will be appended to the - CMOR repository as a new region in the file or will overwrite if region was already present - :type region: str - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param diagnostic: diagnostic used to generate the file - :type diagnostic: Diagnostic - :param cmorized: flag to indicate if file was generated in cmorization process - :type cmorized: bool - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - """ - raise NotImplementedError() - - def get_year(self, domain, var, startdate, member, year, grid=None, box=None): - """ - Ge a file containing all the data for one year for one variable - :param domain: variable's domain - :type domain: Domain - :param var: variable's name - :type var: str - :param startdate: startdate to retrieve - :type startdate: str - :param member: member to retrieve - :type member: int - :param year: year to retrieve - :type year: int - :param grid: variable's grid - :type grid: str - :param box: variable's box - :type box: Box - :return: - """ - raise NotImplementedError() + self.requested_files = {} + + def _get_file_from_storage(self, filepath): + if filepath not in self.requested_files: + self.requested_files[filepath] = NCfile.from_storage(filepath) + file_object = self.requested_files[filepath] + file_object.local_satatus = LocalStatus.PENDING + return self.requested_files[filepath] + + def _declare_generated_file(self, remote_file, domain, final_var, cmor_var, data_convention, + region, diagnostic, grid, var_type, original_var): + if remote_file not in self.requested_files: + self.requested_files[remote_file] = NCfile.to_storage(remote_file) + file_object = self.requested_files[remote_file] + file_object.diagnostic = diagnostic + file_object.var_type = var_type + file_object.grid = grid + file_object.data_manager = self + file_object.domain = domain + file_object.var = original_var + file_object.final_name = final_var + file_object.cmor_var = cmor_var + file_object.region = region + file_object.data_convention = data_convention + file_object.storage_status = StorageStatus.PENDING + return file_object @staticmethod def _get_final_var_name(box, var): @@ -162,12 +62,12 @@ class DataManager(object): if grid: var = '{0}-{1}'.format(var, grid) - if domain in [ModelingRealms.ocean, ModelingRealms.seaIce]: + if domain in [ModelingRealms.ocean, ModelingRealms.seaIce, ModelingRealms.ocnBgchem]: return '{0}_f{1}h'.format(var, self.experiment.ocean_timestep) else: return '{0}_f{1}h'.format(var, self.experiment.atmos_timestep) - def _create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): freq_str = frequency.folder_name(vartype) if not grid: @@ -217,7 +117,7 @@ class DataManager(object): Utils.create_folder_tree(os.path.dirname(link_path)) relative_path = os.path.relpath(filepath, os.path.dirname(link_path)) os.symlink(relative_path, link_path) - except: + except Exception: raise finally: self.lock.release() @@ -261,191 +161,32 @@ class DataManager(object): """ pass + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy -class NetCDFFile(object): - """ - Class to manage netCDF file and pr - - :param remote_file: - :type remote_file: str - :param local_file: - :type local_file: str - :param domain: - :type domain: Domain - :param var: - :type var: str - :param cmor_var: - :type cmor_var: Variable - """ - def __init__(self, remote_file, local_file, domain, var, cmor_var, data_convention, region): - self.remote_file = remote_file - self.local_file = local_file - self.domain = domain - self.var = var - self.cmor_var = cmor_var - self.region = region - self.frequency = None - self.data_convention = data_convention - - def send(self): - Utils.convert2netcdf4(self.local_file) - self._correct_metadata() - self._prepare_region() - self._rename_coordinate_variables() - - Utils.move_file(self.local_file, self.remote_file) - Utils.give_group_write_permissions(self.remote_file) - - def _prepare_region(self): - if not self.region: - return - if not os.path.exists(self.remote_file): - self._add_region_dimension_to_var() - else: - self._update_var_with_region_data() - self._correct_metadata() - Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('--fix_rec_dmn region',)) - - def _update_var_with_region_data(self): - temp = TempFile.get() - shutil.copyfile(self.remote_file, temp) - Utils.nco.ncks(input=temp, output=temp, options=('--mk_rec_dmn region',)) - handler = Utils.openCdf(temp) - handler_send = Utils.openCdf(self.local_file) - value = handler_send.variables[self.var][:] - var_region = handler.variables['region'] - basin_index = np.where(var_region[:] == self.region) - if len(basin_index[0]) == 0: - var_region[var_region.shape[0]] = self.region - basin_index = var_region.shape[0] - 1 - - else: - basin_index = basin_index[0][0] - handler.variables[self.var][..., basin_index] = value - handler.close() - handler_send.close() - Utils.move_file(temp, self.local_file) - - def _add_region_dimension_to_var(self): - handler = Utils.openCdf(self.local_file) - handler.createDimension('region') - var_region = handler.createVariable('region', str, 'region') - var_region[0] = self.region - original_var = handler.variables[self.var] - new_var = handler.createVariable('new_var', original_var.datatype, - original_var.dimensions + ('region',)) - new_var.setncatts({k: original_var.getncattr(k) for k in original_var.ncattrs()}) - value = original_var[:] - new_var[..., 0] = value - handler.close() - Utils.nco.ncks(input=self.local_file, output=self.local_file, options=('-x -v {0}'.format(self.var),)) - Utils.rename_variable(self.local_file, 'new_var', self.var) - - def _correct_metadata(self): - if not self.cmor_var: - return - handler = Utils.openCdf(self.local_file) - var_handler = handler.variables[self.var] - self._fix_variable_name(var_handler) - handler.modeling_realm = self.cmor_var.domain.name - table = self.cmor_var.get_table(self.frequency, self.data_convention) - handler.table_id = 'Table {0} ({1})'.format(table.name, table.date) - if self.cmor_var.units: - self._fix_units(var_handler) - handler.sync() - self._fix_coordinate_variables_metadata(handler) - var_type = var_handler.dtype - handler.close() - self._fix_values_metadata(var_type) - - def _fix_variable_name(self, var_handler): - var_handler.standard_name = self.cmor_var.standard_name - var_handler.long_name = self.cmor_var.long_name - var_handler.short_name = self.cmor_var.short_name - - def _fix_values_metadata(self, var_type): - options = ['-a _FillValue,{0},o,{1},"1.e20"'.format(self.var, var_type.char), - '-a missingValue,{0},o,{1},"1.e20"'.format(self.var, var_type.char)] - if self.cmor_var.valid_min: - options.append('-a valid_min,{0},o,{1},"{2}" '.format(self.var, var_type.char, self.cmor_var.valid_min)) - if self.cmor_var.valid_max: - options.append('-a valid_max,{0},o,{1},"{2}" '.format(self.var, var_type.char, self.cmor_var.valid_max)) - Utils.nco.ncatted(input=self.local_file, output=self.local_file, options=options) - - def _fix_coordinate_variables_metadata(self, handler): - if 'lev' in handler.variables: - handler.variables['lev'].short_name = 'lev' - if self.domain == ModelingRealms.ocean: - handler.variables['lev'].standard_name = 'depth' - if 'lon' in handler.variables: - handler.variables['lon'].short_name = 'lon' - handler.variables['lon'].standard_name = 'longitude' - if 'lat' in handler.variables: - handler.variables['lat'].short_name = 'lat' - handler.variables['lat'].standard_name = 'latitude' - - EQUIVALENT_UNITS = {'-': '1.0', 'fractional': '1.0', 'psu': 'psu'} - - def _fix_units(self, var_handler): - if 'units' not in var_handler.ncattrs(): - return - if var_handler.units.lower() in NetCDFFile.EQUIVALENT_UNITS: - var_handler.units = NetCDFFile.EQUIVALENT_UNITS[var_handler.units.lower()] - elif var_handler.units == 'C' or self.cmor_var.units == 'K': - var_handler.units = 'deg_C' - if self.cmor_var.units != var_handler.units: - self._convert_units(var_handler) - var_handler.units = self.cmor_var.units - - def _convert_units(self, var_handler): - try: - Utils.convert_units(var_handler, self.cmor_var.units) - except ValueError as ex: - Log.warning('Can not convert {3} from {0} to {1}: {2}', var_handler.units, self.cmor_var.units, ex, - self.cmor_var.short_name) - factor, offset = UnitConversion.get_conversion_factor_offset(var_handler.units, - self.cmor_var.units) - - var_handler[:] = var_handler[:] * factor + offset - if 'valid_min' in var_handler.ncattrs(): - var_handler.valid_min = float(var_handler.valid_min) * factor + offset - if 'valid_max' in var_handler.ncattrs(): - var_handler.valid_max = float(var_handler.valid_max) * factor + offset - - def _rename_coordinate_variables(self): - variables = dict() - variables['x'] = 'i' - variables['y'] = 'j' - variables['nav_lat_grid_V'] = 'lat' - variables['nav_lon_grid_V'] = 'lon' - variables['nav_lat_grid_U'] = 'lat' - variables['nav_lon_grid_U'] = 'lon' - variables['nav_lat_grid_T'] = 'lat' - variables['nav_lon_grid_T'] = 'lon' - Utils.rename_variables(self.local_file, variables, False, True) - - def add_diagnostic_history(self, diagnostic): - from earthdiagnostics.earthdiags import EarthDiags - history_line = 'Diagnostic {1} calculated with EarthDiagnostics version {0}'.format(EarthDiags.version, - diagnostic) - self._add_history_line(history_line) - - def add_cmorization_history(self): - from earthdiagnostics.earthdiags import EarthDiags - history_line = 'CMORized with Earthdiagnostics version {0}'.format(EarthDiags.version) - self._add_history_line(history_line) - - def _add_history_line(self, history_line): - utc_datetime = 'UTC ' + datetime.utcnow().isoformat() - history_line = '{0}: {1};'.format(utc_datetime, history_line) - - handler = Utils.openCdf(self.local_file) - try: - history_line = history_line + handler.history - except AttributeError: - history_line = history_line - handler.history = Utils.convert_to_ASCII_if_possible(history_line) - handler.close() + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :rtype: str + """ + raise NotImplementedError('Class must override request_chunk method') class UnitConversion(object): diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index 207554ddf6a87ab33c131329c938eb38fcf371e2..4e90373dbbd6e79389ad0d9210c2751bbac1956e 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,12 +1,23 @@ # coding=utf-8 +import datetime + +from datafile import StorageStatus, LocalStatus from earthdiagnostics.constants import Basins, Basin from earthdiagnostics.frequency import Frequency -from earthdiagnostics.variable_type import VariableType from earthdiagnostics.modelingrealm import ModelingRealms -from earthdiagnostics.variable import VariableManager +from earthdiagnostics.variable_type import VariableType +from publisher import Publisher -class Diagnostic(object): +class DiagnosticStatus(object): + WAITING = 0 + READY = 1 + RUNNING = 2 + COMPLETED = 3 + FAILED = 4 + + +class Diagnostic(Publisher): """ Base class for the diagnostics. Provides a common interface for them and also has a mechanism that allows diagnostic retrieval by name. @@ -22,14 +33,33 @@ class Diagnostic(object): _diag_list = dict() def __init__(self, data_manager): + super(Diagnostic, self).__init__() + self._generated_files = [] self.data_manager = data_manager - self.required_vars = [] - self.generated_vars = [] - self.can_run_multiple_instances = True + self._status = DiagnosticStatus.WAITING + self._requests = [] + self.consumed_time = datetime.timedelta() + self.subjobs = [] def __repr__(self): return str(self) + @property + def status(self): + return self._status + + @status.setter + def status(self, value): + if self._status == value: + return + self._status = value + if self.status == DiagnosticStatus.RUNNING: + for generated_file in self._generated_files: + generated_file.local_status = LocalStatus.COMPUTING + if self.status in (DiagnosticStatus.FAILED, DiagnosticStatus.COMPLETED): + self._unsuscribe_requests() + self.dispatch(self) + @staticmethod def register(cls): """ @@ -58,12 +88,34 @@ class Diagnostic(object): return Diagnostic._diag_list[name] return None - def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, - box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - vartype=VariableType.MEAN): + def compute(self): + """ + Calculates the diagnostic and stores the output + + Must be implemented by derived classes + """ + raise NotImplementedError("Class must override compute method") + + def request_data(self): + """ + Calculates the diagnostic and stores the output + + Must be implemented by derived classes + """ + raise NotImplementedError("Class must override request_data method") + + def declare_data_generated(self): + """ + Calculates the diagnostic and stores the output + + Must be implemented by derived classes + """ + raise NotImplementedError("Class must override declare_data_generated method") + + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN): """ - :param filetosend: :param domain: :type domain: ModelingRealm :param var: @@ -73,29 +125,41 @@ class Diagnostic(object): :param grid: :param region: :param box: - :param rename_var: :param frequency: :type frequency: Frequency - :param year: - :param date_str: - :param move_old: :param vartype: Variable type (mean, statistic) :type vartype: VariableType - :return: + :return: datafile object + :rtype: earthdiagnostics.datafile.DataFile """ if isinstance(region, Basin): region = region.name - self.data_manager.send_file(filetosend, domain, var, startdate, member, chunk, grid, region, - box, rename_var, frequency, year, date_str, move_old, diagnostic=self, - vartype=vartype) + generated_chunk = self.data_manager.declare_chunk(domain, var, startdate, member, chunk, grid, region, box, + diagnostic=self, vartype=vartype, frequency=frequency) + self._generated_files.append(generated_chunk) + return generated_chunk - def compute(self): + def declare_year(self, domain, var, startdate, member, year, grid=None, box=None, + vartype=VariableType.MEAN): """ - Calculates the diagnostic and stores the output - Must be implemented by derived classes + :param domain: + :type domain: ModelingRealm + :param var: + :param startdate: + :param member: + :param grid: + :param box: + :param year: + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: datafile object + :rtype: DataFile """ - raise NotImplementedError("Class must override compute method") + generated_year = self.data_manager.declare_year(domain, var, startdate, member, year, grid, box, + diagnostic=self, vartype=vartype) + self._generated_files.append(generated_year) + return generated_year @classmethod def generate_jobs(cls, diags, options): @@ -135,6 +199,48 @@ class Diagnostic(object): """ return 'Developer must override base class __str__ method' + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + to_modify=False, vartype=VariableType.MEAN): + request = self.data_manager.request_chunk(domain, var, startdate, member, chunk, grid, box, frequency, vartype) + if to_modify: + request.add_modifier(self) + self._requests.append(request) + request.subscribe(self, self._updated_request) + return request + + def request_year(self, domain, var, startdate, member, year, grid=None, box=None, frequency=None, to_modify=False): + request = self.data_manager.request_year(self, domain, var, startdate, member, year, grid, box, frequency) + if to_modify: + request.add_modifier(self) + self._requests.append(request) + request.subscribe(self, self._updated_request) + return request + + def _updated_request(self, request): + if self.status != DiagnosticStatus.WAITING: + return + from datafile import LocalStatus + if request.local_status == LocalStatus.FAILED: + self.message = 'Required file {0} is not available'.format(request.remote_file) + self.status = DiagnosticStatus.FAILED + return + + if request.local_status == LocalStatus.READY: + self.check_is_ready() + + def check_is_ready(self): + if all([request.ready_to_run(self) for request in self._requests]): + self.status = DiagnosticStatus.READY + + def _unsuscribe_requests(self): + for request in self._requests: + request.unsubscribe(self) + + def all_requests_in_storage(self): + return self.pending_requests() == 0 + + def pending_requests(self): + return len([request.storage_status != StorageStatus.READY for request in self._requests]) class DiagnosticOption(object): @@ -191,7 +297,9 @@ class DiagnosticListIntOption(DiagnosticOption): def __init__(self, name, default_value=None, min_limit=None, max_limit=None): super(DiagnosticListIntOption, self).__init__(name, default_value) self.min_limit = min_limit + """ Lower limit """ self.max_limit = max_limit + """ Upper limit """ def parse(self, option_value): option_value = self.check_default(option_value) @@ -199,8 +307,10 @@ class DiagnosticListIntOption(DiagnosticOption): return option_value values = [int(i) for i in option_value.split('-')] for value in values: + # noinspection PyTypeChecker if self.min_limit is not None and value < self.min_limit: raise DiagnosticOptionError('Value {0} is lower than minimum ({1})'.format(value, self.min_limit)) + # noinspection PyTypeChecker if self.max_limit is not None and value > self.max_limit: raise DiagnosticOptionError('Value {0} is higher than maximum ({1})'.format(value, self.max_limit)) @@ -221,20 +331,29 @@ class DiagnosticListFrequenciesOption(DiagnosticOption): class DiagnosticVariableOption(DiagnosticOption): + def __init__(self, var_manager, name='variable', default_value=None): + super(DiagnosticVariableOption, self).__init__(name, default_value) + self.var_manager = var_manager + def parse(self, option_value): option_value = self.check_default(option_value) - real_name = VariableManager().get_variable(option_value, False) + real_name = self.var_manager.get_variable(option_value, False) if real_name is None: return option_value return real_name.short_name class DiagnosticVariableListOption(DiagnosticOption): + + def __init__(self, var_manager, name, default_value=None): + super(DiagnosticVariableListOption, self).__init__(name, default_value) + self.var_manager = var_manager + def parse(self, option_value): option_value = self.check_default(option_value) var_names = [] for value in option_value.split('-'): - real_name = VariableManager().get_variable(value, False) + real_name = self.var_manager.get_variable(value, False) if real_name is None: var_names.append(value) else: @@ -243,11 +362,17 @@ class DiagnosticVariableListOption(DiagnosticOption): class DiagnosticDomainOption(DiagnosticOption): + def __init__(self, name='domain', default_value=None): + super(DiagnosticDomainOption, self).__init__(name, default_value) + def parse(self, option_value): return ModelingRealms.parse(self.check_default(option_value)) class DiagnosticFrequencyOption(DiagnosticOption): + def __init__(self, name='frequency', default_value=None): + super(DiagnosticFrequencyOption, self).__init__(name, default_value) + def parse(self, option_value): return Frequency.parse(self.check_default(option_value)) @@ -277,6 +402,10 @@ class DiagnosticChoiceOption(DiagnosticOption): self.choices = choices self.ignore_case = ignore_case + # To check if it is valid + if default_value is not None: + self.parse(default_value) + def parse(self, value): value = self.check_default(value) if self.ignore_case: diff --git a/earthdiagnostics/earthdiags.py b/earthdiagnostics/earthdiags.py index cc8fe5a90c6e3e227912a0e3c432eee84f1a0bb6..b9ef5aa2935518a28a7094b32e64be73afc7c14e 100755 --- a/earthdiagnostics/earthdiags.py +++ b/earthdiagnostics/earthdiags.py @@ -1,30 +1,24 @@ #!/usr/bin/env python # coding=utf-8 -from six.moves import queue as q import argparse +import os import shutil -import threading -import pkg_resources +import tempfile +from distutils.spawn import find_executable +import bscearth.utils.path import netCDF4 -import operator -import os +import pkg_resources from bscearth.utils.date import * -import bscearth.utils.path -from earthdiagnostics.constants import Basins -from earthdiagnostics.config import Config +from earthdiagnostics import cdftools from earthdiagnostics.cmormanager import CMORManager +from earthdiagnostics.config import Config +from earthdiagnostics.constants import Basins +from earthdiagnostics.obsreconmanager import ObsReconManager from earthdiagnostics.threddsmanager import THREDDSManager -from earthdiagnostics import cdftools from earthdiagnostics.utils import TempFile, Utils -from earthdiagnostics.diagnostic import Diagnostic -from earthdiagnostics.ocean import * -from earthdiagnostics.general import * -from earthdiagnostics.statistics import * -from earthdiagnostics.variable import VariableManager -from earthdiagnostics.diagnostic import DiagnosticOptionError -import tempfile +from work_manager import WorkManager class EarthDiags(object): @@ -106,6 +100,8 @@ class EarthDiags(object): Utils.cdo.debug = True Utils.nco.debug = False # This is due to a bug in nco. Must change when it's solved + Utils.cdo.CDO = find_executable('cdo') + if args.logfilepath: Log.set_file(bscearth.utils.path.expand_path(args.logfilepath)) @@ -151,40 +147,19 @@ class EarthDiags(object): self._prepare_mesh_files() self._initialize_basins() - self._register_diagnostics() - self._prepare_data_manager() # Run diagnostics Log.info('Running diagnostics') - list_jobs = self.prepare_job_list() - self._failed_jobs = [] - - time = datetime.datetime.now() - Log.info("Starting to compute at {0}", time) - self.threads = Utils.available_cpu_count() - if 0 < self.config.max_cores < self.threads: - self.threads = self.config.max_cores - Log.info('Using {0} threads', self.threads) - threads = list() - for num_thread in range(0, self.threads): - self.time[num_thread] = dict() - t = threading.Thread(target=EarthDiags._run_jobs, args=(self, list_jobs, num_thread)) - threads.append(t) - t.start() - - for t in threads: - t.join() - - TempFile.clean() - finish_time = datetime.datetime.now() - Log.result("Diagnostics finished at {0}", finish_time) - Log.result("Elapsed time: {0}\n", finish_time - time) - self.print_errors() - self.print_stats() + + work_manager = WorkManager(self.config, self.data_manager) + work_manager.prepare_job_list() + + result = work_manager.run() + if self.config.auto_clean: self._remove_scratch_dir() - return not self.had_errors + return result def _initialize_basins(self): self._read_basins_from_file('mask_regions.nc') @@ -213,100 +188,10 @@ class EarthDiags(object): self.data_manager = CMORManager(self.config) elif self.config.data_adaptor == 'THREDDS': self.data_manager = THREDDSManager(self.config) + elif self.config.data_adaptor == 'OBSRECON': + self.data_manager = ObsReconManager(self.config) self.data_manager.prepare() - def print_stats(self): - Log.info('Time consumed by each diagnostic class') - Log.info('--------------------------------------') - total = dict() - for num_thread in range(0, self.threads): - for key, value in self.time[num_thread].items(): - if key in total: - total[key] += value - else: - total[key] = value - for diag, time in sorted(total.items(), key=operator.itemgetter(1)): - Log.info('{0:23} {1:}', diag.__name__, time) - - def print_errors(self): - if len(self._failed_jobs) == 0: - return - self.had_errors = True - Log.error('Failed jobs') - Log.error('-----------') - for job in self._failed_jobs: - Log.error(str(job)) - Log.info('') - - def prepare_job_list(self): - list_jobs = q.Queue() - for fulldiag in self.config.get_commands(): - Log.info("Adding {0} to diagnostic list", fulldiag) - diag_options = fulldiag.split(',') - - diag_class = Diagnostic.get_diagnostic(diag_options[0]) - if diag_class: - try: - for job in diag_class.generate_jobs(self, diag_options): - list_jobs.put(job) - continue - except DiagnosticOptionError as ex: - Log.error('Can not configure diagnostic {0}: {1}', diag_options[0], ex) - self.had_errors = True - else: - Log.error('{0} is not an available diagnostic', diag_options[0]) - self.had_errors = True - return list_jobs - - @staticmethod - def _register_diagnostics(): - EarthDiags._register_ocean_diagnostics() - EarthDiags._register_general_diagnostics() - EarthDiags._register_stats_diagnostics() - - @staticmethod - def _register_stats_diagnostics(): - Diagnostic.register(MonthlyPercentile) - Diagnostic.register(ClimatologicalPercentile) - - @staticmethod - def _register_general_diagnostics(): - Diagnostic.register(DailyMean) - Diagnostic.register(MonthlyMean) - Diagnostic.register(YearlyMean) - Diagnostic.register(SimplifyDimensions) - Diagnostic.register(Relink) - Diagnostic.register(RelinkAll) - Diagnostic.register(Scale) - Diagnostic.register(Attribute) - Diagnostic.register(SelectLevels) - Diagnostic.register(Module) - - @staticmethod - def _register_ocean_diagnostics(): - Diagnostic.register(MixedLayerSaltContent) - Diagnostic.register(Siasiesiv) - Diagnostic.register(VerticalMean) - Diagnostic.register(VerticalMeanMeters) - Diagnostic.register(Interpolate) - Diagnostic.register(InterpolateCDO) - Diagnostic.register(Moc) - Diagnostic.register(AreaMoc) - Diagnostic.register(MaxMoc) - Diagnostic.register(Psi) - Diagnostic.register(Gyres) - Diagnostic.register(ConvectionSites) - Diagnostic.register(CutSection) - Diagnostic.register(AverageSection) - Diagnostic.register(MixedLayerHeatContent) - Diagnostic.register(HeatContentLayer) - Diagnostic.register(HeatContent) - Diagnostic.register(RegionMean) - Diagnostic.register(Rotation) - Diagnostic.register(Mxl) - Diagnostic.register(VerticalGradient) - Diagnostic.register(MaskLand) - def clean(self): Log.info('Removing scratch folder...') self._remove_scratch_dir() @@ -341,7 +226,7 @@ class EarthDiags(object): return True def _get_variable_report(self, startdate, member): - var_manager = VariableManager() + var_manager = self.config.var_manager results = list() for var in var_manager.get_all_variables(): if var.domain is None: @@ -381,47 +266,11 @@ class EarthDiags(object): file_handler.flush() file_handler.close() - def _run_jobs(self, queue, numthread): - def _run_job(current_job, retrials=1): - while retrials >= 0: - try: - Log.info('Starting {0}', current_job) - time = datetime.datetime.now() - current_job.compute() - time = datetime.datetime.now() - time - if type(current_job) in self.time[numthread]: - self.time[numthread][type(current_job)] += time - else: - self.time[numthread][type(current_job)] = time - Log.result('Finished {0}', current_job) - return True - except Exception as ex: - retrials -= 1 - Log.error('Job {0} failed: {1}', job, ex) - return False - count = 0 - failed_jobs = list() - - while not queue.empty(): - try: - job = queue.get(timeout=1) - if _run_job(job): - count += 1 - else: - failed_jobs.append(str(job)) - queue.task_done() - except q.Queue.Empty: - continue - - if len(failed_jobs) == 0: - Log.result('Thread {0} finished after taking care of {1} tasks', numthread, count) - else: - Log.result('Thread {0} finished after running successfully {1} of {2} tasks', numthread, count, - count + len(failed_jobs)) - self._failed_jobs += failed_jobs - return - def _prepare_mesh_files(self): + model_version = self.config.experiment.model_version + if not model_version: + Log.info('No model version defined. Skipping mesh files copy!') + return Log.info('Copying mesh files') con_files = self.config.con_files model_version = self.config.experiment.model_version @@ -457,6 +306,7 @@ class EarthDiags(object): Utils.give_group_write_permissions(self.config.scratch_masks) mesh_mask_scratch_path = os.path.join(self.config.scratch_masks, mesh_mask) + if self._copy_file(mesh_mask_path, mesh_mask_scratch_path, restore_meshes): Utils.give_group_write_permissions(mesh_mask_scratch_path) @@ -500,12 +350,14 @@ class EarthDiags(object): return False if not force and os.path.exists(destiny): - if os.stat(source).st_size == os.stat(destiny).st_size: + # Small size differences can be due to the renaming of variables + delta_size = abs(os.stat(source).st_size - os.stat(destiny).st_size) + if delta_size < 512: Log.info('File {0} already exists', destiny) return True - Log.info('Creating file {0}', destiny) - shutil.copy(source, destiny) + Log.info('Copying file {0}', destiny) + shutil.copyfile(source, destiny) Log.info('File {0} ready', destiny) Utils.rename_variables(destiny, self.dic_variables, False, True) return True diff --git a/earthdiagnostics/frequency.py b/earthdiagnostics/frequency.py index 1473fa5a2b8798923f405ba3ba5aaeb5086da53d..09ebe3de5d3b3e972cadcd60b9ff68308c1e81fc 100644 --- a/earthdiagnostics/frequency.py +++ b/earthdiagnostics/frequency.py @@ -38,6 +38,8 @@ class Frequency(object): freq_str = 'clim' elif self in (Frequencies.three_hourly, Frequencies.six_hourly, Frequencies.hourly): freq_str = self.frequency[:-2] + 'hourly' + if vartype != VariableType.MEAN: + freq_str = '{0}_{1}'.format(freq_str, VariableType.to_str(vartype)) else: freq_str = 'monthly_{0}'.format(VariableType.to_str(vartype)) return freq_str diff --git a/earthdiagnostics/general/__init__.py b/earthdiagnostics/general/__init__.py index be95992ffc1fbffea51f3fdd8c202cf45b43a2f0..34820058a09b80bbd78b343158abef14094df0fb 100644 --- a/earthdiagnostics/general/__init__.py +++ b/earthdiagnostics/general/__init__.py @@ -10,3 +10,4 @@ from earthdiagnostics.general.relinkall import RelinkAll from earthdiagnostics.general.simplify_dimensions import SimplifyDimensions from earthdiagnostics.general.select_levels import SelectLevels from earthdiagnostics.general.module import Module +from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris diff --git a/earthdiagnostics/general/attribute.py b/earthdiagnostics/general/attribute.py index 178dae42a8d974ff7034f21fa939bc471b9cddb5..1c6185787d95a7ed3fa3525f477f49ff02d930c8 100644 --- a/earthdiagnostics/general/attribute.py +++ b/earthdiagnostics/general/attribute.py @@ -44,13 +44,13 @@ class Attribute(Diagnostic): self.attributte_value = attributte_value def __str__(self): - return 'Write attributte output Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Attributte:{5}:{6}'.format(self.startdate, self.member, self.chunk, self.domain, - self.variable, self.attributte_name, self.attributte_value) + return 'Write attributte output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Variable: {0.domain}:{0.variable} Attributte: {0.attributte_name}:{0.attributte_value} ' \ + 'Grid: {0.grid}'.format(self) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ - self.domain == other.domain and self.variable == other.variable and \ + self.domain == other.domain and self.variable == other.variable and self.grid == other.grid and \ self.attributte_name == other.attributte_name and self.attributte_value == other.attributte_value @classmethod @@ -65,8 +65,8 @@ class Attribute(Diagnostic): :return: """ - options_available = (DiagnosticVariableOption('variable'), - DiagnosticDomainOption('domain'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticOption('name'), DiagnosticComplexStrOption('value'), DiagnosticOption('grid', '')) @@ -78,18 +78,25 @@ class Attribute(Diagnostic): options['value'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + variable_file = self.variable_file.local_file handler = Utils.openCdf(variable_file) handler.setncattr(self.attributte_name, self.attributte_value) handler.close() if not Utils.check_netcdf_file(variable_file): raise Exception('Attribute {0} can not be set correctly to {1}'.format(self.attributte_name, self.attributte_value)) - self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + + self.corrected.set_local_file(variable_file, self) diff --git a/earthdiagnostics/general/dailymean.py b/earthdiagnostics/general/dailymean.py index cb4133130abf7d853ff08805584ed5c514773e85..7fb4736eba58980d40dce6f247b85f22b91850f3 100644 --- a/earthdiagnostics/general/dailymean.py +++ b/earthdiagnostics/general/dailymean.py @@ -70,9 +70,9 @@ class DailyMean(Diagnostic): :return: """ - options_available = (DiagnosticVariableOption('variable'), - DiagnosticDomainOption('domain'), - DiagnosticFrequencyOption('frequency'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFrequencyOption(), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) job_list = list() @@ -81,25 +81,29 @@ class DailyMean(Diagnostic): options['domain'], options['variable'], options['frequency'], options['grid'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + + def declare_data_generated(self): + self.daymean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.daily, grid=self.grid) + def compute(self): """ Runs the diagnostic """ - day_mean = TempFile.get() - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=self.frequency, grid=self.grid) - handler = Utils.openCdf(variable_file) + temp = TempFile.get() + handler = Utils.openCdf(self.variable_file.local_file) if 'region' in handler.variables: noregion = TempFile.get() - Utils.nco.ncks(input=variable_file, output=noregion, options=('-O -C -x -v region',)) - Utils.cdo.daymean(input=noregion, output=day_mean) - monmean_handler = Utils.openCdf(day_mean) + Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) + Utils.cdo.daymean(input=noregion, output=temp) + os.remove(noregion) + monmean_handler = Utils.openCdf(temp) Utils.copy_variable(handler, monmean_handler, 'region') monmean_handler.close() else: - Utils.cdo.daymean(input=variable_file, output=day_mean) - os.remove(variable_file) - - self.send_file(day_mean, self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.daily, grid=self.grid) + Utils.cdo.daymean(input=self.variable_file.local_file, output=temp) + self.daymean.set_local_file(temp) diff --git a/earthdiagnostics/general/module.py b/earthdiagnostics/general/module.py index a5c869dadd2ea7c1962ee5f9d26390764930d876..f72aa5f568d43f6e70fe277c528ea065d74db443 100644 --- a/earthdiagnostics/general/module.py +++ b/earthdiagnostics/general/module.py @@ -1,6 +1,6 @@ # coding=utf-8 from earthdiagnostics.diagnostic import * -from earthdiagnostics.utils import Utils +from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealm import numpy as np @@ -23,8 +23,6 @@ class Module(Diagnostic): :type member: int :param chunk: chunk's number :type chunk: int : - :param variable: variable's name - :type variable: str :param domain: variable's domain :type domain: ModelingRealm """ @@ -67,10 +65,10 @@ class Module(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticDomainOption('domain'), - DiagnosticVariableOption('componentu'), - DiagnosticVariableOption('componentv'), - DiagnosticVariableOption('module'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'componentu'), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'componentv'), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'module'), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) job_list = list() @@ -80,18 +78,24 @@ class Module(Diagnostic): options['grid'])) return job_list + def request_data(self): + self.component_u_file = self.request_chunk(self.domain, self.componentu, self.startdate, self.member, + self.chunk, grid=self.grid) + self.component_v_file = self.request_chunk(self.domain, self.componentv, self.startdate, self.member, + self.chunk, grid=self.grid) + + def declare_data_generated(self): + self.module_file = self.declare_chunk(self.domain, self.module, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - component_u_file = self.data_manager.get_file(self.domain, self.componentu, self.startdate, self.member, - self.chunk, grid=self.grid) - - component_v_file = self.data_manager.get_file(self.domain, self.componentv, self.startdate, self.member, - self.chunk, grid=self.grid) - - component_u = Utils.openCdf(component_u_file) - component_v = Utils.openCdf(component_v_file) + temp = TempFile.get() + Utils.copy_file(self.component_u_file.local_file, temp) + component_u = Utils.openCdf(temp) + component_v = Utils.openCdf(self.component_v_file.local_file) variable_u = component_u.variables[self.componentu] variable_v = component_v.variables[self.componentv] @@ -105,5 +109,4 @@ class Module(Diagnostic): component_u.close() component_v.close() - self.send_file(component_u_file, self.domain, self.module, self.startdate, self.member, self.chunk, - grid=self.grid, rename_var=self.componentu) + self.module_file.set_local_file(temp, rename_var=self.componentu) diff --git a/earthdiagnostics/general/monthlymean.py b/earthdiagnostics/general/monthlymean.py index ee3161791bb80891e4b4d75de83e96ea7ea636d6..dca5e730af29859750df5055dc331976dfcf9df8 100644 --- a/earthdiagnostics/general/monthlymean.py +++ b/earthdiagnostics/general/monthlymean.py @@ -29,7 +29,7 @@ class MonthlyMean(Diagnostic): :param domain: variable's domain :type domain: ModelingRealm :param frequency: original frequency - :type frequency: str + :type frequency: Frequency :param grid: original data grid :type grid: str """ @@ -68,8 +68,8 @@ class MonthlyMean(Diagnostic): :return: """ - options_available = (DiagnosticVariableOption('variable'), - DiagnosticDomainOption('domain'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticFrequencyOption('frequency', Frequencies.daily), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) @@ -77,29 +77,35 @@ class MonthlyMean(Diagnostic): for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(MonthlyMean(diags.data_manager, startdate, member, chunk, options['domain'], options['variable'], options['frequency'], options['grid'])) + return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + + def declare_data_generated(self): + self.monmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, grid=self.grid) + def compute(self): """ Runs the diagnostic """ - monmean = TempFile.get() - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=self.frequency, grid=self.grid) - handler = Utils.openCdf(variable_file) + handler = Utils.openCdf(self.variable_file.local_file) + temp = TempFile.get() if 'region' in handler.variables: noregion = TempFile.get() - Utils.nco.ncks(input=variable_file, output=noregion, options=('-O -C -x -v region',)) - Utils.cdo.monmean(input=noregion, output=monmean) - monmean_handler = Utils.openCdf(monmean) + Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) + Utils.cdo.monmean(input=noregion, output=temp) + os.remove(noregion) + monmean_handler = Utils.openCdf(temp) Utils.copy_variable(handler, monmean_handler, 'region') monmean_handler.close() else: - Utils.cdo.monmean(input=variable_file, output=monmean) + Utils.cdo.monmean(input=self.variable_file.local_file, output=temp) handler.close() + self.monmean.set_local_file(temp) - os.remove(variable_file) - self.send_file(monmean, self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.monthly, grid=self.grid) diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py index ecfd024748649c6ec9ca19020b3384a05267fe68..60c69f4cf0894b38ec2df16ee2a379f37603587c 100644 --- a/earthdiagnostics/general/relink.py +++ b/earthdiagnostics/general/relink.py @@ -41,11 +41,11 @@ class Relink(Diagnostic): self.domain = domain self.move_old = move_old self.grid = grid + self.var_manager = data_manager.config.var_manager def __str__(self): - return 'Relink output Startdate: {0} Member: {1} Chunk: {2} Move old: {5} ' \ - 'Variable: {3}:{4} Grid: {6}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, - self.move_old, self.grid) + return 'Relink output Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Move old: {0.move_old} ' \ + 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ @@ -63,8 +63,8 @@ class Relink(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), - DiagnosticDomainOption('domain'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticBoolOption('move_old', True), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) @@ -74,11 +74,17 @@ class Relink(Diagnostic): options['domain'], options['variable'], options['move_old'], options['grid'])) return job_list + def request_data(self): + pass + + def declare_data_generated(self): + pass + def compute(self): """ Runs the diagnostic """ - self.data_manager.link_file(self.domain, self.variable, VariableManager().get_variable(self.variable), + self.data_manager.link_file(self.domain, self.variable, self.var_manager.get_variable(self.variable), self.startdate, self.member, self.chunk, move_old=self.move_old, grid=self.grid) diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py index 6ee2d226a763d3fb63260130ddb2b3c6d72363af..d5fffc4ae4ba02b9a23f526c3fd031139050f871 100644 --- a/earthdiagnostics/general/relinkall.py +++ b/earthdiagnostics/general/relinkall.py @@ -47,6 +47,12 @@ class RelinkAll(Diagnostic): job_list.append(RelinkAll(diags.data_manager, startdate)) return job_list + def request_data(self): + pass + + def declare_data_generated(self): + pass + def compute(self): """ Runs the diagnostic diff --git a/earthdiagnostics/general/rewrite.py b/earthdiagnostics/general/rewrite.py index 6b82a1dd1b5d9df39d6dd59a580f24be224c7ea7..2aa937baa7dd988fd872047331f54a39461489f1 100644 --- a/earthdiagnostics/general/rewrite.py +++ b/earthdiagnostics/general/rewrite.py @@ -58,8 +58,8 @@ class Rewrite(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), - DiagnosticDomainOption('domain'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) job_list = list() @@ -68,12 +68,17 @@ class Rewrite(Diagnostic): options['domain'], options['variable'], options['grid'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) - self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + self.corrected.set_local_file(self.variable_file.local_file, self) diff --git a/earthdiagnostics/general/scale.py b/earthdiagnostics/general/scale.py index 28a5d86c9f578a8bf2b3066b675eb1277119dfda..116a978ddc632c5d7e64745f4425ef46c85da68f 100644 --- a/earthdiagnostics/general/scale.py +++ b/earthdiagnostics/general/scale.py @@ -70,8 +70,8 @@ class Scale(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), - DiagnosticDomainOption('domain'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticFloatOption('value'), DiagnosticFloatOption('offset'), DiagnosticOption('grid', ''), @@ -87,12 +87,19 @@ class Scale(Diagnostic): options['grid'], options['min_limit'], options['max_limit'], frequency)) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, frequency=self.frequency, to_modify=True) + + def declare_data_generated(self): + self.corrected = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, frequency=self.frequency) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, frequency=self.frequency) + variable_file = self.variable_file.local_file handler = Utils.openCdf(variable_file) var_handler = handler.variables[self.variable] @@ -100,8 +107,7 @@ class Scale(Diagnostic): if self._check_limits(): var_handler[:] = self.original_values * self.value + self.offset handler.close() - self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid, frequency=self.frequency) + self.corrected.set_local_file(self.variable_file.local_file, self) def _check_limits(self): if not math.isnan(self.min_limit) and (self.original_values.min() < self.min_limit): diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py index c588c5e7803d7cdb9f1cdb3224e750f2c917af4d..1d2fb9cac33b0ebed3d5c3daf8fa3ab5b287b7bf 100644 --- a/earthdiagnostics/general/select_levels.py +++ b/earthdiagnostics/general/select_levels.py @@ -2,7 +2,7 @@ from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, \ DiagnosticVariableListOption, DiagnosticIntOption from earthdiagnostics.modelingrealm import ModelingRealm -from earthdiagnostics.utils import Utils +from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.box import Box @@ -46,9 +46,9 @@ class SelectLevels(Diagnostic): def __str__(self): return 'Select levels Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Levels: {6}-{7} Grid: {5}'.format(self.startdate, self.member, self.chunk, - self.domain, self.variable, - self.grid, self.box.min_depth, self.box.max_depth) + 'Variable: {3}:{4} Levels: {6}-{7} ' \ + 'Grid: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, + self.grid, self.box.min_depth, self.box.max_depth) def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ @@ -65,8 +65,8 @@ class SelectLevels(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticDomainOption('domain'), - DiagnosticVariableListOption('variables'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), DiagnosticIntOption('first_level'), DiagnosticIntOption('last_level'), DiagnosticOption('grid', '')) @@ -81,17 +81,23 @@ class SelectLevels(Diagnostic): options['first_level'], options['last_level'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.result = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + temp = TempFile.get() - Utils.nco.ncks(input=variable_file, output=variable_file, + Utils.nco.ncks(input=self.variable_file, output=temp, options=('-O -d lev,{0.min_depth},{0.max_depth}'.format(self.box),)) - self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + self.result.set_local_file(temp) @staticmethod def _create_var(var_name, var_values, source, destiny): diff --git a/earthdiagnostics/general/simplify_dimensions.py b/earthdiagnostics/general/simplify_dimensions.py index d8261c9cecd9357aa6aab597ce4b0d99c841328b..579a5473faf0dcc0804ba30c361688de6b03c535 100644 --- a/earthdiagnostics/general/simplify_dimensions.py +++ b/earthdiagnostics/general/simplify_dimensions.py @@ -61,8 +61,8 @@ class SimplifyDimensions(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticDomainOption('domain'), - DiagnosticVariableListOption('variables'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) job_list = list() @@ -74,24 +74,31 @@ class SimplifyDimensions(Diagnostic): options['domain'], var, options['grid'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid, to_modify=True) + + def declare_data_generated(self): + self.simplified = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) - - handler = Utils.openCdf(variable_file) + handler = Utils.openCdf(self.variable_file.local_file) if 'i' not in handler.dimensions: raise Exception('Variable {0.domain}:{0.variable} does not have i,j dimensions'.format(self)) lat = handler.variables['lat'] lat_values = lat[:, 0:1] + # noinspection PyTypeChecker if np.any((lat[:] - lat_values) != 0): raise Exception('Latitude is not constant over i dimension for variable ' '{0.domain}:{0.variable}'.format(self)) lon = handler.variables['lon'] lon_values = lon[0:1, :] + # noinspection PyTypeChecker if np.any((lon[:] - lon) != 0): raise Exception('Longitude is not constant over j dimension for variable ' '{0.domain}:{0.variable}'.format(self)) @@ -117,8 +124,7 @@ class SimplifyDimensions(Diagnostic): handler.close() new_file.close() - self.send_file(temp, self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + self.simplified.set_local_file(temp) @staticmethod def _create_var(var_name, var_values, source, destiny): diff --git a/earthdiagnostics/general/verticalmeanmetersiris.py b/earthdiagnostics/general/verticalmeanmetersiris.py new file mode 100644 index 0000000000000000000000000000000000000000..92de0bef9d587e16d2d452f0fc88b9b74fba8f95 --- /dev/null +++ b/earthdiagnostics/general/verticalmeanmetersiris.py @@ -0,0 +1,126 @@ +# coding=utf-8 +import iris +import iris.analysis +import iris.exceptions + +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticFloatOption, DiagnosticDomainOption, \ + DiagnosticVariableOption +from earthdiagnostics.utils import TempFile +from earthdiagnostics.modelingrealm import ModelingRealms + + +class VerticalMeanMetersIris(Diagnostic): + """ + Averages vertically any given variable + + :original author: Virginie Guemas + :contributor: Javier Vegas-Regidor + + :created: February 2012 + :last modified: June 2016 + + :param data_manager: data management object + :type data_manager: DataManager + :param startdate: startdate + :type startdate: str + :param member: member number + :type member: int + :param chunk: chunk's number + :type chunk: int + :param variable: variable to average + :type variable: str + :param box: box used to restrict the vertical mean + :type box: Box + + """ + + alias = 'vmean' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, startdate, member, chunk, domain, variable, box): + Diagnostic.__init__(self, data_manager) + self.startdate = startdate + self.member = member + self.chunk = chunk + self.domain = domain + self.variable = variable + self.box = box + + def __eq__(self, other): + return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ + self.box == other.box and self.variable == other.variable + + def __str__(self): + return 'Vertical mean meters Startdate: {0} Member: {1} Chunk: {2} Variable: {3}:{4} ' \ + 'Box: {5}'.format(self.startdate, self.member, self.chunk, self.domain, self.variable, self.box) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: variable, minimum depth (meters), maximum depth (meters) + :type options: list[str] + :return: + """ + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFloatOption('min_depth', -1), + DiagnosticFloatOption('max_depth', -1), + DiagnosticDomainOption(default_value=ModelingRealms.ocean)) + options = cls.process_options(options, options_available) + + box = Box(True) + if options['min_depth'] >= 0: + box.min_depth = options['min_depth'] + if options['max_depth'] >= 0: + box.max_depth = options['max_depth'] + + job_list = list() + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(VerticalMeanMetersIris(diags.data_manager, startdate, member, chunk, + options['domain'], options['variable'], box)) + return job_list + + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(self.domain, self.variable + 'vmean', self.startdate, self.member, + self.chunk, box=self.box) + + def compute(self): + """ + Runs the diagnostic + """ + iris.FUTURE.netcdf_no_unlimited = True + iris.FUTURE.netcdf_promote = True + + var_cube = iris.load_cube(self.variable_file.local_file) + + lev_names = ('lev', 'depth') + coord = None + for coord_name in lev_names: + try: + coord = var_cube.coord(coord_name) + except iris.exceptions.CoordinateNotFoundError: + pass + + if self.box.min_depth is None: + lev_min = coord.points[0] + else: + lev_min = self.box.min_depth + + if self.box.max_depth is None: + lev_max = coord.points[-1] + else: + lev_max = self.box.max_depth + var_cube = var_cube.extract(iris.Constraint(coord_values= + {coord.var_name: lambda cell: lev_min <= cell <= lev_max})) + var_cube = var_cube.collapsed(coord, iris.analysis.MEAN) + temp = TempFile.get() + iris.save(var_cube, temp, zlib=True) + self.results.set_local_file(temp, rename_var=var_cube.var_name) diff --git a/earthdiagnostics/general/yearlymean.py b/earthdiagnostics/general/yearlymean.py index 07ae10022218b77b6aca5a86e03fa85f27dbcd67..148f0ca210a07a2661bf68ded705a5edef2c46bd 100644 --- a/earthdiagnostics/general/yearlymean.py +++ b/earthdiagnostics/general/yearlymean.py @@ -70,9 +70,9 @@ class YearlyMean(Diagnostic): :return: """ - options_available = (DiagnosticVariableOption('variable'), - DiagnosticDomainOption('domain'), - DiagnosticFrequencyOption('frequency'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticFrequencyOption(default_value=diags.config.frequency), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) job_list = list() @@ -81,25 +81,31 @@ class YearlyMean(Diagnostic): options['domain'], options['variable'], options['frequency'], options['grid'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=self.frequency, grid=self.grid) + + def declare_data_generated(self): + self.yearmean = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + frequency=Frequencies.yearly, grid=self.grid) + def compute(self): """ Runs the diagnostic """ - year_mean = TempFile.get() - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=self.frequency, grid=self.grid) - handler = Utils.openCdf(variable_file) + temp = TempFile.get() + + handler = Utils.openCdf(self.variable_file.local_file) if 'region' in handler.variables: noregion = TempFile.get() - Utils.nco.ncks(input=variable_file, output=noregion, options=('-O -C -x -v region',)) - Utils.cdo.yearmean(input=noregion, output=year_mean) - monmean_handler = Utils.openCdf(year_mean) + Utils.nco.ncks(input=self.variable_file.local_file, output=noregion, options=('-O -C -x -v region',)) + Utils.cdo.yearmean(input=noregion, output=temp) + monmean_handler = Utils.openCdf(temp) Utils.copy_variable(handler, monmean_handler, 'region') monmean_handler.close() else: - Utils.cdo.yearmean(input=variable_file, output=year_mean) - os.remove(variable_file) + Utils.cdo.yearmean(input=self.variable_file.local_file, output=temp) + os.remove(self.variable_file.local_file) - self.send_file(year_mean, self.domain, self.variable, self.startdate, self.member, self.chunk, - frequency=Frequencies.yearly, grid=self.grid) + self.yearmean.set_local_file(temp) diff --git a/earthdiagnostics/modelingrealm.py b/earthdiagnostics/modelingrealm.py index a7a76573d3ef8bfb537e76b8f8b76eaebffe645a..703d5b47d8c28f8b6aa257eeed1fcf4cc29d143a 100644 --- a/earthdiagnostics/modelingrealm.py +++ b/earthdiagnostics/modelingrealm.py @@ -5,16 +5,16 @@ from earthdiagnostics.frequency import Frequencies class ModelingRealm(object): def __init__(self, domain_name): - domain_name = domain_name.lower() - if domain_name == 'seaice': + lower_name = domain_name.lower() + if lower_name == 'seaice': self.name = 'seaIce' - elif domain_name == 'landice': + elif lower_name == 'landice': self.name = 'landIce' - elif domain_name == 'atmoschem': + elif lower_name == 'atmoschem': self.name = 'atmosChem' - elif domain_name == 'ocnbgchem': + elif lower_name == 'ocnbgchem': self.name = 'ocnBgchem' - elif domain_name in ['ocean', 'atmos', 'land', 'aerosol']: + elif lower_name in ['ocean', 'atmos', 'land', 'aerosol']: self.name = domain_name else: raise ValueError('Modelling realm {0} not recognized!'.format(domain_name)) @@ -28,6 +28,9 @@ class ModelingRealm(object): def __str__(self): return self.name + def __repr__(self): + return str(self) + def get_table_name(self, frequency, data_convention): """ Returns the table name for a domain-frequency pair @@ -38,21 +41,24 @@ class ModelingRealm(object): :return: variable's table name :rtype: str """ - if frequency in (Frequencies.monthly, Frequencies.climatology, Frequencies.daily): - if self.name == 'seaIce': - if data_convention in ('specs', 'preface'): - prefix = 'OI' - else: - prefix = 'SI' - elif self.name == 'landIce': - prefix = 'LI' + if self.name == 'seaIce': + if data_convention in ('specs', 'preface'): + prefix = 'OI' else: - prefix = self.name[0].upper() - table_name = prefix + str(frequency) - elif frequency == Frequencies.six_hourly: + prefix = 'SI' + elif self.name == 'landIce': + prefix = 'LI' + else: + prefix = self.name[0].upper() + + if frequency == Frequencies.six_hourly: table_name = '6hrPlev' else: - table_name = frequency.frequency + if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention not in ('specs', + 'preface'): + table_name = prefix + str(frequency) + else: + table_name = frequency.frequency return table_name def get_table(self, frequency, data_convention): diff --git a/earthdiagnostics/obsreconmanager.py b/earthdiagnostics/obsreconmanager.py new file mode 100644 index 0000000000000000000000000000000000000000..661986c9b9d515045e1aca11b56b126c7a1478b9 --- /dev/null +++ b/earthdiagnostics/obsreconmanager.py @@ -0,0 +1,272 @@ +# coding=utf-8 +import os + +from bscearth.utils.log import Log + +from earthdiagnostics.datamanager import DataManager +from earthdiagnostics.variable_type import VariableType + + +class ObsReconManager(DataManager): + """ + Data manager class for CMORized experiments + """ + def __init__(self, config): + super(ObsReconManager, self).__init__(config) + data_folders = self.config.data_dir.split(':') + self.config.data_dir = None + for data_folder in data_folders: + if os.path.isdir(os.path.join(data_folder, self.config.data_type, self.experiment.institute.lower(), + self.experiment.model.lower())): + self.config.data_dir = data_folder + break + + if not self.config.data_dir: + raise Exception('Can not find model data') + + if self.config.data_type in ('obs', 'recon') and self.experiment.chunk_size != 1: + raise Exception('For obs and recon data chunk_size must be always 1') + + # noinspection PyUnusedLocal + def request_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, + vartype=VariableType.MEAN): + filepath = self.get_file_path(startdate, domain, variable, frequency, vartype) + return self._get_file_from_storage(filepath) + + def create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + pass + + # noinspection PyUnusedLocal + def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param domain: CMOR domain + :type domain: str + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + return NotImplementedError + + def get_file_path(self, startdate, domain, var, frequency, vartype, + box=None, grid=None): + """ + Returns the path to a concrete file + :param startdate: file's startdate + :type startdate: str + :param domain: file's domain + :type domain: str + :param var: file's var + :type var: str + :param frequency: file's frequency + :type frequency: Frequency + :param box: file's box + :type box: Box + :param grid: file's grid + :type grid: str + :return: path to the file + :rtype: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + """ + if not frequency: + frequency = self.config.frequency + var = self._get_final_var_name(box, var) + + folder_path = self._get_folder_path(frequency, domain, var, grid, vartype) + file_name = self._get_file_name(var, startdate) + + filepath = os.path.join(folder_path, file_name) + return filepath + + def _get_folder_path(self, frequency, domain, variable, grid, vartype): + + if not frequency.frequency.endswith('hr'): + var_folder = self.get_varfolder(domain, variable, grid) + else: + var_folder = variable + + folder_path = os.path.join(self.config.data_dir, self.config.data_type, + self.experiment.institute.lower(), + self.experiment.model.lower(), + frequency.folder_name(vartype), + var_folder) + return folder_path + + def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): + """ + Ge a file containing all the data for one year for one variable + :param domain: variable's domain + :type domain: str + :param var: variable's name + :type var: str + :param startdate: startdate to retrieve + :type startdate: str + :param member: member to retrieve + :type member: int + :param year: year to retrieve + :type year: int + :param grid: variable's grid + :type grid: str + :param box: variable's box + :type box: Box + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: + """ + raise NotImplementedError() + + def get_var_url(self, var, startdate, frequency, box, vartype): + """ + Get url for dataset + :param var: variable to retrieve + :type var: str + :param startdate: startdate to retrieve + :type startdate: str + :param frequency: frequency to get: + :type frequency: Frequency | None + :param box: box to get + :type box: Box + :param vartype: type of variable + :type vartype: VariableType + :return: + """ + if not frequency: + frequency = self.config.frequency + var = self._get_final_var_name(box, var) + full_path = os.path.join(self.config.data_dir, self.config.data_type, self.experiment.institute, + self.experiment.model, frequency.folder_name(vartype)) + full_path = os.path.join(full_path, var, self._get_file_name(var, startdate)) + return full_path + + def _get_file_name(self, var, startdate): + if startdate: + if self.config.data_type != 'exp': + startdate = startdate[0:6] + return '{0}_{1}.nc'.format(var, startdate) + else: + return '{0}.nc'.format(var) + + def link_file(self, domain, var, cmor_var, startdate, member, chunk=None, grid=None, + frequency=None, year=None, date_str=None, move_old=False, vartype=VariableType.MEAN): + """ + Creates the link of a given file from the CMOR repository. + + :param cmor_var: + :param move_old: + :param date_str: + :param year: if frequency is yearly, this parameter is used to give the corresponding year + :type year: int + :param domain: CMOR domain + :type domain: str + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: str + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + # THREDDSManager does not require links + pass + + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param vartype: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :rtype: str + """ + var = self._get_final_var_name(box, var) + filepath = self.get_file_path(startdate, domain, var, frequency, vartype, grid, box) + Log.debug('{0} requested', filepath) + return self._get_file_from_storage(filepath) + + # noinspection PyUnusedLocal + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param region: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + if not frequency: + frequency = self.config.frequency + original_name = var + cmor_var = self.variable_list.get_variable(var) + if cmor_var: + var = cmor_var.short_name + final_name = var + + filepath = self.get_file_path(startdate, domain, final_name, frequency, vartype, box, grid) + netcdf_file = self._declare_generated_file(filepath, domain, final_name, cmor_var, self.config.data_convention, + region, diagnostic, grid, vartype, original_name) + netcdf_file.frequency = frequency + Log.debug('{0} will be generated', filepath) + return netcdf_file + diff --git a/earthdiagnostics/ocean/areamoc.py b/earthdiagnostics/ocean/areamoc.py index 3ff73c8f6cb4fd59dc85ad1fae0f47940a21d63c..80c18474a3cb5f3d163467a3f73e04a052edf20c 100644 --- a/earthdiagnostics/ocean/areamoc.py +++ b/earthdiagnostics/ocean/areamoc.py @@ -39,6 +39,8 @@ class AreaMoc(Diagnostic): alias = 'mocarea' "Diagnostic alias for the configuration file" + vsftmyz = 'vsftmyz' + def __init__(self, data_manager, startdate, member, chunk, basin, box): Diagnostic.__init__(self, data_manager) self.basin = basin @@ -86,15 +88,26 @@ class AreaMoc(Diagnostic): job_list.append(AreaMoc(diags.data_manager, startdate, member, chunk, options['basin'], box)) return job_list + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, AreaMoc.vsftmyz, + self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(ModelingRealms.ocean, AreaMoc.vsftmyz, + self.startdate, self.member, self.chunk, + box=self.box) + def compute(self): """ Runs the diagnostic """ nco = Utils.nco cdo = Utils.cdo + + temp = TempFile.get() temp2 = TempFile.get() - temp = self.data_manager.get_file(ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.chunk) + Utils.copy_file(self.variable_file.local_file, temp) handler = Utils.openCdf(temp) if 'i' in handler.dimensions: @@ -146,4 +159,4 @@ class AreaMoc(Diagnostic): nco.ncap2(input=temp2, output=temp2, options='-O -s "coslat[lat]=cos(lat[lat]*3.141592657/180.0)"') nco.ncwa(input=temp2, output=temp2, options='-w coslat -a lat') nco.ncks(input=temp2, output=temp2, options='-O -v vsftmyz,time') - self.send_file(temp2, ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.chunk, box=self.box) + self.results.set_local_file(temp2) diff --git a/earthdiagnostics/ocean/averagesection.py b/earthdiagnostics/ocean/averagesection.py index 96c1162f83f243269ba9985065366a39ef0ce923..8ca8abb0a03da07a2f5ea49b92e19f7ae838e2ba 100644 --- a/earthdiagnostics/ocean/averagesection.py +++ b/earthdiagnostics/ocean/averagesection.py @@ -1,10 +1,9 @@ # coding=utf-8 import os from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticIntOption, DiagnosticDomainOption, \ - DiagnosticVariableOption +from earthdiagnostics.diagnostic import * from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms +from earthdiagnostics.modelingrealm import ModelingRealm class AverageSection(Diagnostic): @@ -37,7 +36,7 @@ class AverageSection(Diagnostic): alias = 'avgsection' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, startdate, member, chunk, domain, variable, box): + def __init__(self, data_manager, startdate, member, chunk, domain, variable, box, grid): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -45,14 +44,15 @@ class AverageSection(Diagnostic): self.variable = variable self.domain = domain self.box = box + self.grid = grid def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.box == other.box def __str__(self): - return 'Average section Startdate: {0} Member: {1} Chunk: {2} Box: {3} ' \ - 'Variable: {4}:{5}'.format(self.startdate, self.member, self.chunk, self.box, self.domain, self.variable) + return 'Average section Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Box: {0.box} ' \ + 'Variable: {0.domain}:{0.variable} Grid: {0.grid}'.format(self) @classmethod def generate_jobs(cls, diags, options): @@ -65,12 +65,13 @@ class AverageSection(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticIntOption('min_lon'), DiagnosticIntOption('max_lon'), DiagnosticIntOption('min_lat'), DiagnosticIntOption('max_lat'), - DiagnosticDomainOption('domain', ModelingRealms.ocean)) + DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) box = Box() box.min_lon = options['min_lon'] @@ -80,20 +81,26 @@ class AverageSection(Diagnostic): job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(AverageSection(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], box)) + options['domain'], options['variable'], box, options['grid'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def declare_data_generated(self): + self.mean = self.declare_chunk(self.domain, self.variable + 'mean', self.startdate, self.member, self.chunk, + box=self.box, grid=self.grid) + def compute(self): """ Runs the diagnostic """ temp = TempFile.get() - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid='regular') + variable_file = self.variable_file.local_file Utils.cdo.zonmean(input='-sellonlatbox,{0},{1},{2},{3} {4}'.format(self.box.min_lon, self.box.max_lon, self.box.min_lat, self.box.max_lat, variable_file), output=temp) os.remove(variable_file) - self.send_file(temp, self.domain, self.variable + 'mean', self.startdate, self.member, self.chunk, - box=self.box, grid='regular') + self.mean.set_local_file(temp, rename_var='tos') diff --git a/earthdiagnostics/ocean/convectionsites.py b/earthdiagnostics/ocean/convectionsites.py index e2bcddf69181c540b0332eafbb7cd2b75f4fabcd..74271682754c1072f48ff2eb7e8d2f1735e76d68 100644 --- a/earthdiagnostics/ocean/convectionsites.py +++ b/earthdiagnostics/ocean/convectionsites.py @@ -38,8 +38,6 @@ class ConvectionSites(Diagnostic): self.member = member self.chunk = chunk self.model_version = model_version - self.required_vars = ['vsftbarot'] - self.generated_vars = ['gyres'] self.mlotst_handler = None def __str__(self): @@ -67,6 +65,12 @@ class ConvectionSites(Diagnostic): job_list.append(ConvectionSites(diags.data_manager, startdate, member, chunk, diags.model_version)) return job_list + def request_data(self): + self.mixed_layer = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.sites = self.declare_chunk(ModelingRealms.ocean, 'site', self.startdate, self.member, self.chunk) + def compute(self): """ Runs the diagnostic @@ -80,13 +84,13 @@ class ConvectionSites(Diagnostic): wedell = [225, 280, 1, 50] elif self.model_version in [Models.ECEARTH_3_0_O25L46, Models.ECEARTH_3_0_O25L75, + Models.ECEARTH_3_2_O25L75, Models.ECEARTH_3_2_O1L75, Models.GLORYS2_V1_O25L75]: raise Exception("Option convection not available yet for {0}".format(self.model_version)) else: raise Exception("Input grid {0} not recognized".format(self.model_version)) - mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, - self.chunk) + mlotst_file = self.mixed_layer.local_file output = TempFile.get() self.mlotst_handler = Utils.openCdf(mlotst_file) @@ -115,7 +119,7 @@ class ConvectionSites(Diagnostic): self.mlotst_handler.close() handler.close() - self.send_file(output, ModelingRealms.ocean, 'site', self.startdate, self.member, self.chunk) + self.sites.set_local_file(output) Log.info('Finished convection sites for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/cutsection.py b/earthdiagnostics/ocean/cutsection.py index 4f2fb2e9efd554fea353bd85a98b69ad16a292e1..d4e5f89c9e8c6fc43669b73a522937166d57e02c 100644 --- a/earthdiagnostics/ocean/cutsection.py +++ b/earthdiagnostics/ocean/cutsection.py @@ -50,6 +50,14 @@ class CutSection(Diagnostic): self.zonal = zonal self.value = value + self.box = Box() + if self.zonal: + self.box.max_lon = self.value + self.box.min_lon = self.value + else: + self.box.max_lat = self.value + self.box.min_lat = self.value + def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.domain == other.domain and self.variable == other.variable and self.zonal == other.zonal and \ @@ -71,10 +79,10 @@ class CutSection(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticBoolOption('zonal'), DiagnosticIntOption('value'), - DiagnosticDomainOption('domain', ModelingRealms.ocean)) + DiagnosticDomainOption(default_value=ModelingRealms.ocean)) options = cls.process_options(options, options_available) job_list = list() @@ -83,6 +91,13 @@ class CutSection(Diagnostic): options['domain'], options['variable'], options['zonal'], options['value'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.section = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + box=self.box) + def compute(self): """ Runs the diagnostic @@ -170,14 +185,6 @@ class CutSection(Diagnostic): file_var.missing_value = 1e20 handler.close() - box = Box() - if self.zonal: - box.max_lon = self.value - box.min_lon = self.value - else: - box.max_lat = self.value - box.min_lat = self.value - - self.send_file(temp, self.domain, self.variable, self.startdate, self.member, self.chunk, box=box) + self.section.set_local_file(temp) Log.info('Finished cut section for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) diff --git a/earthdiagnostics/ocean/gyres.py b/earthdiagnostics/ocean/gyres.py index adecfb0ca1a8ec2ea4e7c6285aa006b7cd5d90b8..b01971488d8e19b447c03ff114bcad32302955d0 100644 --- a/earthdiagnostics/ocean/gyres.py +++ b/earthdiagnostics/ocean/gyres.py @@ -39,8 +39,6 @@ class Gyres(Diagnostic): self.member = member self.chunk = chunk self.model_version = model_version - self.required_vars = ['vsftbarot'] - self.generated_vars = ['gyres'] self.var_vsftbarot = None def __eq__(self, other): @@ -48,7 +46,8 @@ class Gyres(Diagnostic): self.model_version == other.model_version def __str__(self): - return 'Gyres Startdate: {0} Member: {1} Chunk: {2}'.format(self.startdate, self.member, self.chunk) + return 'Gyres Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Model version: {0.model_version}'.format(self) @classmethod def generate_jobs(cls, diags, options): @@ -64,10 +63,17 @@ class Gyres(Diagnostic): if len(options) > 1: raise Exception('The gyres diagnostic has no options') job_list = list() + model_version = diags.config.experiment.model_version for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(Gyres(diags.data_manager, startdate, member, chunk, diags.model_version)) + job_list.append(Gyres(diags.data_manager, startdate, member, chunk, model_version)) return job_list + def request_data(self): + self.vsftbarot = self.request_chunk(ModelingRealms.ocean, 'vsftbarot', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.gyre = self.declare_chunk(ModelingRealms.ocean, 'gyre', self.startdate, self.member, self.chunk) + # noinspection PyPep8Naming def compute(self): """ @@ -86,15 +92,14 @@ class Gyres(Diagnostic): subtropInd = [320, 30, 110, 180] ACC = [1, 361, 1, 65] - elif self in [Models.ECEARTH_3_0_O25L46, Models.ECEARTH_3_0_O25L75, - Models.GLORYS2_V1_O25L75]: + elif self.model_version in [Models.ECEARTH_3_0_O25L46, Models.ECEARTH_3_0_O25L75, Models.GLORYS2_V1_O25L75, + Models.ECEARTH_3_2_O1L75, Models.ECEARTH_3_2_O25L75]: raise Exception("Option gyres not available yet for {0}".format(self.model_version)) else: raise Exception("Input grid {0} not recognized".format(self.model_version)) output = TempFile.get() - vsftbarot_file = self.data_manager.get_file(ModelingRealms.ocean, 'vsftbarot', self.startdate, - self.member, self.chunk) + vsftbarot_file = self.vsftbarot.local_file handler_original = Utils.openCdf(vsftbarot_file) self.var_vsftbarot = handler_original.variables['vsftbarot'] @@ -145,7 +150,7 @@ class Gyres(Diagnostic): handler.close() handler_original.close() - self.send_file(output, ModelingRealms.ocean, 'gyre', self.startdate, self.member, self.chunk) + self.gyre.set_local_file(output) Log.info('Finished gyres for startdate {0}, member {1}, chunk {2}', self.startdate, self.member, self.chunk) def _gyre(self, site, invert=False): diff --git a/earthdiagnostics/ocean/heatcontent.py b/earthdiagnostics/ocean/heatcontent.py index a9432167da130e3148b1e8d3f683b2b51967a603..31ac9268b1e302515744091ec3ab87b6d1c08a9a 100644 --- a/earthdiagnostics/ocean/heatcontent.py +++ b/earthdiagnostics/ocean/heatcontent.py @@ -48,17 +48,14 @@ class HeatContent(Diagnostic): self.box = box self.min_level = min_level self.max_level = max_level - self.required_vars = ['so', 'mlotst'] - self.generated_vars = ['scvertsum'] def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.basin == other.basin and self.mxloption == other.mxloption def __str__(self): - return 'Heat content Startdate: {0} Member: {1} Chunk: {2} Mixed layer: {3} Box: {4} ' \ - 'Basin: {5}'.format(self.startdate, self.member, self.chunk, self.mxloption, self.box, - self.basin.fullname) + return 'Heat content Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Mixed layer: {0.mxloption} ' \ + 'Box: {0.box} Basin: {0.basin}'.format(self) @classmethod def generate_jobs(cls, diags, options): @@ -149,17 +146,32 @@ class HeatContent(Diagnostic): handler.close() return depth_t, depth_w + def request_data(self): + self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + if self.mxloption != 0: + self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + if self.box.min_depth == 0: + # For cdftools, this is all levels + box_save = None + else: + box_save = self.box + + self.heatcsum = self.declare_chunk(ModelingRealms.ocean, 'heatcsum', self.startdate, self.member, self.chunk, + box=box_save, region=self.basin.fullname) + self.heatcmean = self.declare_chunk(ModelingRealms.ocean, 'heatcvmean', self.startdate, self.member, self.chunk, + box=box_save, region=self.basin.fullname) + def compute(self): """ Runs the diagnostic """ nco = Utils.nco - temperature_file = self.data_manager.get_file(ModelingRealms.ocean, 'thetao', self.startdate, - self.member, self.chunk) + temperature_file = TempFile.get() + Utils.copy_file(self.thetao.local_file, temperature_file) if self.mxloption != 0: - mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', self.startdate, - self.member, self.chunk) - nco.ncks(input=mlotst_file, output=temperature_file, options=('-A -v mlotst',)) + nco.ncks(input=self.mlotst.local_file, output=temperature_file, options=('-A -v mlotst',)) para = list() if self.min_level != 0: @@ -211,15 +223,7 @@ class HeatContent(Diagnostic): results.close() - if self.box.min_depth == 0: - # For cdftools, this is all levels - box_save = None - else: - box_save = self.box - Utils.setminmax(heatcsum_temp, 'heatcsum') - self.send_file(heatcsum_temp, ModelingRealms.ocean, 'heatcsum', self.startdate, self.member, self.chunk, - box=box_save, region=self.basin.name) + self.heatcsum.set_local_file(heatcsum_temp) Utils.setminmax(heatcvmean_temp, 'heatcvmean') - self.send_file(heatcvmean_temp, ModelingRealms.ocean, 'heatcvmean', self.startdate, self.member, self.chunk, - box=box_save, region=self.basin.name) + self.heatcmean.set_local_file(heatcvmean_temp) diff --git a/earthdiagnostics/ocean/heatcontentlayer.py b/earthdiagnostics/ocean/heatcontentlayer.py index affa0aeaa546cbcef5cc5754d8f2d41f51adca6c..c64b19021e455ecec86dfb501e9f7923a1df205d 100644 --- a/earthdiagnostics/ocean/heatcontentlayer.py +++ b/earthdiagnostics/ocean/heatcontentlayer.py @@ -89,6 +89,12 @@ class HeatContentLayer(Diagnostic): else: raise Exception('gdepw variable can not be found') + e3t_3d = e3t.shape != depth.shape + if e3t_3d: + mask = e3t_3d * mask + else: + e3t = e3t[0, :] + while len(depth.shape) < 4: depth = np.expand_dims(depth, -1) handler.close() @@ -129,15 +135,22 @@ class HeatContentLayer(Diagnostic): weight, min_level, max_level)) return job_list + def request_data(self): + self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.heatc = self.declare_chunk(ModelingRealms.ocean, 'heatc', self.startdate, self.member, self.chunk, + box=self.box) + def compute(self): """ Runs the diagnostic """ nco = Utils.nco + thetao_file = TempFile.get() results = TempFile.get() - thetao_file = self.data_manager.get_file(ModelingRealms.ocean, 'thetao', - self.startdate, self.member, self.chunk) + Utils.copy_file(self.thetao.local_file, thetao_file) handler = Utils.openCdf(thetao_file) Utils.convert_units(handler.variables['thetao'], 'K') @@ -155,4 +168,4 @@ class HeatContentLayer(Diagnostic): handler_results.close() Utils.setminmax(results, 'heatc') - self.send_file(results, ModelingRealms.ocean, 'heatc', self.startdate, self.member, self.chunk, box=self.box) + self.heatc.set_local_file(results) diff --git a/earthdiagnostics/ocean/interpolate.py b/earthdiagnostics/ocean/interpolate.py index 8784fe84feb4dd66811d0984ea0e94aedcdd6863..262f87676c896957bd0320752ae5163ec0bb8e50 100644 --- a/earthdiagnostics/ocean/interpolate.py +++ b/earthdiagnostics/ocean/interpolate.py @@ -84,27 +84,35 @@ class Interpolate(Diagnostic): :return: """ options_available = (DiagnosticOption('target_grid'), - DiagnosticVariableListOption('variable'), - DiagnosticDomainOption('domain', ModelingRealms.ocean), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variable'), + DiagnosticDomainOption(default_value=ModelingRealms.ocean), DiagnosticBoolOption('invert_lat', False), DiagnosticOption('original_grid', '')) options = cls.process_options(options, options_available) job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - for variable in options['variable']: + for var in options['variable']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append( Interpolate(diags.data_manager, startdate, member, chunk, - options['domain'], variable, options['target_grid'], + options['domain'], var, options['target_grid'], diags.config.experiment.model_version, options['invert_lat'], options['original_grid'])) return job_list + def request_data(self): + self.original = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.original_grid) + + def declare_data_generated(self): + self.regridded = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.original_grid) + variable_file = TempFile.get() + Utils.copy_file(self.original.local_file, variable_file) Utils.rename_variables(variable_file, {'i': 'x', 'j': 'y'}, must_exist=False, rename_dimension=True) cdo = Utils.cdo nco = Utils.nco @@ -148,7 +156,7 @@ class Interpolate(Diagnostic): if not has_levels: nco.ncks(input=temp2, output=temp2, options=('-v {0},lat,lon,time'.format(self.variable),)) - self.send_file(temp2, self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) + self.regridded.set_local_file(temp2) def _get_level_file(self, lev): if not self.tempTemplate: @@ -164,12 +172,16 @@ class Interpolate(Diagnostic): nco.ncwa(input=temp, output=temp, options=('-h -a lev',)) else: shutil.copy(input_file, temp) + + weights_file = '/esnas/autosubmit/con_files/weigths/{0}/rmp_{0}_to_{1}_lev{2}.nc'.format(self.model_version, + self.grid, lev + 1) + if not os.path.isfile(weights_file): + raise Exception('Level {0} weights file does not exist for model {1} ' + 'and grid {2}'.format(lev+1, self.model_version, self.grid)) namelist_file = TempFile.get(suffix='') scrip_use_in = open(namelist_file, 'w') scrip_use_in.writelines("&remap_inputs\n") - scrip_use_in.writelines(" remap_wgt = '/esnas/autosubmit/con_files/" - "weigths/{0}/rmp_{0}_to_{1}_lev{2}.nc'\n".format(self.model_version, self.grid, - lev + 1)) + scrip_use_in.writelines(" remap_wgt = '{0}'\n".format(weights_file)) scrip_use_in.writelines(" infile = '{0}'\n".format(temp)) scrip_use_in.writelines(" invertlat = FALSE\n") scrip_use_in.writelines(" var = '{0}'\n".format(self.variable)) @@ -178,7 +190,7 @@ class Interpolate(Diagnostic): scrip_use_in.writelines("/\n") scrip_use_in.close() Utils.execute_shell_command('/home/Earth/jvegas/pyCharm/cfutools/interpolation/scrip_use ' - '{0}'.format(namelist_file), Log.DEBUG) + '{0}'.format(namelist_file), Log.INFO) os.remove(namelist_file) nco.ncecat(input=temp, output=temp, options=("-h",)) shutil.move(temp, self._get_level_file(lev)) diff --git a/earthdiagnostics/ocean/interpolatecdo.py b/earthdiagnostics/ocean/interpolatecdo.py index 432cd9ee591b958c8897a83d57ed021f63454abb..0bc7709b51a8e5b4162562b8be1b19778d273e1d 100644 --- a/earthdiagnostics/ocean/interpolatecdo.py +++ b/earthdiagnostics/ocean/interpolatecdo.py @@ -1,10 +1,11 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import * -from earthdiagnostics.utils import Utils, TempFile +import os -from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms import numpy as np -import os + +from earthdiagnostics.diagnostic import * +from earthdiagnostics.modelingrealm import ModelingRealm, ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class InterpolateCDO(Diagnostic): @@ -62,13 +63,13 @@ class InterpolateCDO(Diagnostic): def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.model_version == other.model_version and self.domain == other.domain and \ - self.variable == other.variable and self.grid == other.grid and self.original_grid == other.original_grid + self.variable == other.variable and self.mask_oceans == other.mask_oceans and self.grid == other.grid and \ + self.original_grid == other.original_grid def __str__(self): - return 'Interpolate with CDO Startdate: {0} Member: {1} Chunk: {2} ' \ - 'Variable: {3}:{4} Target grid: {5} ' \ - 'Model: {6}' .format(self.startdate, self.member, self.chunk, self.domain, self.variable, self.grid, - self.model_version) + return 'Interpolate with CDO Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} ' \ + 'Variable: {0.domain}:{0.variable} Target grid: {0.grid} Original grid: {0.original_grid} ' \ + 'Mask ocean: {0.mask_oceans} Model: {0.model_version}'.format(self) @classmethod def generate_jobs(cls, diags, options): @@ -81,9 +82,9 @@ class InterpolateCDO(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), + options_available = (DiagnosticDomainOption(default_value=ModelingRealms.ocean), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), DiagnosticOption('target_grid', diags.config.experiment.atmos_grid.lower()), - DiagnosticDomainOption('domain', ModelingRealms.ocean), DiagnosticChoiceOption('method', InterpolateCDO.METHODS, InterpolateCDO.BILINEAR), DiagnosticBoolOption('mask_oceans', True), DiagnosticOption('original_grid', ''), @@ -96,6 +97,17 @@ class InterpolateCDO(Diagnostic): job_list = list() weights = TempFile.get() method = options['method'].lower() + cls._compute_weights(diags, method, options, target_grid, weights) + for var in options['variables']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(InterpolateCDO(diags.data_manager, startdate, member, chunk, + options['domain'], var, target_grid, + diags.config.experiment.model_version, options['mask_oceans'], + options['original_grid'], weights)) + return job_list + + @classmethod + def _compute_weights(cls, diags, method, options, target_grid, weights): if options['weights_from_mask']: temp = cls.get_sample_grid_file() else: @@ -112,13 +124,6 @@ class InterpolateCDO(Diagnostic): Utils.cdo.gencon2(target_grid, input=temp, output=weights) os.remove(temp) - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(InterpolateCDO(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], target_grid, - diags.config.experiment.model_version, options['mask_oceans'], - options['original_grid'], weights)) - return job_list - @classmethod def get_sample_grid_file(cls): temp = TempFile.get() @@ -176,20 +181,28 @@ class InterpolateCDO(Diagnostic): target_grid = 't340grid' return target_grid + def request_data(self): + self.original = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.original_grid) + + def declare_data_generated(self): + self.regridded = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.original_grid) + variable_file = TempFile.get() + Utils.copy_file(self.original.local_file, variable_file) Utils.rename_variables(variable_file, {'jpib': 'i', 'jpjb': 'j', 'x': 'i', 'y': 'j', 'time_counter': 'time', 't': 'time', 'SSTK_ens0': 'tos', 'SSTK_ens1': 'tos', 'SSTK_ens2': 'tos', 'nav_lat': 'lat', 'nav_lon': 'lon'}, must_exist=False, rename_dimension=True) - handler = Utils.openCdf(variable_file) var = handler.variables[self.variable] + units = var.units coordinates = list() for dim in var.dimensions: if dim == 'i': @@ -214,7 +227,13 @@ class InterpolateCDO(Diagnostic): temp = TempFile.get() Utils.cdo.remap(','.join((self.grid.split('_')[0], self.weights)), input=variable_file, output=temp) - self.send_file(temp, self.domain, self.variable, self.startdate, self.member, self.chunk, grid=self.grid) + + handler = Utils.openCdf(temp) + handler.variables[self.variable].units = units + handler.close() + + self.regridded.set_local_file(temp) + diff --git a/earthdiagnostics/ocean/mask_land.py b/earthdiagnostics/ocean/mask_land.py index 7f4271ca60b9b298f21cde77883ba58befec9faa..a7af9aaa04719fc12837178e4a8af396dc3121f5 100644 --- a/earthdiagnostics/ocean/mask_land.py +++ b/earthdiagnostics/ocean/mask_land.py @@ -1,7 +1,7 @@ # coding=utf-8 -from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, \ +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableListOption, \ DiagnosticDomainOption, DiagnosticChoiceOption, DiagnosticOption -from earthdiagnostics.utils import Utils +from earthdiagnostics.utils import Utils, TempFile import numpy as np @@ -25,7 +25,6 @@ class MaskLand(Diagnostic): """ alias = 'maskland' - "Diagnostic alias for the configuration file" def __init__(self, data_manager, startdate, member, chunk, domain, variable, mask, grid): Diagnostic.__init__(self, data_manager) @@ -57,33 +56,51 @@ class MaskLand(Diagnostic): :return: """ options_available = (DiagnosticDomainOption('domain'), - DiagnosticVariableOption('variable'), + DiagnosticVariableListOption(diags.data_manager.config.var_manager, 'variables'), DiagnosticChoiceOption('cell', ('t', 'u', 'v', 'f', 'w'), 't'), DiagnosticOption('grid', '')) options = cls.process_options(options, options_available) - mask_file = Utils.openCdf('mask.nc') cell_point = options['cell'] # W and T share the same mask if cell_point == 'w': cell_point = 't' - mask = mask_file.variables['{0}mask'.format(cell_point)][:].astype(float) - mask[mask == 0] = np.nan + + mask = cls._get_mask(cell_point) job_list = list() - for startdate, member, chunk in diags.config.experiment.get_chunk_list(): - job_list.append(MaskLand(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], mask, options['grid'])) + for var in options['variables']: + for startdate, member, chunk in diags.config.experiment.get_chunk_list(): + job_list.append(MaskLand(diags.data_manager, startdate, member, chunk, + options['domain'], var, mask, options['grid'])) return job_list + @classmethod + def _get_mask(cls, cell_point): + mask_file = Utils.openCdf('mask.nc') + mask = mask_file.variables['{0}mask'.format(cell_point)][:].astype(float) + mask[mask == 0] = np.nan + mask_file.close() + return mask + + "Diagnostic alias for the configuration file" + + def request_data(self): + self.var_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def declare_data_generated(self): + self.masked_file = self.declare_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, - self.chunk, grid=self.grid) + temp = TempFile.get() + Utils.copy_file(self.var_file.local_file, temp) - handler = Utils.openCdf(variable_file) + handler = Utils.openCdf(temp) if 'lev' not in handler.dimensions: mask = self.mask[:, 0, ...] else: @@ -91,6 +108,5 @@ class MaskLand(Diagnostic): handler.variables[self.variable][:] *= mask handler.close() - self.send_file(variable_file, self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + self.masked_file.set_local_file(temp) diff --git a/earthdiagnostics/ocean/maxmoc.py b/earthdiagnostics/ocean/maxmoc.py index 291a28b8fed4a39de6426995b5e3483c7646e1cf..2be7110e75edca01ff46cc3cbc877e5807990621 100644 --- a/earthdiagnostics/ocean/maxmoc.py +++ b/earthdiagnostics/ocean/maxmoc.py @@ -1,13 +1,11 @@ # coding=utf-8 import netCDF4 import numpy as np -import os from bscearth.utils.log import Log from earthdiagnostics.constants import Basins from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticBasinOption, DiagnosticFloatOption -from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.utils import Utils +from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.variable_type import VariableType @@ -39,14 +37,14 @@ class MaxMoc(Diagnostic): alias = 'mocmax' "Diagnostic alias for the configuration file" + vsftmyz = 'vsftmyz' + def __init__(self, data_manager, startdate, member, year, basin, box): Diagnostic.__init__(self, data_manager) self.basin = basin self.startdate = startdate self.member = member self.year = year - self.required_vars = ['vo'] - self.generated_vars = ['vsftmyz'] self.box = box def __str__(self): @@ -91,13 +89,37 @@ class MaxMoc(Diagnostic): job_list.append(MaxMoc(diags.data_manager, startdate, member, year, options['basin'], box)) return job_list + def request_data(self): + self.variable_file = self.request_year(ModelingRealms.ocean, MaxMoc.vsftmyz, + self.startdate, self.member, self.year) + + def declare_data_generated(self): + + self.results = {'vsftmyzmax': self.declare_year(ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, + self.year, box=self.box, vartype=VariableType.STATISTIC), + 'vsftmyzmaxlev': self.declare_year(ModelingRealms.ocean, 'vsftmyzmaxlev', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC), + 'vsftmyzmaxlat': self.declare_year(ModelingRealms.ocean, 'vsftmyzmaxlat', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC), + 'vsftmyzmin': self.declare_year(ModelingRealms.ocean, 'vsftmyzmin', self.startdate, self.member, + self.year, box=self.box, vartype=VariableType.STATISTIC), + 'vsftmyzminlev': self.declare_year(ModelingRealms.ocean, 'vsftmyzminlev', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC), + 'vsftmyzminlat': self.declare_year(ModelingRealms.ocean, 'vsftmyzminlat', self.startdate, + self.member, self.year, box=self.box, + vartype=VariableType.STATISTIC)} + def compute(self): """ Runs the diagnostic """ nco = Utils.nco - temp = self.data_manager.get_year(ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.year) + temp = TempFile.get() + Utils.copy_file(self.variable_file.local_file, temp) handler = Utils.openCdf(temp) if 'i' in handler.dimensions: @@ -127,7 +149,6 @@ class MaxMoc(Diagnostic): Log.info('Computing year {0}', str(self.year)) moc = handler.variables['vsftmyz'][:, lev_inds, lat_inds, basin_index] handler.close() - os.remove(temp) moc = np.mean(moc, 0) @@ -148,7 +169,7 @@ class MaxMoc(Diagnostic): Log.info('Maximum {0} Sv, latitude {1} depth {2} m', maximum, max_lat, max_lev) Log.info('Minimum {0} Sv, latitude {1} depth {2} m', minimum, min_lat, min_lev) - handler = self._create_output_file(temp) + handler, temp = self._create_output_file() var = handler.createVariable('vsftmyzmax', float, ('time',)) var.long_name = 'Maximum_Overturning' var.units = 'Sverdrup' @@ -156,10 +177,9 @@ class MaxMoc(Diagnostic): var.valid_max = 1000. var[0] = maximum handler.close() - self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency=Frequencies.yearly, year=self.year, vartype=VariableType.STATISTIC) + self.results['vsftmyzmax'].set_local_file(temp) - handler = self._create_output_file(temp) + handler, temp = self._create_output_file() var = handler.createVariable('vsftmyzmaxlat', float, ('time',)) var.long_name = 'Latitude_of_Maximum_Overturning' var.units = 'Degrees' @@ -167,10 +187,9 @@ class MaxMoc(Diagnostic): var.valid_max = 90. var[0] = max_lat handler.close() - self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency=Frequencies.yearly, year=self.year, vartype=VariableType.STATISTIC) + self.results['vsftmyzmaxlat'].set_local_file(temp) - handler = self._create_output_file(temp) + handler, temp = self._create_output_file() var = handler.createVariable('vsftmyzmaxlev', float, ('time',)) var.long_name = 'Depth_of_Maximum_Overturning' var.units = 'Meters' @@ -178,10 +197,9 @@ class MaxMoc(Diagnostic): var.valid_max = 10000. var[0] = max_lev handler.close() - self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency=Frequencies.yearly, year=self.year, vartype=VariableType.STATISTIC) + self.results['vsftmyzmaxlev'].set_local_file(temp) - handler = self._create_output_file(temp) + handler, temp = self._create_output_file() var = handler.createVariable('vsftmyzmin', float, ('time',)) var.long_name = 'Minimum_Overturning' var.units = 'Sverdrup' @@ -189,10 +207,9 @@ class MaxMoc(Diagnostic): var.valid_max = 1000. var[0] = minimum handler.close() - self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency=Frequencies.yearly, year=self.year, vartype=VariableType.STATISTIC) + self.results['vsftmyzmin'].set_local_file(temp) - handler = self._create_output_file(temp) + handler, temp = self._create_output_file() var = handler.createVariable('vsftmyzminlat', float, ('time',)) var.long_name = 'Latitude_of_Minimum_Overturning' var.units = 'Degrees' @@ -200,10 +217,9 @@ class MaxMoc(Diagnostic): var.valid_max = 90. var[0] = min_lat handler.close() - self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency=Frequencies.yearly, year=self.year, vartype=VariableType.STATISTIC) + self.results['vsftmyzminlat'].set_local_file(temp) - handler = self._create_output_file(temp) + handler, temp = self._create_output_file() var = handler.createVariable('vsftmyzminlev', float, ('time',)) var.long_name = 'Depth_of_Minimum_Overturning' var.units = 'Meters' @@ -211,14 +227,14 @@ class MaxMoc(Diagnostic): var.valid_max = 10000. var[0] = min_lev handler.close() - self.send_file(temp, ModelingRealms.ocean, 'vsftmyzmax', self.startdate, self.member, box=self.box, - frequency=Frequencies.yearly, year=self.year, vartype=VariableType.STATISTIC) + self.results['vsftmyzminlev'].set_local_file(temp) - def _create_output_file(self, temp): + def _create_output_file(self): + temp = TempFile.get() handler = netCDF4.Dataset(temp, 'w') handler.createDimension('time') time = handler.createVariable('time', 'i2', ('time',)) time.calendar = 'gregorian' time.units = 'days since January 1, {0}'.format(self.year) - return handler + return handler, temp diff --git a/earthdiagnostics/ocean/mixedlayerheatcontent.py b/earthdiagnostics/ocean/mixedlayerheatcontent.py index cc7e3b5b3d9cd551eecde05f15658028381a78bf..e81624d2555f7eb4db05dcdbccaff430d2878e13 100644 --- a/earthdiagnostics/ocean/mixedlayerheatcontent.py +++ b/earthdiagnostics/ocean/mixedlayerheatcontent.py @@ -63,16 +63,20 @@ class MixedLayerHeatContent(Diagnostic): job_list.append(MixedLayerHeatContent(diags.data_manager, startdate, member, chunk)) return job_list + def request_data(self): + self.thetao = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.ohcsum = self.declare_chunk(ModelingRealms.ocean, 'ohcvsumlotst', self.startdate, self.member, self.chunk) + def compute(self): """ Runs the diagnostic """ - temperature_file = self.data_manager.get_file(ModelingRealms.ocean, 'thetao', - self.startdate, self.member, self.chunk) - mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', - self.startdate, self.member, self.chunk) - - Utils.nco.ncks(input=mlotst_file, output=temperature_file, options=('-A -v mlotst',)) + temperature_file = TempFile.get() + Utils.copy_file(self.thetao.local_file, temperature_file) + Utils.nco.ncks(input=self.mlotst.local_file, output=temperature_file, options=('-A -v mlotst',)) temp = TempFile.get() cdftools.run('cdfmxlheatc', input=temperature_file, output=temp) @@ -81,4 +85,4 @@ class MixedLayerHeatContent(Diagnostic): Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlheatc': 'ohcvsumlotst'}, False, True) Utils.setminmax(temp, 'ohcvsumlotst') - self.send_file(temp, ModelingRealms.ocean, 'ohcvsumlotst', self.startdate, self.member, self.chunk) + self.ohcsum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mixedlayersaltcontent.py b/earthdiagnostics/ocean/mixedlayersaltcontent.py index 0a0ea4782aca1e2a2b2e2f2ad6c98dea9bd15f41..ad350961fdcabed51b378f9dd2fd26bc79bef164 100644 --- a/earthdiagnostics/ocean/mixedlayersaltcontent.py +++ b/earthdiagnostics/ocean/mixedlayersaltcontent.py @@ -61,15 +61,20 @@ class MixedLayerSaltContent(Diagnostic): job_list.append(MixedLayerSaltContent(diags.data_manager, startdate, member, chunk)) return job_list + def request_data(self): + self.so = self.request_chunk(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) + self.mlotst = self.request_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.sosum = self.declare_chunk(ModelingRealms.ocean, 'scvsummlotst', self.startdate, self.member, self.chunk) + def compute(self): """ Runs the diagnostic """ - salinity_file = self.data_manager.get_file(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) - mlotst_file = self.data_manager.get_file(ModelingRealms.ocean, 'mlotst', - self.startdate, self.member, self.chunk) - - Utils.nco.ncks(input=mlotst_file, output=salinity_file, options=('-A -v mlotst',)) + salinity_file = TempFile.get() + Utils.copy_file(self.so.local_file, salinity_file) + Utils.nco.ncks(input=self.mlotst.local_file, output=salinity_file, options=('-A -v mlotst',)) temp = TempFile.get() cdftools.run('cdfmxlsaltc', input=salinity_file, output=temp) @@ -77,4 +82,4 @@ class MixedLayerSaltContent(Diagnostic): Utils.rename_variables(temp, {'x': 'i', 'y': 'j', 'somxlsaltc': 'scvsummlotst'}, False, True) Utils.setminmax(temp, 'scvsummlotst') - self.send_file(temp, ModelingRealms.ocean, 'scvsummlotst', self.startdate, self.member, self.chunk) + self.sosum.set_local_file(temp) diff --git a/earthdiagnostics/ocean/moc.py b/earthdiagnostics/ocean/moc.py index 93d9c44705d3f53c58a0c4759a6d68cfefcbff15..7686e84b3a616984c53e070840220220e6a2860d 100644 --- a/earthdiagnostics/ocean/moc.py +++ b/earthdiagnostics/ocean/moc.py @@ -32,6 +32,8 @@ class Moc(Diagnostic): alias = 'moc' "Diagnostic alias for the configuration file" + vsftmyz = 'vsftmyz' + def __init__(self, data_manager, startdate, member, chunk): Diagnostic.__init__(self, data_manager) self.startdate = startdate @@ -64,17 +66,21 @@ class Moc(Diagnostic): job_list.append(Moc(diags.data_manager, startdate, member, chunk)) return job_list + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(ModelingRealms.ocean, Moc.vsftmyz, self.startdate, self.member, self.chunk) + def compute(self): """ Runs the diagnostic """ temp = TempFile.get() - input_file = self.data_manager.get_file(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) - Log.debug('Computing MOC') - cdftools.run('cdfmoc', input=input_file, output=temp) - Utils.nco.ncks(input=input_file, output=temp, options=('-A -v lev',)) + cdftools.run('cdfmoc', input=self.variable_file.local_file, output=temp) + Utils.nco.ncks(input=self.variable_file.local_file, output=temp, options=('-A -v lev',)) Utils.convert2netcdf4(temp) Log.debug('Reformatting variables') @@ -112,4 +118,4 @@ class Moc(Diagnostic): options=('-O -x -v zomsfglo,zomsfatl,zomsfpac,zomsfinp,zomsfind,zomsfinp0',)) Utils.setminmax(temp, 'vsftmyz') - self.send_file(temp, ModelingRealms.ocean, 'vsftmyz', self.startdate, self.member, self.chunk) + self.results.set_local_file(temp) diff --git a/earthdiagnostics/ocean/mxl.py b/earthdiagnostics/ocean/mxl.py index b3ac1eb6336e4bcff3fcba30993c8a118a070664..fe531b76cd0ce4e30a27a7eee4bee4206368ffa8 100644 --- a/earthdiagnostics/ocean/mxl.py +++ b/earthdiagnostics/ocean/mxl.py @@ -54,16 +54,19 @@ class Mxl(Diagnostic): job_list.append(Mxl(diags.data_manager, startdate, member, chunk)) return job_list + def request_data(self): + self.thetao_file = self.request_chunk(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) + self.so_file = self.request_chunk(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.mlotst_file = self.declare_chunk(ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk) + def compute(self): """ Runs the diagnostic """ temp = TempFile.get() - thetao = self.data_manager.get_file(ModelingRealms.ocean, 'thetao', self.startdate, self.member, self.chunk) - so = self.data_manager.get_file(ModelingRealms.ocean, 'so', self.startdate, self.member, self.chunk) - cdftools.run('cdfmxl', input=[thetao, so], output=temp, options='-nc4') - os.remove(thetao) - os.remove(so) + cdftools.run('cdfmxl', input=[self.thetao_file, self.so_file], output=temp, options='-nc4') temp2 = TempFile.get() source = Utils.openCdf(temp) destiny = Utils.openCdf(temp2, 'w') @@ -72,6 +75,5 @@ class Mxl(Diagnostic): Utils.copy_variable(source, destiny, 'lon') source.close() destiny.close() - self.send_file(temp2, ModelingRealms.ocean, 'mlotst', self.startdate, self.member, self.chunk, - rename_var='somxl010') + self.mlotst_file.set_local_file(temp2, rename_var='somxl010') os.remove(temp) diff --git a/earthdiagnostics/ocean/psi.py b/earthdiagnostics/ocean/psi.py index 072bfb875b0db305a08263a2405d71e85aa9d50d..fd1ee553ead2b07aa9bca2191f6fbb98194d5b71 100644 --- a/earthdiagnostics/ocean/psi.py +++ b/earthdiagnostics/ocean/psi.py @@ -28,13 +28,13 @@ class Psi(Diagnostic): alias = 'psi' "Diagnostic alias for the configuration file" + vsftbarot = 'vsftbarot' + def __init__(self, data_manager, startdate, member, chunk): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member self.chunk = chunk - self.required_vars = ['vo', 'uo'] - self.generated_vars = ['vsftbarot'] def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk @@ -60,14 +60,19 @@ class Psi(Diagnostic): job_list.append(Psi(diags.data_manager, startdate, member, chunk)) return job_list + def request_data(self): + self.uo = self.request_chunk(ModelingRealms.ocean, 'uo', self.startdate, self.member, self.chunk) + self.vo = self.request_chunk(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.psi = self.declare_chunk(ModelingRealms.ocean, Psi.vsftbarot, self.startdate, self.member, self.chunk) + def compute(self): """ Runs the diagnostic """ temp = TempFile.get() - input_file_u = self.data_manager.get_file(ModelingRealms.ocean, 'uo', self.startdate, self.member, self.chunk) - input_file_v = self.data_manager.get_file(ModelingRealms.ocean, 'vo', self.startdate, self.member, self.chunk) - cdftools.run('cdfpsi', input=[input_file_u, input_file_v], output=temp, options='-mean -mask') - Utils.rename_variable(temp, 'sobarstf', 'vsftbarot') - Utils.setminmax(temp, 'vsftbarot') - self.send_file(temp, ModelingRealms.ocean, 'vsftbarot', self.startdate, self.member, self.chunk) + cdftools.run('cdfpsi', input=[self.uo.local_file, self.vo.local_file], output=temp, options='-mean -mask') + Utils.rename_variable(temp, 'sobarstf', Psi.vsftbarot) + Utils.setminmax(temp, Psi.vsftbarot) + self.psi.set_local_file(temp) diff --git a/earthdiagnostics/ocean/regionmean.py b/earthdiagnostics/ocean/regionmean.py index 4a2209f5cb95d900c8ab0b9bc535bacb376e4b04..1d6cebb8d7961a60f2ce172b5cfd0e5a5d8727b9 100644 --- a/earthdiagnostics/ocean/regionmean.py +++ b/earthdiagnostics/ocean/regionmean.py @@ -6,8 +6,8 @@ from earthdiagnostics.box import Box from earthdiagnostics.constants import Basins from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticIntOption, DiagnosticDomainOption, \ DiagnosticBoolOption, DiagnosticBasinOption, DiagnosticVariableOption -from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.utils import Utils, TempFile class RegionMean(Diagnostic): @@ -51,16 +51,16 @@ class RegionMean(Diagnostic): self.basin = basin self.variance = variance self.grid = grid - self.required_vars = [variable] - self.generated_vars = [variable + 'vmean'] + self.declared = {} def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ self.box == other.box and self.variable == other.variable def __str__(self): - return 'Region mean Startdate: {0} Member: {1} Chunk: {2} Variable: {3} ' \ - 'Box: {4}'.format(self.startdate, self.member, self.chunk, self.variable, self.box) + return 'Region mean Startdate: {0.startdate} Member: {0.member} Chunk: {0.chunk} Variable: {0.variable} ' \ + 'Grid point: {0.grid_point} Box: {0.box} Save 3D: {0.save3d} Save variance: {0.variance} ' \ + 'Original grid: {0.grid}'.format(self) @classmethod def generate_jobs(cls, diags, options): @@ -73,9 +73,9 @@ class RegionMean(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticDomainOption('domain'), - DiagnosticVariableOption('variable'), - DiagnosticOption('grid_point', ''), + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticOption('grid_point', 'T'), DiagnosticBasinOption('basin', Basins().Global), DiagnosticIntOption('min_depth', 0), DiagnosticIntOption('max_depth', 0), @@ -95,14 +95,31 @@ class RegionMean(Diagnostic): options['save3D'], options['basin'], options['variance'], options['grid'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk, + grid=self.grid) + + def declare_data_generated(self): + if self.box.min_depth == 0: + # To cdftools, this means all levels + box_save = None + else: + box_save = self.box + + self.declare_var('mean', False, box_save) + self.declare_var('mean', True, box_save) + + if self.variance: + self.declare_var('var', False, box_save) + self.declare_var('var', True, box_save) + def compute(self): """ Runs the diagnostic """ mean_file = TempFile.get() - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk, - grid=self.grid) + variable_file = self.variable_file.local_file handler = Utils.openCdf(variable_file) self.save3d &= 'lev' in handler.dimensions @@ -119,22 +136,16 @@ class RegionMean(Diagnostic): cdftools.run('cdfmean', input=variable_file, output=mean_file, options=cdfmean_options) Utils.rename_variables(mean_file, {'gdept': 'lev', 'gdepw': 'lev'}, must_exist=False, rename_dimension=True) - if self.box.min_depth == 0: - # To cdftools, this means all levels - box_save = None - else: - box_save = self.box - - self.send_var('mean', False, box_save, mean_file) - self.send_var('mean', True, box_save, mean_file) + self.send_var('mean', False, mean_file) + self.send_var('mean', True, mean_file) if self.variance: - self.send_var('var', False, box_save, mean_file) - self.send_var('var', True, box_save, mean_file) + self.send_var('var', False, mean_file) + self.send_var('var', True, mean_file) os.remove(mean_file) - def send_var(self, var, threed, box_save, mean_file): + def send_var(self, var, threed, mean_file): if threed: if not self.save3d: return False @@ -147,7 +158,17 @@ class RegionMean(Diagnostic): levels = '' temp2 = TempFile.get() - Utils.nco.ncks(input=mean_file, output=temp2, options=('-O -v {0},lat,lon{1}'.format(original_name, levels),)) - self.send_file(temp2, ModelingRealms.ocean, final_name, self.startdate, self.member, self.chunk, - box=box_save, rename_var=original_name, region=self.basin, grid=self.grid) + Utils.nco.ncks(input=mean_file, output=temp2, options=('-v {0},lat,lon{1}'.format(original_name, levels),)) + self.declared[final_name].set_local_file(temp2, rename_var=original_name) + + def declare_var(self, var, threed, box_save): + if threed: + if not self.save3d: + return False + final_name = '{1}3d{0}'.format(var, self.variable) + else: + final_name = '{1}{0}'.format(var, self.variable) + + self.declared[final_name] = self.declare_chunk(ModelingRealms.ocean, final_name, self.startdate, self.member, + self.chunk, box=box_save, region=self.basin, grid=self.grid) diff --git a/earthdiagnostics/ocean/rotation.py b/earthdiagnostics/ocean/rotation.py index ed949cc6447508f66f713612e48d846d504f7a44..bdd6132c4cba6c04878ded48c8961eef25b88f2e 100644 --- a/earthdiagnostics/ocean/rotation.py +++ b/earthdiagnostics/ocean/rotation.py @@ -1,7 +1,6 @@ # coding=utf-8 import shutil from bscearth.utils.log import Log -import os from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticVariableOption from earthdiagnostics.utils import Utils, TempFile @@ -26,8 +25,6 @@ class Rotation(Diagnostic): :type member: int :param chunk: chunk's number :type chunk: int - :param variable: variable's name - :type variable: str :param domain: variable's domain :type domain: Domain """ @@ -66,9 +63,9 @@ class Rotation(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variableu'), - DiagnosticVariableOption('variablev'), - DiagnosticDomainOption('domain', ModelingRealms.ocean), + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager, 'variableu'), + DiagnosticVariableOption(diags.data_manager.config.var_manager, 'variablev'), + DiagnosticDomainOption(default_value=ModelingRealms.ocean), DiagnosticOption('executable', '/home/Earth/jvegas/pyCharm/cfutools/interpolation/rotateUVorca')) options = cls.process_options(options, options_available) @@ -80,14 +77,22 @@ class Rotation(Diagnostic): options['executable'])) return job_list + def request_data(self): + self.ufile = self.request_chunk(self.domain, self.variableu, self.startdate, self.member, self.chunk) + self.vfile = self.request_chunk(self.domain, self.variablev, self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.urotated_file = self.declare_chunk(self.domain, self.variableu, self.startdate, self.member, self.chunk, + grid='rotated') + self.vrotated_file = self.declare_chunk(self.domain, self.variablev, self.startdate, self.member, self.chunk, + grid='rotated') + def compute(self): """ Runs the diagnostic """ - self.ufile = self.data_manager.get_file(self.domain, self.variableu, self.startdate, self.member, self.chunk) - self.vfile = self.data_manager.get_file(self.domain, self.variablev, self.startdate, self.member, self.chunk) - handler = Utils.openCdf(self.ufile) + handler = Utils.openCdf(self.ufile.local_file) if 'lev' in handler.dimensions: self.num_levels = handler.dimensions['lev'].size self.has_levels = True @@ -102,20 +107,17 @@ class Rotation(Diagnostic): urotated = self._merge_levels(self.variableu, 'u') vrotated = self._merge_levels(self.variablev, 'v') - ufile_handler = Utils.openCdf(self.ufile) + ufile_handler = Utils.openCdf(self.ufile.local_file) self._add_metadata_and_vars(ufile_handler, urotated, self.variableu) ufile_handler.close() - os.remove(self.ufile) + self.urotated_file.set_local_file(urotated) - vfile_handler = Utils.openCdf(self.vfile) + vfile_handler = Utils.openCdf(self.vfile.local_file) self._add_metadata_and_vars(vfile_handler, vrotated, self.variablev) vfile_handler.close() - os.remove(self.vfile) - - self.send_file(urotated, self.domain, self.variableu, self.startdate, self.member, self.chunk, grid='rotated') - self.send_file(vrotated, self.domain, self.variablev, self.startdate, self.member, self.chunk, grid='rotated') + self.vrotated_file.set_local_file(urotated) - def _merge_levels(self, var, direction): + def _merge_levels(self, var, direction): temp = TempFile.get() if self.has_levels: Utils.nco.ncecat(input=self._get_level_file(0, direction), output=temp, @@ -131,8 +133,8 @@ class Rotation(Diagnostic): return temp def _rotate_level(self, lev): - ufile = self._extract_level(self.ufile, self.variableu, lev) - vfile = self._extract_level(self.vfile, self.variablev, lev) + ufile = self._extract_level(self.ufile.local_file, self.variableu, lev) + vfile = self._extract_level(self.vfile.local_file, self.variablev, lev) namelist_file = self._create_namelist(ufile, self._get_level_file(lev, 'u'), vfile, self._get_level_file(lev, 'v')) Utils.execute_shell_command('{0} {1}'.format(self.executable, namelist_file), Log.INFO) diff --git a/earthdiagnostics/ocean/siasiesiv.py b/earthdiagnostics/ocean/siasiesiv.py index 8a586291030d49e03c44044b4a185bf23904ca5d..d16295a458be9d6b8dec18a8a5a71720f07ba745 100644 --- a/earthdiagnostics/ocean/siasiesiv.py +++ b/earthdiagnostics/ocean/siasiesiv.py @@ -12,7 +12,6 @@ import numpy as np from earthdiagnostics.modelingrealm import ModelingRealms from earthdiagnostics.constants import Basins -from earthdiagnostics.variable import VariableManager class Siasiesiv(Diagnostic): @@ -55,8 +54,7 @@ class Siasiesiv(Diagnostic): self.member = member self.chunk = chunk self.mask = mask - self.required_vars = ['sit', 'sic'] - self.generated_vars = ['siextents', 'sivols', 'siareas', 'siextentn', 'sivoln', 'siarean'] + self.generated = {} def __str__(self): return 'Siasiesiv Startdate: {0} Member: {1} Chunk: {2} Basin: {3}'.format(self.startdate, self.member, @@ -93,24 +91,35 @@ class Siasiesiv(Diagnostic): return job_list + def request_data(self): + self.sit = self.request_chunk(ModelingRealms.seaIce, 'sit', self.startdate, self.member, self.chunk) + self.sic = self.request_chunk(ModelingRealms.seaIce, 'sic', self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self._declare_var('sivols') + self._declare_var('siareas') + self._declare_var('siextents') + + self._declare_var('sivoln') + self._declare_var('siarean') + self._declare_var('siextentn') + + def _declare_var(self, var_name): + self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name, self.startdate, self.member, + self.chunk, region=self.basin.fullname) + def compute(self): """ Runs the diagnostic """ - var_manager = VariableManager() - sit_var = var_manager.get_variable('sit').short_name - sic_var = var_manager.get_variable('sic').short_name - - sit_file = self.data_manager.get_file(ModelingRealms.seaIce, sit_var, self.startdate, self.member, self.chunk) - sit_handler = Utils.openCdf(sit_file) - sit = np.asfortranarray(sit_handler.variables[sit_var][:]) + sit_handler = Utils.openCdf(self.sit.local_file) + sit = np.asfortranarray(sit_handler.variables['sit'][:]) timesteps = sit_handler.dimensions['time'].size sit_handler.close() - sic_file = self.data_manager.get_file(ModelingRealms.seaIce, sic_var, self.startdate, self.member, self.chunk) - sic_handler = Utils.openCdf(sic_file) - Utils.convert_units(sic_handler.variables[sic_var], '1.0') - sic = np.asfortranarray(sic_handler.variables[sic_var][:]) + sic_handler = Utils.openCdf(self.sic.local_file) + Utils.convert_units(sic_handler.variables['sic'], '1.0') + sic = np.asfortranarray(sic_handler.variables['sic'][:]) sic_handler.close() result = np.empty((8, timesteps)) @@ -118,30 +127,17 @@ class Siasiesiv(Diagnostic): result[:, t] = cdftoolspython.icediag.icediags(Siasiesiv.e1t, Siasiesiv.e2t, self.mask, Siasiesiv.gphit, sit[t, :], sic[t, :]) - self.send_file(self._extract_variable_and_rename(sit_file, result[4, :], 'sivols', '10^9 m3'), - ModelingRealms.seaIce, 'sivols', self.startdate, self.member, self.chunk, - region=self.basin.name) - self.send_file(self._extract_variable_and_rename(sit_file, result[5, :], 'siareas', '10^9 m2'), - ModelingRealms.seaIce, 'siareas', self.startdate, self.member, self.chunk, - region=self.basin.name) - self.send_file(self._extract_variable_and_rename(sit_file, result[7, :], 'siextents', '10^9 m2'), - ModelingRealms.seaIce, 'siextents', self.startdate, self.member, self.chunk, - region=self.basin.name) - - self.send_file(self._extract_variable_and_rename(sit_file, result[0, :], 'sivoln', '10^9 m3'), - ModelingRealms.seaIce, 'sivoln', self.startdate, self.member, self.chunk, - region=self.basin.name) - self.send_file(self._extract_variable_and_rename(sit_file, result[1, :], 'siarean', '10^9 m2'), - ModelingRealms.seaIce, 'siarean', self.startdate, self.member, self.chunk, - region=self.basin.name) - self.send_file(self._extract_variable_and_rename(sit_file, result[3, :], 'siextentn', '10^9 m2'), - ModelingRealms.seaIce, 'siextentn', self.startdate, self.member, self.chunk, - region=self.basin.name) - - @staticmethod - def _extract_variable_and_rename(reference_file, values, cmor_name, units): + self._extract_variable_and_rename(result[4, :], 'sivols', '10^9 m3') + self._extract_variable_and_rename(result[5, :], 'siareas', '10^9 m2') + self._extract_variable_and_rename(result[7, :], 'siextents', '10^9 m2') + + self._extract_variable_and_rename(result[0, :], 'sivoln', '10^9 m3') + self._extract_variable_and_rename(result[1, :], 'siarean', '10^9 m2') + self._extract_variable_and_rename(result[3, :], 'siextentn', '10^9 m2') + + def _extract_variable_and_rename(self, values, cmor_name, units): temp = TempFile.get() - reference_handler = Utils.openCdf(reference_file) + reference_handler = Utils.openCdf(self.sit.local_file) os.remove(temp) handler = netCDF4.Dataset(temp, 'w') @@ -157,5 +153,5 @@ class Siasiesiv(Diagnostic): new_var[:] = values new_var.valid_max = np.max(values) handler.close() - return temp + self.generated[cmor_name].set_local_file(temp) diff --git a/earthdiagnostics/ocean/verticalgradient.py b/earthdiagnostics/ocean/verticalgradient.py index 67a5011a3c22ae194d92379a8c7f85efdaa90265..9cd9ab6101875ea64c2f3c2a788ff423fa23fdae 100644 --- a/earthdiagnostics/ocean/verticalgradient.py +++ b/earthdiagnostics/ocean/verticalgradient.py @@ -63,7 +63,7 @@ class VerticalGradient(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticIntOption('upper_level', 1), DiagnosticIntOption('low_level', 2)) options = cls.process_options(options, options_available) @@ -80,14 +80,19 @@ class VerticalGradient(Diagnostic): options['variable'], box)) return job_list + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, + self.member, self.chunk) + + def declare_data_generated(self): + self.gradient_file = self.declare_chunk(ModelingRealms.ocean, self.variable + 'vgrad', + self.startdate, self.member, self.chunk, box=self.box) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(ModelingRealms.ocean, self.variable, self.startdate, self.member, - self.chunk) - - handler = Utils.openCdf(variable_file) + handler = Utils.openCdf(self.variable_file) if 'lev' not in handler.dimensions: raise Exception('Variable {0} does not have a level dimension') var_handler = handler.variables[self.variable] @@ -108,6 +113,5 @@ class VerticalGradient(Diagnostic): new_var.long_name += ' Vertical gradient' new_var.standard_name += '_vertical_gradient' - self.send_file(temp, ModelingRealms.ocean, self.variable + 'vgrad', self.startdate, self.member, self.chunk, - box=self.box) + self.gradient_file.set_local_file(temp) diff --git a/earthdiagnostics/ocean/verticalmean.py b/earthdiagnostics/ocean/verticalmean.py index 693aa080cdc2fe780f8406886b41930b76072b9f..21bd4a88bc171a15fa6358364a23f99264b03328 100644 --- a/earthdiagnostics/ocean/verticalmean.py +++ b/earthdiagnostics/ocean/verticalmean.py @@ -64,7 +64,7 @@ class VerticalMean(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticIntOption('min_depth', -1), DiagnosticIntOption('max_depth', -1)) options = cls.process_options(options, options_available) @@ -81,15 +81,20 @@ class VerticalMean(Diagnostic): options['variable'], box)) return job_list + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(ModelingRealms.ocean, self.variable + 'vmean', self.startdate, self.member, + self.chunk, box=self.box) + def compute(self): """ Runs the diagnostic """ temp = TempFile.get() - variable_file = self.data_manager.get_file(ModelingRealms.ocean, self.variable, self.startdate, self.member, - self.chunk) - - handler = Utils.openCdf(variable_file) + handler = Utils.openCdf(self.variable_file.local_file) if self.box.min_depth is None: lev_min = handler.variables['lev'][0] else: @@ -101,9 +106,8 @@ class VerticalMean(Diagnostic): lev_max = self.box.max_depth handler.close() - cdftools.run('cdfvertmean', input=variable_file, output=temp, options=[self.variable, 'T', lev_min, lev_max, - '-debug']) + cdftools.run('cdfvertmean', input=self.variable_file.local_file, output=temp, + options=[self.variable, 'T', lev_min, lev_max, '-debug']) Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) - self.send_file(temp, ModelingRealms.ocean, self.variable + 'vmean', self.startdate, self.member, self.chunk, - box=self.box, rename_var='{0}_vert_mean'.format(self.variable)) + self.results.set_local_file(temp, rename_var='{0}_vert_mean'.format(self.variable)) diff --git a/earthdiagnostics/ocean/verticalmeanmeters.py b/earthdiagnostics/ocean/verticalmeanmeters.py index 7fbe1e423a7d1278ede9496034f023f5bcc46cb0..8951f493229004fc2ba3da3d7274e3216c9dbb3f 100644 --- a/earthdiagnostics/ocean/verticalmeanmeters.py +++ b/earthdiagnostics/ocean/verticalmeanmeters.py @@ -2,7 +2,7 @@ from earthdiagnostics import cdftools from earthdiagnostics.box import Box from earthdiagnostics.diagnostic import Diagnostic, DiagnosticFloatOption, DiagnosticDomainOption, \ - DiagnosticVariableOption + DiagnosticVariableOption, DiagnosticChoiceOption from earthdiagnostics.utils import Utils, TempFile from earthdiagnostics.modelingrealm import ModelingRealms @@ -35,7 +35,7 @@ class VerticalMeanMeters(Diagnostic): alias = 'vertmeanmeters' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, startdate, member, chunk, domain, variable, box): + def __init__(self, data_manager, startdate, member, chunk, domain, variable, box, grid_point): Diagnostic.__init__(self, data_manager) self.startdate = startdate self.member = member @@ -43,8 +43,7 @@ class VerticalMeanMeters(Diagnostic): self.domain = domain self.variable = variable self.box = box - self.required_vars = [variable] - self.generated_vars = [variable + 'vmean'] + self.grid_point = grid_point def __eq__(self, other): return self.startdate == other.startdate and self.member == other.member and self.chunk == other.chunk and \ @@ -65,10 +64,11 @@ class VerticalMeanMeters(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticVariableOption('variable'), + options_available = (DiagnosticVariableOption(diags.data_manager.config.var_manager), DiagnosticFloatOption('min_depth', -1), DiagnosticFloatOption('max_depth', -1), - DiagnosticDomainOption('domain', ModelingRealms.ocean)) + DiagnosticDomainOption(default_value=ModelingRealms.ocean), + DiagnosticChoiceOption('grid_point', ('T', 'U', 'V'), 'T')) options = cls.process_options(options, options_available) box = Box(True) @@ -80,18 +80,23 @@ class VerticalMeanMeters(Diagnostic): job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(VerticalMeanMeters(diags.data_manager, startdate, member, chunk, - options['domain'], options['variable'], box)) + options['domain'], options['variable'], box, options['grid_point'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(ModelingRealms.ocean, self.variable, self.startdate, self.member, + self.chunk) + + def declare_data_generated(self): + self.results = self.declare_chunk(self.domain, self.variable + 'vmean', self.startdate, self.member, + self.chunk, box=self.box) + def compute(self): """ Runs the diagnostic """ temp = TempFile.get() - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, - self.chunk) - - handler = Utils.openCdf(variable_file) + handler = Utils.openCdf(self.variable_file.local_file) if self.box.min_depth is None: lev_min = handler.variables['lev'][0] else: @@ -103,8 +108,7 @@ class VerticalMeanMeters(Diagnostic): lev_max = self.box.max_depth handler.close() - cdftools.run('cdfvertmean', input=variable_file, output=temp, options=[self.variable, 'T', lev_min, lev_max, - '-debug']) + cdftools.run('cdfvertmean', input=self.variable_file.local_file, output=temp, + options=[self.variable, self.grid_point, lev_min, lev_max, '-debug']) Utils.setminmax(temp, '{0}_vert_mean'.format(self.variable)) - self.send_file(temp, self.domain, self.variable + 'vmean', self.startdate, self.member, self.chunk, - box=self.box, rename_var='{0}_vert_mean'.format(self.variable)) + self.results.set_local_file(temp, rename_var='{0}_vert_mean'.format(self.variable)) diff --git a/earthdiagnostics/publisher.py b/earthdiagnostics/publisher.py new file mode 100644 index 0000000000000000000000000000000000000000..4b318b2cafabbdac1ef3d3ed24e8e33f07b9ed79 --- /dev/null +++ b/earthdiagnostics/publisher.py @@ -0,0 +1,46 @@ +# coding=utf-8 +class Publisher(object): + """ + Base class to provide functionality to notify updates to other objects + """ + def __init__(self): + self._subscribers = dict() + + def subscribe(self, who, callback=None): + """ + Add a suscriber to the current publisher + + :param who: subscriber to add + :type who: object + :param callback: method to execute when publisher updates + :type callback: callable | NoneType + """ + if callback is None: + callback = getattr(who, 'update') + self._subscribers[who] = callback + + def unsubscribe(self, who): + """ + Removes a suscriber from the current publisher + + :param who: suscriber to remove + :type who: object + """ + del self._subscribers[who] + + def dispatch(self, *args): + """ + Notify update to all the suscribers + + :param args: arguments to pass + """ + for subscriber, callback in self._subscribers.items(): + # noinspection PyCallingNonCallable + callback(*args) + + @property + def suscribers(self): + """ + List of suscribers of this publisher + """ + return self._subscribers.keys() diff --git a/earthdiagnostics/statistics/__init__.py b/earthdiagnostics/statistics/__init__.py index 4ec6fc47f10ceb5d2e106a5a2f5f7e0577ff2e87..2424b99360092efc33d5a92b8d03c888a3882ea1 100644 --- a/earthdiagnostics/statistics/__init__.py +++ b/earthdiagnostics/statistics/__init__.py @@ -1,3 +1,5 @@ # coding=utf-8 from monthlypercentile import MonthlyPercentile from climatologicalpercentile import ClimatologicalPercentile +from daysoverpercentile import DaysOverPercentile +from discretize import Discretize diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 12c1f2fc6c7ede5e6784cd2a49b6f30b296f0e67..229ad02073a7f156fd3db0064c31b6e8508249ad 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -1,17 +1,22 @@ # coding=utf-8 +import iris +import iris.coord_categorisation +import iris.coords +import iris.exceptions +import numpy as np +import six from bscearth.utils.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, \ - DiagnosticListIntOption, DiagnosticIntOption + DiagnosticIntOption, DiagnosticListIntOption from earthdiagnostics.frequency import Frequencies -from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.utils import TempFile from earthdiagnostics.variable_type import VariableType -import numpy as np class ClimatologicalPercentile(Diagnostic): """ - Calculates the climatological percentiles for the given leadtimes + Calculates the climatological percentiles for the given leadtime :param data_manager: data management object :type data_manager: DataManager @@ -24,35 +29,30 @@ class ClimatologicalPercentile(Diagnostic): alias = 'climpercent' "Diagnostic alias for the configuration file" - def __init__(self, data_manager, domain, variable, leadtimes, num_bins, experiment_config): + Percentiles = np.array([0.1, 0.25, 0.33, 0.5, 0.66, 0.75, 0.9]) + + def __init__(self, data_manager, domain, variable, start_year, end_year, + forecast_month, experiment_config): Diagnostic.__init__(self, data_manager) self.variable = variable self.domain = domain - self.leadtimes = leadtimes self.experiment_config = experiment_config - self.realizations = None - self.lat_len = None - self.lon_len = None - self.num_bins = num_bins - self._bins = None - self.percentiles = np.array([0.1, 0.25, 0.33, 0.5, 0.66, 0.75, 0.9]) - self.cmor_var = data_manager.variable_list.get_variable(variable, silent=True) - if self.cmor_var and self.cmor_var.valid_max and self.cmor_var.valid_min: - self.max_value = float(self.cmor_var.valid_max) - self.min_value = float(self.cmor_var.valid_min) - self.check_limit_values = False - else: - self.min_value = None - self.max_value = None - self.check_limit_values = True + self.start_year = start_year + self.end_year = end_year + self.forecast_month = forecast_month + self.distribution = None + + self.leadtime_files = {} def __eq__(self, other): - return self.domain == other.domain and self.variable == other.variable and self.leadtimes == other.leadtimes + return self.domain == other.domain and self.variable == other.variable and \ + self.start_year == other.start_year and self.end_year == other.end_year and \ + self.forecast_month == other.forecast_month def __str__(self): - return 'Climatological percentile Variable: {0}:{1} Leadtimes: {2} ' \ - 'Bins: {3}'.format(self.domain, self.variable, self.leadtimes, self.num_bins) + return 'Climatological percentile Variable: {0.domain}:{0.variable} Period: {0.start_year}-{0.end_year} ' \ + 'Forecast month: {0.forecast_month}'.format(self) @classmethod def generate_jobs(cls, diags, options): @@ -65,146 +65,84 @@ class ClimatologicalPercentile(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticDomainOption('domain'), - DiagnosticVariableOption('variable'), - DiagnosticListIntOption('leadtimes'), - DiagnosticIntOption('bins', 2000)) + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticIntOption('start_year'), + DiagnosticIntOption('end_year'), + DiagnosticListIntOption('forecast_month'), + ) options = cls.process_options(options, options_available) job_list = list() - job_list.append(ClimatologicalPercentile(diags.data_manager, options['domain'], options['variable'], - options['leadtimes'], options['bins'], - diags.config.experiment)) + for forecast_month in options['forecast_month']: + job_list.append(ClimatologicalPercentile(diags.data_manager, options['domain'], options['variable'], + options['start_year'], options['end_year'], + forecast_month, diags.config.experiment)) return job_list + def requested_startdates(self): + return ['{0}{1:02}01'.format(year, self.forecast_month) for year in range(self.start_year, self.end_year+1)] + + def request_data(self): + for startdate in self.requested_startdates(): + if startdate not in self.leadtime_files: + self.leadtime_files[startdate] = {} + Log.debug('Retrieving startdate {0}', startdate) + self.leadtime_files[startdate] = self.request_chunk(self.domain, '{0}_dis'.format(self.variable), startdate, + None, None, vartype=VariableType.STATISTIC) + + def declare_data_generated(self): + var_name = '{0.variable}prct{0.start_year}{0.forecast_month}-{0.end_year}{0.forecast_month:02d}'.format(self) + self.percentiles_file = self.declare_chunk(self.domain, var_name, None, None, None, + frequency=Frequencies.climatology, vartype=VariableType.STATISTIC) + def compute(self): """ Runs the diagnostic """ - member_files = self._get_data() - - distribution = self._get_distribution(member_files) - - percentile_values = self._calculate_percentiles(distribution) - + iris.FUTURE.netcdf_promote = True + self._get_distribution() + percentile_values = self._calculate_percentiles() self._save_results(percentile_values) def _save_results(self, percentile_values): temp = TempFile.get() - handler = Utils.openCdf(temp, 'w') - - handler.createDimension('percentile', len(self.percentiles)) - percentile_var = handler.createVariable('percentile', float, ('percentile',)) - percentile_var[:] = self.percentiles - - handler.createDimension('lat', self.lat.size) - lat_var = handler.createVariable('lat', float, ('lat',)) - lat_var[:] = self.lat - - handler.createDimension('lon', self.lon.size) - lon_var = handler.createVariable('lon', float, ('lon',)) - lon_var[:] = self.lon + iris.FUTURE.netcdf_no_unlimited = True + iris.save(percentile_values.merge_cube(), temp, zlib=True) + self.percentiles_file.set_local_file(temp, rename_var='percent') - percentile_var = handler.createVariable('percent', float, ('percentile', 'lat', 'lon')) - percentile_var[...] = percentile_values - - handler.close() - - self.send_file(temp, self.domain, self.variable + '_percentiles', None, None, frequency=Frequencies.climatology, - rename_var='percent', vartype=VariableType.STATISTIC) - - def _calculate_percentiles(self, distribution): + def _calculate_percentiles(self): Log.debug('Calculating percentiles') + bins = self.distribution.coord('bin').points + def calculate(point_distribution): cs = np.cumsum(point_distribution) total = cs[-1] - percentile_values = self.percentiles * total + percentile_values = ClimatologicalPercentile.Percentiles * total index = np.searchsorted(cs, percentile_values) - return [(self._bins[i + 1] + self._bins[i]) / 2 for i in index] - - distribution = np.apply_along_axis(calculate, 0, distribution) - return distribution - - def _get_distribution(self, member_files): - distribution = None - for memberfile in member_files: - Log.debug('Discretizing file {0}', memberfile) - handler = Utils.openCdf(memberfile) - for realization in range(self.realizations): - if distribution is None: - distribution = self._calculate_distribution(handler, realization) - else: - distribution += self._calculate_distribution(handler, realization) - handler.close() - return distribution - - def _get_data(self): - member_files = list() - for startdate, member in self.experiment_config.get_member_list(): - Log.debug('Retrieving startdate {0}', startdate) - memberfile = self.data_manager.get_leadtimes(self.domain, self.variable, startdate, member, self.leadtimes) - - Log.debug('Getting data for startdate {0}', startdate) - handler = Utils.openCdf(memberfile) - self._get_value_interval(handler) - self._get_realizations_present(handler) - self._get_var_size(handler) - handler.close() - - member_files.append(memberfile) - return member_files - - def _get_realizations_present(self, handler): - realizations = 1 - if 'realization' in handler.dimensions: - realizations = handler.dimensions['realization'].size - if 'ensemble' in handler.dimensions: - realizations = handler.dimensions['ensemble'].size - if self.realizations is None: - self.realizations = realizations - if realizations != self.realizations: - # noinspection PyNoneFunctionAssignment - self.realizations = min(self.realizations, realizations) - Log.warning('Different number of realizations in the data used by diagnostic {0}', self) - - def _get_value_interval(self, handler): - if not self.check_limit_values: - return - - values = handler.variables[self.variable][:] - file_max = np.amax(values) - file_min = np.amin(values) - self.max_value = max(self.min_value, file_max) - if self.min_value is None: - self.min_value = file_min - else: - self.min_value = min(self.min_value, file_min) - - def _calculate_distribution(self, handler, realization): - Log.debug('Discretizing realization {0}', realization) - - def calculate_histogram(time_series): - histogram, self._bins = np.histogram(time_series, bins=self.num_bins, - range=(self.min_value, self.max_value)) - return histogram - - var = handler.variables[self.variable] - if 'realization' in var.dimensions or 'ensemble' in var.dimensions: - return np.apply_along_axis(calculate_histogram, 0, var[:, realization, ...]) - else: - return np.apply_along_axis(calculate_histogram, 0, var[:]) - - def _get_var_size(self, handler): - if self.lat_len is not None: - return - self.lat = handler.variables['latitude'][:] - self.lon = handler.variables['longitude'][:] - - - - - - - - + return [bins[i] for i in index] + + results = iris.cube.CubeList() + percentile_coord = iris.coords.DimCoord(ClimatologicalPercentile.Percentiles, long_name='percentile') + print(self.distribution) + for leadtime_slice in self.distribution.slices_over('leadtime'): + result = iris.cube.Cube(np.apply_along_axis(calculate, 0, leadtime_slice.data), var_name='percent', + units=self.distribution.coord('bin').units) + result.add_dim_coord(percentile_coord, 0) + result.add_dim_coord(leadtime_slice.coord('latitude'), 1) + result.add_dim_coord(leadtime_slice.coord('longitude'), 2) + result.add_aux_coord(leadtime_slice.coord('leadtime')) + results.append(result) + return results + + def _get_distribution(self): + for startdate, startdate_file in six.iteritems(self.leadtime_files): + Log.info('Getting data for startdate {0}', startdate) + data_cube = iris.load_cube(startdate_file.local_file) + if self.distribution is None: + self.distribution = data_cube + else: + self.distribution += data_cube + if len(self.distribution.coords('leadtime')) == 0: + self.distribution.add_aux_coord(iris.coords.AuxCoord(1, var_name='leadtime', units='months')) diff --git a/earthdiagnostics/statistics/daysoverpercentile.py b/earthdiagnostics/statistics/daysoverpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..aad012a492728d7d2a5c320441dc88c398b79e64 --- /dev/null +++ b/earthdiagnostics/statistics/daysoverpercentile.py @@ -0,0 +1,243 @@ +# coding=utf-8 +import os + +import iris +import iris.analysis +import iris.coord_categorisation +import iris.coords +import iris.exceptions +import numpy as np +from bscearth.utils.date import parse_date, add_months +from bscearth.utils.log import Log +from iris.time import PartialDateTime + +from earthdiagnostics.diagnostic import * +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile +from earthdiagnostics.utils import Utils, TempFile + + +class DaysOverPercentile(Diagnostic): + """ + Calculates the montlhy percentiles + + :param data_manager: data management object + :type data_manager: DataManager + :param variable: variable to average + :type variable: str + """ + + alias = 'daysover' + "Diagnostic alias for the configuration file" + + def __init__(self, data_manager, domain, variable, start_year, end_year, startdate, forecast_month): + Diagnostic.__init__(self, data_manager) + self.variable = variable + self.domain = domain + self.start_year = start_year + self.end_year = end_year + self.forecast_month = forecast_month + self.startdate = startdate + + def __eq__(self, other): + return self.startdate == other.startdate and self.domain == other.domain and \ + self.variable == other.variable and self.start_year == other.start_year and \ + self.end_year == other.end_year + + def __str__(self): + return 'Days over percentile Startdate: {0.startdate} Variable: {0.domain}:{0.variable} ' \ + 'Climatology: {0.start_year}-{0.end_year}'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain, variable, percentil number, maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticOption('variable'), + DiagnosticIntOption('start_year'), + DiagnosticIntOption('end_year'), + DiagnosticListIntOption('forecast_month'),) + options = cls.process_options(options, options_available) + + job_list = list() + for startdate in diags.config.experiment.startdates: + for forecast_month in options['forecast_month']: + job_list.append(DaysOverPercentile(diags.data_manager, options['domain'], options['variable'], + options['start_year'], options['end_year'], + startdate, forecast_month)) + return job_list + + def request_data(self): + var_name = '{0.variable}prct{0.start_year}{0.forecast_month}-{0.end_year}{0.forecast_month:02d}'.format(self) + self.percentiles_file = self.request_chunk(self.domain, var_name, None, None, None, + frequency=Frequencies.climatology) + + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, None, None) + + def declare_data_generated(self): + var_over = self.variable + '_daysover_q{0}_{1.start_year}-{1.end_year}' + var_below = self.variable + '_daysbelow_q{0}_{1.start_year}-{1.end_year}' + self.days_over_file = {} + self.days_below_file = {} + for perc in ClimatologicalPercentile.Percentiles: + self.days_over_file[perc] = self.declare_chunk(self.domain, var_over.format(int(perc * 100), self), + self.startdate, None, + None, frequency=Frequencies.monthly, + vartype=VariableType.STATISTIC) + + self.days_below_file[perc] = self.declare_chunk(self.domain, var_below.format(int(perc * 100), self), + self.startdate, None, + None, frequency=Frequencies.monthly, + vartype=VariableType.STATISTIC) + + def compute(self): + """ + Runs the diagnostic + """ + iris.FUTURE.netcdf_promote = True + percentiles = iris.load_cube(self.percentiles_file.local_file) + + handler = Utils.openCdf(self.variable_file.local_file) + if 'realization' in handler.variables: + handler.variables[self.variable].coordinates = 'realization' + handler.close() + var = iris.load_cube(self.variable_file.local_file) + date = parse_date(self.startdate) + lead_date = add_months(date, 1, self.data_manager.config.experiment.calendar) + leadtimes = {1: PartialDateTime(lead_date.year, lead_date.month, lead_date.day)} + + def assign_leadtime(coord, x): + # noinspection PyBroadException + try: + leadtime_month = 1 + partial_date = leadtimes[leadtime_month] + while coord.units.num2date(x) >= partial_date: + leadtime_month += 1 + try: + partial_date = leadtimes[leadtime_month] + except KeyError: + new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) + partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) + leadtimes[leadtime_month] = partial_date + return leadtime_month + except Exception: + pass + iris.coord_categorisation.add_categorised_coord(var, 'leadtime', 'time', assign_leadtime) + iris.coord_categorisation.add_year(var, 'time') + iris.coord_categorisation.add_day_of_year(var, 'time') + try: + realization_coord = var.coord('realization') + except iris.exceptions.CoordinateNotFoundError: + realization_coord = None + self.lat_coord = var.coord('latitude') + self.lon_coord = var.coord('longitude') + results_over = {perc: iris.cube.CubeList() for perc in ClimatologicalPercentile.Percentiles} + results_below = {perc: iris.cube.CubeList() for perc in ClimatologicalPercentile.Percentiles} + + var_daysover = 'days_over' + var_days_below = 'days_below' + long_name_days_over = 'Proportion of days over a given percentile for {0.start_year}-{0.end_year} ' \ + 'climatology'.format(self) + long_name_days_below = 'Proportion of days below a given percentile for {0.start_year}-{0.end_year} ' \ + 'climatology'.format(self) + + for leadtime in leadtimes.keys(): + Log.debug('Computing startdate {0} leadtime {1}', self.startdate, leadtime) + leadtime_slice = var.extract(iris.Constraint(leadtime=leadtime)) + if len(percentiles.coords('leadtime')) > 0: + percentiles_leadtime = percentiles.extract(iris.Constraint(leadtime=leadtime)) + else: + percentiles_leadtime = percentiles + time_coord = iris.coords.AuxCoord.from_coord(leadtime_slice.coord('time')) + first_time = time_coord.points[0] + last_time = time_coord.points[-1] + timesteps = leadtime_slice.coord('time').shape[0] + time_coord = time_coord.copy(first_time + (last_time - first_time) / 2, (first_time, last_time)) + for percentile_slice in percentiles_leadtime.slices_over('percentile'): + percentile = percentile_slice.coord('percentile').points[0] + + # noinspection PyTypeChecker + days_over = np.sum(leadtime_slice.data > percentile_slice.data, 0) / float(timesteps) + result = self.create_results_cube(days_over, percentile, realization_coord, + time_coord, var_daysover, long_name_days_over) + results_over[percentile].append(result) + + # noinspection PyTypeChecker + days_below = np.sum(leadtime_slice.data < percentile_slice.data, 0) / float(timesteps) + result = self.create_results_cube(days_below, percentile, realization_coord, + time_coord, var_days_below, long_name_days_below) + results_below[percentile].append(result) + + Log.debug('Saving percentiles startdate {0}', self.startdate) + for perc in ClimatologicalPercentile.Percentiles: + iris.FUTURE.netcdf_no_unlimited = True + self.days_over_file[perc].set_local_file(self.save_to_file(perc, results_over, var_daysover), + rename_var=var_daysover) + self.days_below_file[perc].set_local_file(self.save_to_file(perc, results_below, var_days_below), + rename_var=var_days_below) + + del self.days_over_file + del self.days_below_file + del self.lat_coord + del self.lon_coord + + def save_to_file(self, perc, results_over, var_daysover): + temp = TempFile.get() + iris.save(results_over[perc].merge_cube(), temp, zlib=True, unlimited_dimensions=['time']) + Utils.rename_variables(temp, {'dim2': 'ensemble', 'dim1': 'ensemble'}, + must_exist=False, rename_dimension=True) + handler = Utils.openCdf(temp) + if 'time' not in handler.dimensions: + new_file = TempFile.get() + new_handler = Utils.openCdf(new_file, 'w') + + new_handler.createDimension('time', 1) + for dimension in handler.dimensions: + Utils.copy_dimension(handler, new_handler, dimension) + + for variable in handler.variables.keys(): + if variable in (var_daysover, 'time', 'time_bnds'): + continue + Utils.copy_variable(handler, new_handler, variable) + old_var = handler.variables[var_daysover] + new_var = new_handler.createVariable(var_daysover, old_var.dtype, ('time',) + old_var.dimensions, + zlib=True, fill_value=1.0e20) + Utils.copy_attributes(new_var, old_var) + new_var[0, :] = old_var[:] + + old_var = handler.variables['time'] + new_var = new_handler.createVariable('time', old_var.dtype, ('time',)) + Utils.copy_attributes(new_var, old_var) + new_var[0] = old_var[0] + + old_var = handler.variables['time_bnds'] + new_var = new_handler.createVariable('time_bnds', old_var.dtype, ('time',) + old_var.dimensions) + Utils.copy_attributes(new_var, old_var) + new_var[0, :] = old_var[:] + + new_handler.close() + os.remove(temp) + temp = new_file + handler.close() + return temp + + def create_results_cube(self, days_over, percentile, realization_coord, time_coord, + var_name, long_name): + result = iris.cube.Cube(days_over.astype(np.float32), var_name=var_name, long_name=long_name, units=1.0) + if realization_coord is not None: + result.add_aux_coord(realization_coord, 0) + result.add_dim_coord(self.lat_coord, 1) + result.add_dim_coord(self.lon_coord, 2) + else: + result.add_dim_coord(self.lat_coord, 0) + result.add_dim_coord(self.lon_coord, 1) + result.add_aux_coord(iris.coords.AuxCoord(percentile, long_name='percentile')) + result.add_aux_coord(time_coord) + return result diff --git a/earthdiagnostics/statistics/discretize.py b/earthdiagnostics/statistics/discretize.py new file mode 100644 index 0000000000000000000000000000000000000000..81ed8232b578f0e4a682b0c3972c1a1fb50fc485 --- /dev/null +++ b/earthdiagnostics/statistics/discretize.py @@ -0,0 +1,262 @@ +# coding=utf-8 +import math + +import iris +import iris.coord_categorisation +import iris.coords +import iris.exceptions +import iris.unit +import numpy as np +import psutil +import six +from bscearth.utils.date import parse_date, add_months, add_days +from bscearth.utils.log import Log +from iris.cube import Cube +from iris.time import PartialDateTime + +from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, \ + DiagnosticIntOption, DiagnosticFloatOption +from earthdiagnostics.utils import Utils, TempFile +from earthdiagnostics.variable_type import VariableType + + +class Discretize(Diagnostic): + """ + Discretizes a variable + + :param data_manager: data management object + :type data_manager: DataManager + :param variable: variable to average + :type variable: str + """ + + alias = 'discretize' + "Diagnostic alias for the configuration file" + + Percentiles = np.array([0.1, 0.25, 0.33, 0.5, 0.66, 0.75, 0.9]) + + def __init__(self, data_manager, startdate, domain, variable, num_bins, min_value, max_value): + Diagnostic.__init__(self, data_manager) + + self.startdate = startdate + self.variable = variable + self.domain = domain + + self.realizations = None + self.num_bins = num_bins + self._bins = None + self.cmor_var = data_manager.variable_list.get_variable(variable, silent=True) + + if not math.isnan(min_value): + self.min_value = min_value + self.check_min_value = False + elif self.cmor_var and self.cmor_var.valid_min: + self.min_value = float(self.cmor_var.valid_min) + self.check_min_value = False + else: + self.min_value = None + self.check_min_value = True + + if not math.isnan(max_value): + self.max_value = max_value + self.check_max_value = False + elif self.cmor_var and self.cmor_var.valid_min: + self.max_value = float(self.cmor_var.valid_max) + self.check_max_value = False + else: + self.max_value = None + self.check_max_value = True + + self.process = psutil.Process() + + def print_memory_used(self): + Log.debug('Memory: {0:.2f} GB'.format(self.process.memory_info().rss / 1024.0**3)) + + @property + def bins(self): + if self._bins is None: + return self.num_bins + return self._bins + + @bins.setter + def bins(self, value): + self._bins = value + + def __eq__(self, other): + return self.domain == other.domain and self.variable == other.variable and self.num_bins == other.num_bins and \ + self.min_value == other.min_value and self.max_value == other.max_value and \ + self.startdate == other.startdate + + def __str__(self): + return 'Discretizing variable: {0.domain}:{0.variable} Startdate: {0.startdate} ' \ + 'Bins: {0.num_bins} Range: [{0.min_value}, {0.max_value}]'.format(self) + + @classmethod + def generate_jobs(cls, diags, options): + """ + Creates a job for each chunk to compute the diagnostic + + :param diags: Diagnostics manager class + :type diags: Diags + :param options: domain, variable, percentil number, maximum depth (level) + :type options: list[str] + :return: + """ + options_available = (DiagnosticDomainOption(), + DiagnosticVariableOption(diags.data_manager.config.var_manager), + DiagnosticIntOption('bins', 2000), + DiagnosticFloatOption('min_value', float('nan')), + DiagnosticFloatOption('max_value', float('nan')), + ) + options = cls.process_options(options, options_available) + + job_list = list() + for startdate in diags.config.experiment.startdates: + job_list.append(Discretize(diags.data_manager, startdate, options['domain'], options['variable'], + options['bins'], options['min_value'], options['max_value'])) + return job_list + + def request_data(self): + self.original_data = self.request_chunk(self.domain, self.variable, self.startdate, None, None) + + def declare_data_generated(self): + var_name = '{0.variable}_dis'.format(self) + self.discretized_data = self.declare_chunk(self.domain, var_name, self.startdate, None, None, + vartype=VariableType.STATISTIC) + + def compute(self): + """ + Runs the diagnostic + """ + self.print_memory_used() + iris.FUTURE.netcdf_promote = True + self._load_cube() + self.print_memory_used() + self._get_value_interval() + self.print_memory_used() + Log.info('Range: [{0}, {1}]', self.min_value, self.max_value) + self._get_distribution() + self.print_memory_used() + self._save_results() + self.print_memory_used() + del self.distribution + del self.data_cube + self.print_memory_used() + + def _load_cube(self): + + handler = Utils.openCdf(self.original_data.local_file) + if 'realization' in handler.variables: + handler.variables[self.variable].coordinates = 'realization' + handler.close() + data_cube = iris.load_cube(self.original_data.local_file) + + date = parse_date(self.startdate) + lead_date = add_months(date, 1, self.data_manager.config.experiment.calendar) + leadtimes = {1: PartialDateTime(lead_date.year, lead_date.month, lead_date.day)} + + def assign_leadtime(coord, x): + leadtime_month = 1 + partial_date = leadtimes[leadtime_month] + while coord.units.num2date(x) >= partial_date: + leadtime_month += 1 + try: + partial_date = leadtimes[leadtime_month] + except KeyError: + new_date = add_months(date, leadtime_month, self.data_manager.config.experiment.calendar) + partial_date = PartialDateTime(new_date.year, new_date.month, new_date.day) + leadtimes[leadtime_month] = partial_date + return leadtime_month + + iris.coord_categorisation.add_categorised_coord(data_cube, 'leadtime', 'time', assign_leadtime) + self.data_cube = data_cube + + def _save_results(self): + Log.debug('Saving results...') + + bins = np.zeros(self.num_bins) + bins_bounds = np.zeros((self.num_bins, 2)) + + for x in range(self.num_bins): + bins[x] = (self.bins[x+1] - self.bins[x]) / 2 + self.bins[x] + bins_bounds[x, 0] = self.bins[x] + bins_bounds[x, 1] = self.bins[x+1] + + bins_coord = iris.coords.DimCoord(bins, var_name='bin', units=self.data_cube.units, bounds=bins_bounds) + + cubes = iris.cube.CubeList() + + date = parse_date(self.startdate) + date = add_days(date, 14, self.data_manager.config.experiment.calendar) + + for leadtime, distribution in six.iteritems(self.distribution): + leadtime_cube = Cube(distribution.astype(np.uint32), var_name=self.data_cube.var_name, + standard_name=self.data_cube.standard_name, units='1') + leadtime_cube.add_dim_coord(bins_coord, 0) + leadtime_cube.add_dim_coord(self.data_cube.coord('latitude'), 1) + leadtime_cube.add_dim_coord(self.data_cube.coord('longitude'), 2) + leadtime_cube.add_aux_coord(iris.coords.AuxCoord(leadtime, + var_name='leadtime', + units='months')) + lead_date = add_months(date, leadtime - 1, self.data_manager.config.experiment.calendar) + leadtime_cube.add_aux_coord(iris.coords.AuxCoord(iris.unit.date2num(lead_date, + unit='days since 1950-01-01', + calendar="standard"), + var_name='time', + units='days since 1950-01-01')) + + cubes.append(leadtime_cube) + temp = TempFile.get() + iris.FUTURE.netcdf_no_unlimited = True + iris.save(cubes.merge_cube(), temp, zlib=True) + self.discretized_data.set_local_file(temp, rename_var=self.data_cube.var_name) + + def _get_distribution(self): + self.distribution = {} + Log.debug('Discretizing...') + for leadtime in set(self.data_cube.coord('leadtime').points): + Log.debug('Discretizing leadtime {0}', leadtime) + leadtime_cube = self.data_cube.extract(iris.Constraint(leadtime=leadtime)) + if 'realization' in leadtime_cube.coords(): + for realization_cube in self.data_cube.slices_over('realization'): + Log.debug('Discretizing realization {0}', realization_cube.coord('realization').points[0]) + self.print_memory_used() + if leadtime not in self.distribution: + self.distribution[leadtime] = self._calculate_distribution(realization_cube) + else: + self.distribution[leadtime] += self._calculate_distribution(realization_cube) + else: + self.print_memory_used() + self.distribution[leadtime] = self._calculate_distribution(leadtime_cube) + + # noinspection PyTypeChecker + def _get_value_interval(self): + if self.check_min_value or self.check_max_value: + Log.debug('Calculating max and min values...') + for time_slice in self.data_cube.slices_over('time'): + if self.check_min_value: + file_min = np.amin(time_slice.data) + if self.min_value is None: + self.min_value = file_min + self.min_value = min(self.min_value, file_min) + + if self.check_max_value: + file_max = np.amax(time_slice.data) + self.max_value = max(self.max_value, file_max) + + def _calculate_distribution(self, data_cube): + def calculate_histogram(time_series): + histogram, self.bins = np.histogram(time_series, bins=self.bins, + range=(self.min_value, self.max_value)) + return histogram + + return np.apply_along_axis(calculate_histogram, 0, data_cube.data) + + + + + + + + + diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index eab4ea0260b1d752792a5b315643643974bffdf0..1ecb5c44276b68650216a0e79a7d239c13a499a6 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -1,6 +1,4 @@ # coding=utf-8 -import shutil - from bscearth.utils.log import Log from earthdiagnostics.diagnostic import Diagnostic, DiagnosticOption, DiagnosticDomainOption, DiagnosticListIntOption @@ -59,24 +57,52 @@ class MonthlyPercentile(Diagnostic): :type options: list[str] :return: """ - options_available = (DiagnosticOption('domain'), - DiagnosticDomainOption('variable'), - DiagnosticListIntOption('percentiles', None, 0, 100)) + options_available = (DiagnosticDomainOption(), + DiagnosticOption('variable'), + DiagnosticListIntOption('percentiles', [], 0, 100)) options = cls.process_options(options, options_available) job_list = list() for startdate, member, chunk in diags.config.experiment.get_chunk_list(): job_list.append(MonthlyPercentile(diags.data_manager, startdate, member, chunk, - options['variable'], options['domain'], options['percentiles'])) + options['domain'], options['variable'], options['percentiles'])) return job_list + def request_data(self): + self.variable_file = self.request_chunk(self.domain, self.variable, self.startdate, self.member, self.chunk) + + def declare_data_generated(self): + self.max_file = self.declare_chunk(self.domain, self.variable_max, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, vartype=VariableType.STATISTIC) + self.min_file = self.declare_chunk(self.domain, self.variable_min, self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, vartype=VariableType.STATISTIC) + self.percentile_file = {} + for percentile in self.percentiles: + self.percentile_file[percentile] = self.declare_chunk(self.domain, self.percentile(percentile), + self.startdate, self.member, self.chunk, + frequency=Frequencies.monthly, + vartype=VariableType.STATISTIC) + + self.declare_chunk(self.domain, '{0}_q{1}'.format(self.variable, percentile), self.startdate, + self.member, self.chunk, frequency=Frequencies.monthly, vartype=VariableType.STATISTIC) + + @property + def variable_max(self): + return '{0}max'.format(self.variable) + + @property + def variable_min(self): + return '{0}min'.format(self.variable) + + def percentile(self, percentile): + return '{0}_q{1}'.format(self.variable, percentile) + def compute(self): """ Runs the diagnostic """ - variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) temp = TempFile.get() - handler = Utils.openCdf(variable_file) + handler = Utils.openCdf(self.variable_file.local_file) datetimes = Utils.get_datetime_from_netcdf(handler) handler.close() @@ -97,25 +123,39 @@ class MonthlyPercentile(Diagnostic): if start_index != 0 or end_index != datetimes.size - 1: start_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[start_index]) end_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[end_index]) - Utils.cdo.seldate('{0},{1}'.format(start_date, end_date), input=variable_file, output=temp) + Utils.cdo.seldate('{0},{1}'.format(start_date, end_date), input=self.variable_file.local_file, output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - shutil.move(temp, variable_file) + else: + Utils.copy_file(self.variable_file.local_file, temp) Log.debug('Computing minimum') monmin_file = TempFile.get() - Utils.cdo.monmin(input=variable_file, output=monmin_file) + Utils.cdo.monmin(input=temp, output=monmin_file) Log.debug('Computing maximum') monmax_file = TempFile.get() - Utils.cdo.monmax(input=variable_file, output=monmax_file) + Utils.cdo.monmax(input=temp, output=monmax_file) for percentile in self.percentiles: Log.debug('Computing percentile {0}', percentile) - Utils.cdo.monpctl(str(percentile), input=[variable_file, monmin_file, monmax_file], output=temp) + Utils.cdo.monpctl(str(percentile), input=[temp, monmin_file, monmax_file], output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, percentile), self.startdate, - self.member, self.chunk, frequency=Frequencies.monthly, rename_var=self.variable, - vartype=VariableType.STATISTIC) + handler = Utils.openCdf(monmax_file) + handler.variables[self.variable].long_name += ' {0} Percentile'.format(percentile) + handler.close() + self.percentiles[percentile].set_local_file(temp, rename_var=self.variable) + + Utils.rename_variable(monmax_file, 'lev', 'ensemble', False, True) + handler = Utils.openCdf(monmax_file) + handler.variables[self.variable].long_name += ' Monthly Maximum' + handler.close() + self.max_file.set_local_file(monmax_file, rename_var=self.variable) + + Utils.rename_variable(monmin_file, 'lev', 'ensemble', False, True) + handler = Utils.openCdf(monmin_file) + handler.variables[self.variable].long_name += ' Monthly Minimum' + handler.close() + self.min_file.set_local_file(monmin_file, rename_var=self.variable) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index 76ebe329c149c9efbc772d8b1f7e7ce4ffa9f866..8637121a55d0595b3e008cdf5c47675d0a7bbe29 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,12 +1,20 @@ # coding=utf-8 import os +from time import strptime + +import iris +import netCDF4 +import numpy as np from bscearth.utils.date import parse_date, add_months, chunk_start_date, chunk_end_date +from bscearth.utils.log import Log +from iris.coords import DimCoord +from cf_units import Unit -from earthdiagnostics.datamanager import DataManager, NetCDFFile +from datafile import DataFile, StorageStatus, LocalStatus +from earthdiagnostics.datamanager import DataManager from earthdiagnostics.utils import TempFile, Utils from datetime import datetime -from earthdiagnostics.variable import VariableManager from earthdiagnostics.variable_type import VariableType @@ -40,7 +48,7 @@ class THREDDSManager(DataManager): 'month', self.experiment.calendar) end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) - thredds_subset = THREDDSSubset(aggregation_path, variable, startdate, end_chunk).get_url() + thredds_subset = THREDDSSubset(aggregation_path, "", variable, startdate, end_chunk) selected_months = ','.join([str(add_months(startdate, i, self.experiment.calendar).month) for i in leadtimes]) temp = TempFile.get() if self.config.data_type == 'exp': @@ -51,6 +59,7 @@ class THREDDSManager(DataManager): Utils.cdo.selmonth(selected_months, input=thredds_subset, output=temp) return temp + # noinspection PyUnusedLocal def file_exists(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, vartype=VariableType.MEAN): """ @@ -83,108 +92,8 @@ class THREDDSManager(DataManager): self.experiment.calendar) end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) - thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) - return thredds_subset.check() - - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, - vartype=VariableType.MEAN): - """ - Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy - - :param domain: CMOR domain - :type domain: str - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: Frequency - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - :return: path to the copy created on the scratch folder - :rtype: str - """ - aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) - - start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', - self.experiment.calendar) - end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) - - thredds_subset = THREDDSSubset(aggregation_path, var, start_chunk, end_chunk) - return thredds_subset.download() - - def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, - rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False, vartype=VariableType.MEAN): - """ - Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge - with already existing ones as needed - - :param move_old: if true, moves files following older conventions that may be found on the links folder - :type move_old: bool - :param date_str: exact date_str to use in the cmorized file - :type: str - :param year: if frequency is yearly, this parameter is used to give the corresponding year - :type year: int - :param rename_var: if exists, the given variable will be renamed to the one given by var - :type rename_var: str - :param filetosend: path to the file to send to the CMOR repository - :type filetosend: str - :param region: specifies the region represented by the file. If it is defined, the data will be appended to the - CMOR repository as a new region in the file or will overwrite if region was already present - :type region: str - :param domain: CMOR domain - :type domain: Domain - :param var: variable name - :type var: str - :param startdate: file's startdate - :type startdate: str - :param member: file's member - :type member: int - :param chunk: file's chunk - :type chunk: int - :param grid: file's grid (only needed if it is not the original) - :type grid: str - :param box: file's box (only needed to retrieve sections or averages) - :type box: Box - :param frequency: file's frequency (only needed if it is different from the default) - :type frequency: str - :param diagnostic: diagnostic used to generate the file - :type diagnostic: Diagnostic - :param cmorized: flag to indicate if file was generated in cmorization process - :type cmorized: bool - :param vartype: Variable type (mean, statistic) - :type vartype: VariableType - """ - if cmorized: - raise ValueError('cmorized is not supported in THREDDS manager') - original_var = var - cmor_var = VariableManager().get_variable(var) - var = self._get_final_var_name(box, var) - - if rename_var and rename_var != var: - Utils.rename_variable(filetosend, rename_var, var) - elif original_var != var: - Utils.rename_variable(filetosend, original_var, var) - - if not frequency: - frequency = self.config.frequency - - filepath = self.get_file_path(startdate, domain, var, frequency, vartype, box, grid) - netcdf_file = NetCDFFile(filepath, filetosend, domain, var, cmor_var, self.config.data_convention, region) - if diagnostic: - netcdf_file.add_diagnostic_history(diagnostic) - else: - raise ValueError('You must provide a diagnostic to store data using the THREDDSmanager') - netcdf_file.send() + thredds_subset = THREDDSSubset(aggregation_path, "", var, start_chunk, end_chunk) + return thredds_subset def get_file_path(self, startdate, domain, var, frequency, vartype, box=None, grid=None): @@ -231,6 +140,7 @@ class THREDDSManager(DataManager): var_folder) return folder_path + # noinspection PyUnusedLocal def get_year(self, domain, var, startdate, member, year, grid=None, box=None, vartype=VariableType.MEAN): """ Ge a file containing all the data for one year for one variable @@ -253,7 +163,7 @@ class THREDDSManager(DataManager): :return: """ aggregation_path = self.get_var_url(var, startdate, None, box, vartype) - thredds_subset = THREDDSSubset(aggregation_path, var, datetime(year, 1, 1), datetime(year+1, 1, 1)) + thredds_subset = THREDDSSubset(aggregation_path, "", var, datetime(year, 1, 1), datetime(year+1, 1, 1)) return thredds_subset.download() def get_var_url(self, var, startdate, frequency, box, vartype): @@ -295,6 +205,7 @@ class THREDDSManager(DataManager): """ Creates the link of a given file from the CMOR repository. + :param cmor_var: :param move_old: :param date_str: :param year: if frequency is yearly, this parameter is used to give the corresponding year @@ -323,92 +234,159 @@ class THREDDSManager(DataManager): # THREDDSManager does not require links pass + def request_chunk(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VariableType.MEAN): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param vartype: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :return: path to the copy created on the scratch folder + :rtype: str + """ + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) + file_path = self.get_file_path(startdate, domain, var, frequency, vartype, box=box) + + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', + self.experiment.calendar) + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + + thredds_subset = THREDDSSubset(aggregation_path, file_path, var, start_chunk, end_chunk) + thredds_subset.local_status = LocalStatus.PENDING + self.requested_files[file_path] = thredds_subset + return thredds_subset + + # noinspection PyUnusedLocal + def declare_chunk(self, domain, var, startdate, member, chunk, grid=None, region=None, box=None, frequency=None, + vartype=VariableType.MEAN, diagnostic=None): + """ + Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy + + :param diagnostic: + :param region: + :param domain: CMOR domain + :type domain: Domain + :param var: variable name + :type var: str + :param startdate: file's startdate + :type startdate: str + :param member: file's member + :type member: int + :param chunk: file's chunk + :type chunk: int + :param grid: file's grid (only needed if it is not the original) + :type grid: str|NoneType + :param box: file's box (only needed to retrieve sections or averages) + :type box: Box + :param frequency: file's frequency (only needed if it is different from the default) + :type frequency: Frequency|NoneType + :param vartype: Variable type (mean, statistic) + :type vartype: VariableType + :return: path to the copy created on the scratch folder + :rtype: str + """ + aggregation_path = self.get_var_url(var, startdate, frequency, box, vartype) + file_path = self.get_file_path(startdate, domain, var, frequency, vartype, box=box) + + start_chunk = chunk_start_date(parse_date(startdate), chunk, self.experiment.chunk_size, 'month', + self.experiment.calendar) + end_chunk = chunk_end_date(start_chunk, self.experiment.chunk_size, 'month', self.experiment.calendar) + final_name = self._get_final_var_name(box, var) + if file_path in self.requested_files: + thredds_subset = self.requested_files[file_path] + else: + thredds_subset = THREDDSSubset(aggregation_path, file_path, var, start_chunk, end_chunk) + self.requested_files[file_path] = thredds_subset + + thredds_subset.final_name = final_name + thredds_subset.diagnostic = diagnostic + thredds_subset.storage_status = StorageStatus.PENDING + return thredds_subset + class THREDDSError(Exception): pass -class THREDDSSubset: - def __init__(self, thredds_path, var, start_time, end_time): +class THREDDSSubset(DataFile): + def __init__(self, thredds_path, file_path, var, start_time, end_time): + """ + + :param thredds_path: + :param file_path: + :param var: + :type var: str + :param start_time: + :param end_time: + """ + super(THREDDSSubset, self).__init__() self.thredds_path = thredds_path - self.var = var + self.remote_file = file_path + if '_f' in var: + self.var = var[:var.index('_f')] + self.hourly = var[var.index('_f'):] + else: + self.var = var + self.hourly = '' self.dimension_indexes = {} self.handler = None self.start_time = start_time self.end_time = end_time - def get_url(self): - self.handler = Utils.openCdf(self.thredds_path) - self._read_metadata() - self.handler.close() - - self._get_time_indexes() - - return self._get_subset_url() + def __str__(self): + return 'THREDDS {0.thredds_path} ({0.start_time}-{0.end_time})'.format(self) def download(self): - url = self.get_url() - return self._download_url(url) - - def check(self): - # noinspection PyBroadException try: - self.handler = Utils.openCdf(self.get_url()) - self.handler.close() - return True - except Exception: - return False - - def _read_metadata(self): - self.var_dimensions = self.handler.variables[self.var].dimensions - for dimension in self.var_dimensions: - if dimension == 'time': - continue - self.dimension_indexes[dimension] = (0, self.handler.dimensions[dimension].size - 1) - - if 'time' in self.var_dimensions: - self.times = Utils.get_datetime_from_netcdf(self.handler) - - def _get_time_indexes(self): - if 'time' not in self.var_dimensions: + Log.debug('Downloading thredds subset {0}...', self) + iris.FUTURE.netcdf_promote = True + iris.FUTURE.netcdf_no_unlimited = True + with iris.FUTURE.context(cell_datetime_objects=True): + time_constraint = iris.Constraint(time=lambda cell: self.start_time <= cell.point <= self.end_time) + var_cube = iris.load_cube(self.thredds_path, constraint=time_constraint, callback=self._correct_cube) + + if not self.local_file: + self.local_file = TempFile.get() + iris.save(var_cube, self.local_file, zlib=True) + if not Utils.check_netcdf_file(self.local_file): + raise Exception('netcdf check for downloaded file failed') + Log.info('Request {0} ready!', self) + self.local_status = LocalStatus.READY + except Exception as ex: + Log.error('Can not retrieve {0} from server: {1}'.format(self, ex)) + self.local_status = LocalStatus.FAILED + + # noinspection PyUnusedLocal,PyMethodMayBeStatic + def _correct_cube(self, cube, field, filename): + if not cube.coords('time'): return - - time_start = 0 - while time_start < self.times.size and self.times[time_start] < self.start_time: - time_start += 1 - if time_start == self.times.size: - raise Exception('Timesteps not available for interval {0}-{1}'.format(self.start_time, self.end_time)) - time_end = time_start - if self.times[time_end] >= self.end_time: - raise Exception('Timesteps not available for interval {0}-{1}'.format(self.start_time, self.end_time)) - while time_end < self.times.size - 1 and self.times[time_end + 1] < self.end_time: - time_end += 1 - self.dimension_indexes['time'] = (time_start, time_end) - - @staticmethod - def _download_url(url): - temp = TempFile.get() - Utils.execute_shell_command(['nccopy', '-s', '-d', '-4', url, temp]) - if not Utils.check_netcdf_file(temp): - raise THREDDSError('Can not retrieve {0} from server'.format(url)) - return temp - - def _get_subset_url(self): - var_slice = self.var - dimensions_slice = '' - - for dimension in self.var_dimensions: - slice_index = self._get_slice_index(self.dimension_indexes[dimension]) - var_slice += slice_index - if dimension == 'ensemble': - dimension = 'realization' - dimensions_slice += '{0}{1},'.format(dimension, slice_index) - - return '{0}?{1}{2}'.format(self.thredds_path, dimensions_slice, var_slice) - - @staticmethod - def _get_slice_index(index_tuple): - return '[{0[0]}:1:{0[1]}]'.format(index_tuple) - - - + time = cube.coord('time') + if time.units.origin.startswith('month'): + ref = strptime(time.units.origin[time.units.origin.index(' since ') + 7:], '%Y-%m-%d %H:%M:%S') + helper = np.vectorize(lambda x: datetime(year=ref.tm_year + int(x) / 12, + month=int(x-1) % 12 + 1, + day=ref.tm_mday)) + times = np.round(time.points + ref.tm_mon) + dates = helper(times) + dates = netCDF4.date2num(dates, units='days since 1850-01-01', calendar=time.units.calendar) + new_time = DimCoord(dates, standard_name=time.standard_name, long_name=time.long_name, + var_name=time.var_name, attributes=time.attributes, + units=Unit('days since 1850-01-01', time.units.calendar)) + [dimension] = cube.coord_dims(time) + cube.remove_coord(time) + cube.add_dim_coord(new_time, dimension) diff --git a/earthdiagnostics/utils.py b/earthdiagnostics/utils.py index 2e4c57779456d5091a5d1ede3a4c404cdd58cc1f..42304aa02ed840edb038165a8b7a4fc8757d0990 100644 --- a/earthdiagnostics/utils.py +++ b/earthdiagnostics/utils.py @@ -1,25 +1,27 @@ # coding=utf-8 -import hashlib +import datetime +import os +import re import shutil +import stat import subprocess +import sys import tarfile +import tempfile +from contextlib import contextmanager +import iris +import iris.exceptions import netCDF4 import numpy as np -import os -import stat -import re -import tempfile - import six +import xxhash from bscearth.utils.log import Log -from cdo import Cdo, CDOException +from cdo import Cdo from cfunits import Units from nco import Nco from earthdiagnostics.constants import Basins -from contextlib import contextmanager -import sys @contextmanager @@ -33,6 +35,10 @@ def suppress_stdout(): sys.stdout = old_stdout +class File(object): + pass + + class Utils(object): """ Container class for miscellaneous utility methods @@ -77,6 +83,7 @@ class Utils(object): :param variable_list: list of variables in which valid_min and valid_max will be set :type variable_list: str | list """ + # noinspection PyTypeChecker if isinstance(variable_list, six.string_types): variable_list = variable_list.split() @@ -162,8 +169,9 @@ class Utils(object): def check_netcdf_file(filepath): with suppress_stdout(): try: - Utils.cdo.showvar(input=filepath) - except CDOException: + iris.FUTURE.netcdf_promote = True + iris.load(filepath) + except iris.exceptions.IrisError: return False return True @@ -191,6 +199,7 @@ class Utils(object): # noinspection PyPep8Naming @staticmethod def convert_to_ASCII_if_possible(string, encoding='ascii'): + # noinspection PyTypeChecker if isinstance(string, six.string_types): try: return string.encode(encoding) @@ -217,10 +226,12 @@ class Utils(object): handler.sync() @staticmethod - def copy_file(source, destiny): + def copy_file(source, destiny, save_hash=False): """ Copies a file from source to destiny, creating dirs if necessary + :param save_hash: if True, stores hash value in a file + :type save_hash: bool :param source: path to source :type source: str :param destiny: path to destiny @@ -236,17 +247,22 @@ class Utils(object): if not os.path.exists(dirname_path): raise ex hash_destiny = None - hash_original = Utils.get_file_hash(source) + Log.debug('Hashing original file... {0}', datetime.datetime.now()) + hash_original = Utils.get_file_hash(source, use_stored=True) - retrials = 5 + retrials = 3 while hash_original != hash_destiny: if retrials == 0: - raise Exception('Can not move {0} to {1}'.format(source, destiny)) + raise Exception('Can not copy {0} to {1}'.format(source, destiny)) + Log.debug('Copying... {0}', datetime.datetime.now()) shutil.copyfile(source, destiny) - hash_destiny = Utils.get_file_hash(destiny) + Log.debug('Hashing copy ... {0}', datetime.datetime.now()) + hash_destiny = Utils.get_file_hash(destiny, save=save_hash) + retrials -= 1 + Log.info('Finished {0}', datetime.datetime.now()) @staticmethod - def move_file(source, destiny): + def move_file(source, destiny, save_hash=False): """ Moves a file from source to destiny, creating dirs if necessary @@ -254,8 +270,10 @@ class Utils(object): :type source: str :param destiny: path to destiny :type destiny: str + :param save_hash: if True, stores hash value in a file + :type save_hash: bool """ - Utils.copy_file(source, destiny) + Utils.copy_file(source, destiny, save_hash) os.remove(source) @staticmethod @@ -289,27 +307,50 @@ class Utils(object): shutil.rmtree(source) @staticmethod - def get_file_hash(filepath): + def get_file_hash(filepath, use_stored=False, save=False): """ - Returns the MD5 hash for the given filepath + Returns the xxHash hash for the given filepath :param filepath: path to the file to compute hash on :type filepath:str - :return: file's MD5 hash + :param use_stored: if True, try to read the hash value from file + :type use_stored: bool + :param save: if True, stores hash value in a file + :type save: bool + :return: file's xxHash hash :rtype: str """ - blocksize = 65536 - hasher = hashlib.md5() + if use_stored: + hash_file = Utils._get_hash_filename(filepath) + if os.path.isfile(hash_file): + hash_value = open(hash_file, 'r').readline() + return hash_value + + blocksize = 104857600 + hasher = xxhash.xxh64() with open(filepath, 'rb') as afile: buf = afile.read(blocksize) while len(buf) > 0: hasher.update(buf) buf = afile.read(blocksize) - return hasher.hexdigest() + hash_value = hasher.hexdigest() + if save: + hash_file = open(Utils._get_hash_filename(filepath), 'w') + hash_file.write(hash_value) + hash_file.close() + + return hash_value + + @staticmethod + def _get_hash_filename(filepath): + folder = os.path.dirname(filepath) + filename = os.path.basename(filepath) + hash_file = os.path.join(folder, '.{0}.xxhash64.hash'.format(filename)) + return hash_file @staticmethod def execute_shell_command(command, log_level=Log.DEBUG): """ - Executes a sheel command + Executes a sheel commandsi :param command: command to execute Log.info('Detailed time for diagnostic class') @@ -318,6 +359,7 @@ class Utils(object): :return: command output :rtype: list """ + # noinspection PyTypeChecker if isinstance(command, six.string_types): command = command.split() process = subprocess.Popen(command, stdout=subprocess.PIPE) @@ -587,7 +629,7 @@ class Utils(object): # noinspection PyBroadException try: os.makedirs(path) - except: + except Exception: # Here we can have a race condition. Let's check again for existence and rethrow if still not exists if not os.path.isdir(path): raise @@ -645,6 +687,7 @@ class Utils(object): :param force: if True, it will overwrite unzipped files :type force: bool """ + # noinspection PyTypeChecker if isinstance(files, six.string_types): files = [files] for filepath in files: @@ -724,3 +767,4 @@ class TempFile(object): if os.path.exists(temp_file): os.remove(temp_file) TempFile.files = list() + diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index dc5745fd724c1ea4f90794292ff5904b9964bb4e..9a623469694438ea5bf71afd49c703fd868cb448 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -18,8 +18,6 @@ class VariableJsonException(Exception): class VariableManager(object): - __metaclass__ = SingletonType - def __init__(self): self._cmor_tables_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'cmor_tables') self._aliases_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'variable_alias') @@ -358,7 +356,7 @@ class Variable(object): parsed[0]) return parsed[0] - if not domains[0]: + elif len(domains) == 0: Log.warning('Variable {0} has no modeling realm defined'.format(self.short_name)) return None else: @@ -374,9 +372,6 @@ class Variable(object): self.valid_min = var_line[7].strip() self.valid_max = var_line[8].strip() self.grid = var_line[9].strip() - for table in var_line[10].strip().split(':'): - if table: - self.add_table(table) def get_table(self, frequency, data_convention): for table, priority in self.tables: @@ -385,7 +380,7 @@ class Variable(object): if self.domain: table_name = self.domain.get_table_name(frequency, data_convention) return CMORTable(table_name, frequency, 'December 2013') - return self.tables[0] + raise ValueError('Can not get table for {0} and frequency {1}'.format(self, frequency)) @staticmethod def _select_most_specific(parsed): diff --git a/earthdiagnostics/variable_alias/cmip6.csv b/earthdiagnostics/variable_alias/cmip6.csv index 63708fcb0e9725d1a29c4e9aaeef2495ee6cbed7..e6c8a679f5ea3fc1969546c0f8a6de6d71769da4 100644 --- a/earthdiagnostics/variable_alias/cmip6.csv +++ b/earthdiagnostics/variable_alias/cmip6.csv @@ -71,3 +71,11 @@ mun,mun,, mud,mud,, ppnewn,ppnewn,, ppnewd,ppnewd,, +alb_ice,sialb,, +qsr3d,rsdo,, +hflx_rnf_cea,hfrunoffds2d,, +hflx_rain_cea,hfrainds,, +hflx_cal_cea,hfibthermds2d,, +rain,prra,, +calving,ficeberg2d,, + diff --git a/earthdiagnostics/variable_alias/default.csv b/earthdiagnostics/variable_alias/default.csv index 274b0b756484c0c4204e2b9fcdacfbedd1cbbabe..1baafce74f5d88a9bcefb2ef5aa17152fa7d28b2 100644 --- a/earthdiagnostics/variable_alias/default.csv +++ b/earthdiagnostics/variable_alias/default.csv @@ -298,12 +298,5 @@ qtr_ice,qtr,, var78,tclw,, var79,tciw,, rho,rhopoto,, -alb_ice,sialb,, qsr,rsntds,, -qsr3d,rsdo,, -hflx_rnf_cea,hfrunoffds2d,, -hflx_rain_cea,hfrainds,, -hflx_cal_cea,hfibthermds2d,, -rain,prra,, -runoffs,friver,, -calving,ficeberg2d,, \ No newline at end of file +runoffs,friver,, \ No newline at end of file diff --git a/earthdiagnostics/variable_alias/primavera.csv b/earthdiagnostics/variable_alias/primavera.csv index cc704fb78757ecf18a133b5fe4a4655ab0185b63..db38cd5c1c7448696e8583c0d94739cce1790e54 100644 --- a/earthdiagnostics/variable_alias/primavera.csv +++ b/earthdiagnostics/variable_alias/primavera.csv @@ -1,2 +1,9 @@ Aliases,Shortname,Basin,Grid iiceconc:siconc:soicecov:ileadfra:sic,siconc,, +alb_ice,sialb,, +qsr3d,rsdo,, +hflx_rnf_cea,hfrunoffds2d,, +hflx_rain_cea,hfrainds,, +hflx_cal_cea,hfibthermds2d,, +rain,prra,, +calving,ficeberg2d,, diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..ef3a62ea119458e629c15ecc752d412f837be9f0 --- /dev/null +++ b/earthdiagnostics/work_manager.py @@ -0,0 +1,285 @@ +# coding=utf-8 +import datetime +import operator +import threading +import time + +from bscearth.utils.log import Log +# noinspection PyCompatibility +from concurrent.futures import ThreadPoolExecutor + +from earthdiagnostics.datafile import StorageStatus, LocalStatus +from earthdiagnostics.diagnostic import DiagnosticStatus, Diagnostic, DiagnosticOptionError +from earthdiagnostics.general import * +from earthdiagnostics.ocean import * +from earthdiagnostics.statistics import * +from earthdiagnostics.utils import Utils, TempFile + + +class WorkManager(object): + + def __init__(self, config, data_manager): + self.jobs = None + self.config = config + self.time = {} + self.had_errors = False + self.data_manager = data_manager + + def prepare_job_list(self): + self._register_diagnostics() + list_jobs = list() + for fulldiag in self.config.get_commands(): + Log.info("Adding {0} to diagnostic list", fulldiag) + diag_options = fulldiag.split(',') + + diag_class = Diagnostic.get_diagnostic(diag_options[0]) + if diag_class: + try: + for job in diag_class.generate_jobs(self, diag_options): + list_jobs.append(job) + continue + except DiagnosticOptionError as ex: + Log.error('Can not configure diagnostic {0}: {1}', diag_options[0], ex) + self.had_errors = True + else: + Log.error('{0} is not an available diagnostic', diag_options[0]) + self.had_errors = True + self.jobs = list_jobs + + def run(self): + time = datetime.datetime.now() + Log.info("Starting to compute at {0}", time) + self.threads = Utils.available_cpu_count() + if 0 < self.config.max_cores < self.threads: + self.threads = self.config.max_cores + Log.info('Using {0} threads', self.threads) + + self.downloader = Downloader() + self.uploader = ThreadPoolExecutor(self.config.parallel_uploads) + self.executor = ThreadPoolExecutor(self.threads) + + for job in self.jobs: + job.request_data() + job.declare_data_generated() + job.subscribe(self, self._job_status_changed) + for subjob in job.subjobs: + subjob.subscribe(self, self._job_status_changed) + job.check_is_ready() + + for file_object in self.data_manager.requested_files.values(): + file_object.subscribe(self, self._file_object_status_changed) + if file_object.download_required(): + self.downloader.submit(file_object) + + self.downloader.start() + self.lock = threading.Lock() + self.lock.acquire() + + self.check_completion() + self.lock.acquire() + + self.downloader.shutdown() + self.executor.shutdown() + self.uploader.shutdown(True) + + TempFile.clean() + finish_time = datetime.datetime.now() + Log.result("Diagnostics finished at {0}", finish_time) + Log.result("Elapsed time: {0}\n", finish_time - time) + self.print_errors() + self.print_stats() + return not self.had_errors + + def _job_status_changed(self, job): + if job.status == DiagnosticStatus.READY: + self.executor.submit(self._run_job, job) + self.check_completion() + + def _file_object_status_changed(self, file_object): + if file_object.download_required(): + self.downloader.submit(file_object) + return + if file_object.upload_required(): + self.uploader.submit(file_object.upload) + return + self.check_completion() + + def check_completion(self): + for job in self.jobs: + if job.status in (DiagnosticStatus.READY, DiagnosticStatus.RUNNING): + return False + + if job.status == DiagnosticStatus.WAITING: + if job.all_requests_in_storage(): + return False + + for request in self.data_manager.requested_files.values(): + if request.storage_status == StorageStatus.UPLOADING: + return + if request.local_status == LocalStatus.DOWNLOADING: + return + if request.upload_required(): + return + if request.download_required(): + return + self.lock.release() + return True + + def print_stats(self): + Log.info('Time consumed by each diagnostic class') + Log.info('--------------------------------------') + + times = {} + for job in self.jobs: + job_type = job.alias + if job_type in times.keys(): + times[job_type] += job.consumed_time + else: + times[job_type] = job.consumed_time + + for diag in sorted(times, key=operator.itemgetter(1)): + Log.info('{0:23} {1:}', diag, times[diag]) + + def print_errors(self): + failed = [job for job in self.jobs if job.status == DiagnosticStatus.FAILED] + if len(failed) == 0: + return + self.had_errors = True + Log.error('Failed jobs') + Log.error('-----------') + for job in failed: + Log.error('{0}: {0.message}', job) + Log.error('Total wasted time: {0}', sum([job.consumed_time for job in failed], datetime.timedelta())) + Log.info('') + + @staticmethod + def _run_job(job): + time = datetime.datetime.now() + try: + Log.info('Starting {0}', job) + job.status = DiagnosticStatus.RUNNING + job.compute() + except Exception as ex: + job.consumed_time = datetime.datetime.now() - time + job.message = str(ex) + Log.error('Job {0} failed: {1}', job, ex) + job.status = DiagnosticStatus.FAILED + return False + + job.consumed_time = datetime.datetime.now() - time + Log.result('Finished {0}', job) + job.status = DiagnosticStatus.COMPLETED + return True + count = 0 + failed_jobs = list() + + @staticmethod + def _register_diagnostics(): + WorkManager._register_ocean_diagnostics() + WorkManager._register_general_diagnostics() + WorkManager._register_stats_diagnostics() + + @staticmethod + def _register_stats_diagnostics(): + Diagnostic.register(MonthlyPercentile) + Diagnostic.register(ClimatologicalPercentile) + Diagnostic.register(DaysOverPercentile) + Diagnostic.register(Discretize) + + @staticmethod + def _register_general_diagnostics(): + Diagnostic.register(DailyMean) + Diagnostic.register(MonthlyMean) + Diagnostic.register(YearlyMean) + Diagnostic.register(Rewrite) + Diagnostic.register(Relink) + Diagnostic.register(RelinkAll) + Diagnostic.register(Scale) + Diagnostic.register(Attribute) + Diagnostic.register(Module) + Diagnostic.register(VerticalMeanMetersIris) + Diagnostic.register(SimplifyDimensions) + + @staticmethod + def _register_ocean_diagnostics(): + Diagnostic.register(MixedLayerSaltContent) + Diagnostic.register(Siasiesiv) + Diagnostic.register(VerticalMean) + Diagnostic.register(VerticalMeanMeters) + Diagnostic.register(Interpolate) + Diagnostic.register(InterpolateCDO) + Diagnostic.register(Moc) + Diagnostic.register(AreaMoc) + Diagnostic.register(MaxMoc) + Diagnostic.register(Psi) + Diagnostic.register(Gyres) + Diagnostic.register(ConvectionSites) + Diagnostic.register(CutSection) + Diagnostic.register(AverageSection) + Diagnostic.register(MixedLayerHeatContent) + Diagnostic.register(HeatContentLayer) + Diagnostic.register(HeatContent) + Diagnostic.register(RegionMean) + Diagnostic.register(Rotation) + Diagnostic.register(VerticalGradient) + + +class Downloader(object): + def __init__(self): + self._downloads = [] + self._lock = threading.Lock() + self._wait = threading.Semaphore() + self.stop = False + + def start(self): + self._thread = threading.Thread(target=self.downloader) + self._thread.start() + + def submit(self, datafile): + self._lock.acquire() + self._downloads.append(datafile) + self._lock.release() + + def downloader(self): + try: + def suscribers_waiting(datafile): + waiting = 0 + for diag in datafile.suscribers: + if not isinstance(diag, Diagnostic): + continue + if diag.pending_requests() == 1: + waiting += 1 + return waiting + + def prioritize(datafile1, datafile2): + waiting = suscribers_waiting(datafile1) - suscribers_waiting(datafile2) + if waiting: + return -waiting + + suscribers = len(datafile1.suscribers) - len(datafile2.suscribers) + if suscribers: + return -suscribers + + size = datafile1.size - datafile2.size + if size: + return -size + return 0 + + while True: + with self._lock: + if len(self._downloads) == 0: + if self.stop: + return + time.sleep(0.01) + break + self._downloads.sort(prioritize) + datafile = self._downloads[0] + self._downloads.remove(datafile) + datafile.download() + except Exception as ex: + Log.critical('Unhandled error at downloader: {0}', ex) + + def shutdown(self): + self.stop = True + self._thread.join() + diff --git a/launch_diags.sh b/launch_diags.sh index bbce66f32d8831fcd94074842312462e1c02089e..477a1ca4069f72efad559e9052a14227487b5318 100755 --- a/launch_diags.sh +++ b/launch_diags.sh @@ -1,24 +1,26 @@ #!/usr/bin/env bash -#SBATCH -n 1 -#SBATCH -w gustafson -#SBATCH --time 72:00:00 +#SBATCH -n 4 +#SBATCH --time 7-00:00:00 #SBATCH --error=job.%J.err #SBATCH --output=job.%J.out -set -xv -PATH_TO_CONF_FILE=~/earthdiagnostics/diags.conf -PATH_TO_DIAGNOSTICS=~/earthdiagnostics -PATH_TO_VIRTUALENV=~jvegas/virtualenvs/diags/bin + +PATH_TO_CONF_FILE=~jvegas/earthdiagnostics/diags.conf +PATH_TO_DIAGNOSTICS=~jvegas/earthdiagnostics +PATH_TO_CONDAENV=diags module purge module load NCO/4.5.4-foss-2015a module load CDO/1.7.2-foss-2015a -module load CDFTOOLS/3.0a5-foss-2015a +module load CDFTOOLS/3.0a8-foss-2015a +module load Miniconda2 + +set -xv -source ${PATH_TO_VIRTUALENV}/activate +source activate diags export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH} cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/ -./earthdiags.py -f ${PATH_TO_CONF_FILE} +./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE} diff --git a/model_diags.conf b/model_diags.conf index 9edfcef0ed30ea723d52e93455c4f4c5965c189c..e97f27731821d14e8013b25c3782f0de06a793fc 100644 --- a/model_diags.conf +++ b/model_diags.conf @@ -121,7 +121,7 @@ CHUNKS = # If true, CMORizes atmosphere files. Default = True # ATMOSPHERE_FILES = True -# You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var +# You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var, i.e ocean:thetao atmos:tas # VARIABLE_LIST = # Variables to be CMORized from the grib atmospheric files, separated by comma. diff --git a/model_launch_diags.sh b/model_launch_diags.sh new file mode 100755 index 0000000000000000000000000000000000000000..7a226961832a92029c8617ede91486743f48ef07 --- /dev/null +++ b/model_launch_diags.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +#SBATCH -n 1 +#SBATCH --time 7-00:00:00 +#SBATCH --error=earthdiags.%J.err +#SBATCH --output=earthdiags.%J.out + +PATH_TO_CONF_FILE=~jvegas/earthdiagnostics/diags.conf + +module purge +module load earthdiagnostics + +set -xv + +earthdiags -lc DEBUG -f ${PATH_TO_CONF_FILE} diff --git a/setup.py b/setup.py index 2f8ecf7faec2aa3471a378e1f0479d6733e75956..411833062675cf4e5a4c926c2a544d0d5a599b32 100644 --- a/setup.py +++ b/setup.py @@ -5,8 +5,8 @@ Installation script for EarthDiagnostics package """ from os import path -from setuptools import setup from setuptools import find_packages +from setuptools import setup here = path.abspath(path.dirname(__file__)) @@ -25,8 +25,9 @@ setup( url='http://www.bsc.es/projects/earthsciences/autosubmit/', keywords=['climate', 'weather', 'diagnostic'], setup_requires=['pyproj'], - install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo==1.3.0', 'nco>=0.0.3', 'cfunits>=1.1.4', 'coverage', - 'pygrib', 'openpyxl', 'mock'], + install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo>=1.3.4', 'nco>=0.0.3', 'iris>=1.12.0', 'coverage', + 'pygrib', 'openpyxl', 'mock', 'futures', 'cf_units', 'cfunits', 'xxhash', 'six', 'psutil', + 'exrex'], packages=find_packages(), include_package_data=True, scripts=['bin/earthdiags'] diff --git a/test.py b/test.py deleted file mode 100644 index a6785ecb07d4ac8b805008f56b6e62194a127096..0000000000000000000000000000000000000000 --- a/test.py +++ /dev/null @@ -1,30 +0,0 @@ -# coding=utf-8 -""" -Script to run the tests for EarthDiagnostics and generate the code coverage report -""" -import coverage -import unittest -import os -cov = coverage.Coverage() -cov.set_option("run:branch", True) -cov.start() - -# noinspection PyPep8 -import test.unit - -suite = unittest.TestLoader().loadTestsFromModule(test.unit) -unittest.TextTestRunner(verbosity=2).run(suite) -cov.stop() -cov.save() - -source_files = list() -for path, dirs, files in os.walk('earthdiagnostics'): - for filename in files: - if filename.endswith('.py'): - source_files.append(os.path.join(path, filename)) - -cov.report(source_files) -cov.html_report(source_files) - - - diff --git a/test/run_test.py b/test/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..59eec7e98a18694833f660a50431d2ea696de630 --- /dev/null +++ b/test/run_test.py @@ -0,0 +1,26 @@ +# coding=utf-8 +""" +Script to run the tests for EarthDiagnostics and generate the code coverage report +""" + +import coverage +import unittest +import os +work_path = os.path.abspath('.') +source_path = os.path.join(work_path, '..', 'earthdiagnostics', '*') +print(source_path) +cov = coverage.Coverage(include=source_path) +cov.set_option("run:branch", True) +cov.set_option("html:title", 'Coverage report for EarthDiagnostics') + +cov.start() +suite = unittest.TestLoader().discover('.') +unittest.TextTestRunner(verbosity=2).run(suite) +cov.stop() + +cov.save() +cov.report() +cov.html_report() + + + diff --git a/test/unit/__init__.py b/test/unit/__init__.py index c6c22c08a512bc55b0fa9e8315ebae360f21110f..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -1,30 +1 @@ # coding=utf-8 -from test_data_manager import TestConversion -# from test.unit.test_variable import TestVariable -from test_constants import TestBasin -from test_box import TestBox -from test_diagnostic import * -from test_cdftools import TestCDFTools -from test_utils import TestTempFile, TestUtils -from test_psi import TestPsi -from test_areamoc import TestAreaMoc -from test_averagesection import TestAverageSection -from test_cutsection import TestCutSection -from test_convectionsites import TestConvectionSites -from test_frequency import TestFrequency -from test_gyres import TestGyres -from test_heatcontentlayer import TestHeatContentLayer -from test_maxmoc import TestMaxMoc -from test_mixedlayerheatcontent import TestMixedLayerHeatContent -from test_mixedlayersaltcontent import TestMixedLayerSaltContent -from test_moc import TestMoc -from test_modelling_realm import TestModellingRealms, TestModellingRealm -from test_siasiesiv import TestSiasiesiv -from test_verticalmean import TestVerticalMean -from test_verticalmeanmeters import TestVerticalMeanMeters -from test_monthlymean import TestMonthlyMean -from test_rewrite import TestRewrite -from test_variable_type import TestVariableType -from test_monthlypercentile import TestMonthlyPercentile -from test_climatologicalpercentile import TestClimatologicalPercentile -from test_variable import TestCMORTable, TestVariableAlias diff --git a/test/unit/general/__init__.py b/test/unit/general/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/unit/general/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/unit/general/test_attribute.py b/test/unit/general/test_attribute.py new file mode 100644 index 0000000000000000000000000000000000000000..da6d4146b090a3e529678a332f1ba9fe705a8eb1 --- /dev/null +++ b/test/unit/general/test_attribute.py @@ -0,0 +1,54 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.attribute import Attribute +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestAttribute(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = Attribute.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'att', 'value']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', '', + 'att', 'value')) + self.assertEqual(jobs[1], Attribute(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', '', + 'att', 'value')) + + jobs = Attribute.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', 'att', 'value', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', 'grid', + 'att', 'value')) + self.assertEqual(jobs[1], Attribute(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', 'grid', + 'att', 'value')) + + with self.assertRaises(Exception): + Attribute.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + Attribute.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 'att', 'value') + self.assertEquals(str(mixed), 'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 ' + 'Variable: atmos:var Attributte: att:value Grid: grid') diff --git a/test/unit/general/test_dailymean.py b/test/unit/general/test_dailymean.py new file mode 100644 index 0000000000000000000000000000000000000000..ec85d9dcea51e903b8be04b8f3c57b4a9769eb2b --- /dev/null +++ b/test/unit/general/test_dailymean.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.dailymean import DailyMean +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestDailyMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = DailyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '6hr']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], DailyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + Frequencies.six_hourly, '')) + self.assertEqual(jobs[1], DailyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + Frequencies.six_hourly, '')) + + jobs = DailyMean.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', '3h', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], DailyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', + Frequencies.three_hourly, 'grid')) + self.assertEqual(jobs[1], DailyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', + Frequencies.three_hourly, 'grid')) + + with self.assertRaises(Exception): + DailyMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + DailyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = DailyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + self.assertEquals(str(mixed), 'Calculate daily mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/general/test_module.py b/test/unit/general/test_module.py new file mode 100644 index 0000000000000000000000000000000000000000..e01ecf8c277db292cc8c786f399f473caa826229 --- /dev/null +++ b/test/unit/general/test_module.py @@ -0,0 +1,54 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.module import Module +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestModule(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = Module.generate_jobs(self.diags, ['diagnostic', 'atmos', 'varu', 'varv', 'varmodule']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, + 'varu', 'varv', 'varmodule', '')) + self.assertEqual(jobs[1], Module(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, + 'varu', 'varv', 'varmodule', '')) + + jobs = Module.generate_jobs(self.diags, ['diagnostic', 'seaIce', 'varu', 'varv', 'varmodule', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Module(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, + 'varu', 'varv', 'varmodule', 'grid')) + self.assertEqual(jobs[1], Module(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, + 'varu', 'varv', 'varmodule', 'grid')) + + with self.assertRaises(Exception): + Module.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + Module.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = Module(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'varu', 'varv', 'varmodule', 'grid') + self.assertEquals(str(mixed), 'Calculate module Startdate: 20010101 Member: 0 Chunk: 0 ' + 'Variables: atmos:varu,varv,varmodule Grid: grid') diff --git a/test/unit/test_monthlymean.py b/test/unit/general/test_monthlymean.py similarity index 89% rename from test/unit/test_monthlymean.py rename to test/unit/general/test_monthlymean.py index e2165f5dee2f7bb65259926f1c946ea3801af080..b31561ac1fbfe9a9d8e40e80a9329b141cb27ed8 100644 --- a/test/unit/test_monthlymean.py +++ b/test/unit/general/test_monthlymean.py @@ -1,10 +1,11 @@ # coding=utf-8 from unittest import TestCase +from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.box import Box from earthdiagnostics.frequency import Frequencies from earthdiagnostics.general.monthlymean import MonthlyMean -from mock import Mock +from mock import Mock, patch from earthdiagnostics.modelingrealm import ModelingRealms @@ -24,23 +25,27 @@ class TestMonthlyMean(TestCase): self.mixed = MonthlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): - jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'var', 'ocean']) + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', Frequencies.daily, '')) self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', Frequencies.daily, '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'var', 'atmos', 'monthly']) + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'monthly']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', Frequencies.monthly, '')) self.assertEqual(jobs[1], MonthlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', Frequencies.monthly, '')) - jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'var', 'seaice', 'mon', 'grid']) + jobs = MonthlyMean.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', 'mon', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', Frequencies.monthly, 'grid')) diff --git a/test/unit/general/test_relink.py b/test/unit/general/test_relink.py new file mode 100644 index 0000000000000000000000000000000000000000..835b0bcd6f944847430720abade1e52dc95357b3 --- /dev/null +++ b/test/unit/general/test_relink.py @@ -0,0 +1,61 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.relink import Relink +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestRelink(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = Relink.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, + 'var', True, '')) + self.assertEqual(jobs[1], Relink(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, + 'var', True, '')) + + jobs = Relink.generate_jobs(self.diags, ['diagnostic', 'seaIce', 'var', 'False']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, + 'var', False, '')) + self.assertEqual(jobs[1], Relink(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, + 'var', False, '')) + + jobs = Relink.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'True', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, + 'var', True, 'grid')) + self.assertEqual(jobs[1], Relink(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, + 'var', True, 'grid')) + + with self.assertRaises(Exception): + Relink.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + Relink.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + mixed = Relink(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', True, 'grid') + self.assertEquals(str(mixed), 'Relink output Startdate: 20010101 Member: 0 Chunk: 0 Move old: True ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/general/test_relinkall.py b/test/unit/general/test_relinkall.py new file mode 100644 index 0000000000000000000000000000000000000000..cf8c9a163437d5d868401a54b6df62195d0e1f7a --- /dev/null +++ b/test/unit/general/test_relinkall.py @@ -0,0 +1,39 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.general.relinkall import RelinkAll +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestRelinkAll(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = RelinkAll.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0], RelinkAll(self.data_manager, '20010101')) + + with self.assertRaises(Exception): + RelinkAll.generate_jobs(self.diags, ['diagnostic', '0']) + + def test_str(self): + mixed = RelinkAll(self.data_manager, '20010101') + self.assertEquals(str(mixed), 'Relink all output Startdate: 20010101') diff --git a/test/unit/test_rewrite.py b/test/unit/general/test_rewrite.py similarity index 87% rename from test/unit/test_rewrite.py rename to test/unit/general/test_rewrite.py index 25380fcd81a752a85e679ff93787aa2ec4d0bf77..202e2c39afddfaae8e23f3edd2be7ff3f74a336d 100644 --- a/test/unit/test_rewrite.py +++ b/test/unit/general/test_rewrite.py @@ -1,9 +1,10 @@ # coding=utf-8 from unittest import TestCase +from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.box import Box from earthdiagnostics.general.rewrite import Rewrite -from mock import Mock +from mock import Mock, patch from earthdiagnostics.modelingrealm import ModelingRealms @@ -23,14 +24,18 @@ class TestRewrite(TestCase): self.mixed = Rewrite(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', 'grid') + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): - jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'var', 'atmos']) + jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'original')) self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'original')) - jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'var', 'ocean', 'grid']) + jobs = Rewrite.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], Rewrite(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid')) self.assertEqual(jobs[1], Rewrite(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid')) diff --git a/test/unit/general/test_scale.py b/test/unit/general/test_scale.py new file mode 100644 index 0000000000000000000000000000000000000000..e7697cc26076c55e7a8d50ea65031a74305d3ac0 --- /dev/null +++ b/test/unit/general/test_scale.py @@ -0,0 +1,71 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.scale import Scale +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestScale(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', '', + float('nan'), float('nan'), Frequencies.monthly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', '', + float('nan'), float('nan'), Frequencies.monthly)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + float('nan'), float('nan'), Frequencies.monthly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + float('nan'), float('nan'), Frequencies.monthly)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.monthly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.monthly)) + + jobs = Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly)) + self.assertEqual(jobs[1], Scale(self.data_manager, '20010101', 0, 1, 0, 0, ModelingRealms.atmos, 'var', 'grid', + 0, 100, Frequencies.three_hourly)) + + with self.assertRaises(DiagnosticOptionError): + Scale.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + Scale.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '0', 'grid', '0', '100', '3hr', + 'extra']) + + def test_str(self): + mixed = Scale(self.data_manager, '20010101', 0, 0, 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 100, + Frequencies.three_hourly) + self.assertEquals(str(mixed), 'Scale output Startdate: 20010101 Member: 0 Chunk: 0 Scale value: 0 Offset: 0 ' + 'Variable: atmos:var Frequency: 3hr') diff --git a/test/unit/general/test_select_levels.py b/test/unit/general/test_select_levels.py new file mode 100644 index 0000000000000000000000000000000000000000..32e7424da69948ecf986eb8295b3e4a0b03bb168 --- /dev/null +++ b/test/unit/general/test_select_levels.py @@ -0,0 +1,66 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.select_levels import SelectLevels +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestSelectLevels(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '20']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + '', 0, 20)) + self.assertEqual(jobs[1], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + '', 0, 20)) + + jobs = SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var1-var2', '0', '20']) + self.assertEqual(len(jobs), 4) + self.assertEqual(jobs[0], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var1', + '', 0, 20)) + self.assertEqual(jobs[1], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var1', + '', 0, 20)) + self.assertEqual(jobs[2], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var2', + '', 0, 20)) + self.assertEqual(jobs[3], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var2', + '', 0, 20)) + + jobs = SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '20', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + 'grid', 0, 20)) + self.assertEqual(jobs[1], SelectLevels(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + 'grid', 0, 20)) + + with self.assertRaises(DiagnosticOptionError): + SelectLevels.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + SelectLevels.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', '0', '20', 'grid', 'extra']) + + def test_str(self): + mixed = SelectLevels(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', 0, 20) + self.assertEquals(str(mixed), 'Select levels Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Levels: 0-20 Grid: grid') diff --git a/test/unit/general/test_simplify_dimensions.py b/test/unit/general/test_simplify_dimensions.py new file mode 100644 index 0000000000000000000000000000000000000000..429ad6f2344ed5d5dd4f67d38c89ed53b1ffd807 --- /dev/null +++ b/test/unit/general/test_simplify_dimensions.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.simplify_dimensions import SimplifyDimensions +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestSimplifyDimensions(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SimplifyDimensions(self.data_manager, '20010101', 0, 0, + ModelingRealms.atmos, 'var', '')) + self.assertEqual(jobs[1], SimplifyDimensions(self.data_manager, '20010101', 0, 1, + ModelingRealms.atmos, 'var', '')) + + jobs = SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], SimplifyDimensions(self.data_manager, '20010101', 0, 0, + ModelingRealms.atmos, 'var', 'grid')) + self.assertEqual(jobs[1], SimplifyDimensions(self.data_manager, '20010101', 0, 1, + ModelingRealms.atmos, 'var', 'grid')) + + with self.assertRaises(DiagnosticOptionError): + SimplifyDimensions.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + SimplifyDimensions.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'grid', 'extra']) + + def test_str(self): + mixed = SimplifyDimensions(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid') + self.assertEquals(str(mixed), 'Simplify dimension Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Grid: grid') diff --git a/test/unit/general/test_verticalmeanmetersiris.py b/test/unit/general/test_verticalmeanmetersiris.py new file mode 100644 index 0000000000000000000000000000000000000000..cd2876fe9f2a62b91ee28616a5dee862b8f79dbf --- /dev/null +++ b/test/unit/general/test_verticalmeanmetersiris.py @@ -0,0 +1,74 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.general.verticalmeanmetersiris import VerticalMeanMetersIris +from earthdiagnostics.frequency import Frequencies +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestVerticalMeanMetersIris(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.startdates = ['20010101', ] + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + box = Box(True) + + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, + ModelingRealms.ocean, 'var', box)) + self.assertEqual(jobs[1], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 1, + ModelingRealms.ocean, 'var', box)) + + box = Box(True) + box.min_depth = 0 + box.max_depth = 100 + + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, + ModelingRealms.ocean, 'var', box)) + self.assertEqual(jobs[1], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 1, + ModelingRealms.ocean, 'var', box)) + + jobs = VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100', 'seaIce']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, + ModelingRealms.seaIce, 'var', box)) + self.assertEqual(jobs[1], VerticalMeanMetersIris(self.data_manager, '20010101', 0, 1, + ModelingRealms.seaIce, 'var', box)) + + with self.assertRaises(DiagnosticOptionError): + VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + VerticalMeanMetersIris.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100', 'seaIce', 'extra']) + + def test_str(self): + box = Box(True) + box.min_depth = 0 + box.max_depth = 100 + mixed = VerticalMeanMetersIris(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', box) + self.assertEquals(str(mixed), 'Vertical mean meters Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Box: 0-100m') diff --git a/test/unit/general/test_yearlymean.py b/test/unit/general/test_yearlymean.py new file mode 100644 index 0000000000000000000000000000000000000000..dcf5ad75ea16b60f061c44d17e8eada524cdca56 --- /dev/null +++ b/test/unit/general/test_yearlymean.py @@ -0,0 +1,63 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from earthdiagnostics.box import Box +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.general.yearlymean import YearlyMean +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestYearlyMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.frequency = Frequencies.monthly + + self.box = Box() + self.box.min_depth = 0 + self.box.max_depth = 100 + + self.mixed = YearlyMean(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', 'freq', '') + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + Frequencies.monthly, '')) + self.assertEqual(jobs[1], YearlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + Frequencies.monthly, '')) + + jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'atmos', 'var', 'day']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', + Frequencies.daily, '')) + self.assertEqual(jobs[1], YearlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', + Frequencies.daily, '')) + + jobs = YearlyMean.generate_jobs(self.diags, ['diagnostic', 'seaice', 'var', 'mon', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], YearlyMean(self.data_manager, '20010101', 0, 0, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) + self.assertEqual(jobs[1], YearlyMean(self.data_manager, '20010101', 0, 1, ModelingRealms.seaIce, 'var', + Frequencies.monthly, 'grid')) + + with self.assertRaises(Exception): + YearlyMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + YearlyMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + self.assertEquals(str(self.mixed), 'Calculate yearly mean Startdate: 20000101 Member: 1 Chunk: 1 ' + 'Variable: ocean:var Original frequency: freq Grid: ') diff --git a/test/unit/ocean/__init__.py b/test/unit/ocean/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/unit/ocean/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/unit/test_areamoc.py b/test/unit/ocean/test_areamoc.py similarity index 66% rename from test/unit/test_areamoc.py rename to test/unit/ocean/test_areamoc.py index 15d4bcd750f752c2989c7f259fb45b201f1d3d58..c1361b8393676bc6fee3106ebf1bd451b34abd92 100644 --- a/test/unit/test_areamoc.py +++ b/test/unit/ocean/test_areamoc.py @@ -2,9 +2,9 @@ from unittest import TestCase from earthdiagnostics.box import Box -from earthdiagnostics.constants import Basins +from earthdiagnostics.constants import Basins, Basin from earthdiagnostics.ocean.areamoc import AreaMoc -from mock import Mock +from mock import Mock, patch class TestAreaMoc(TestCase): @@ -12,6 +12,9 @@ class TestAreaMoc(TestCase): def setUp(self): self.data_manager = Mock() self.diags = Mock() + self.basins = Mock() + self.basins.Global = Basin('Global') + self.basins.Atlantic = Basin('Atlantic') self.box = Box() self.box.min_lat = 0 @@ -20,18 +23,29 @@ class TestAreaMoc(TestCase): self.box.max_depth = 0 self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.psi = AreaMoc(self.data_manager, '20000101', 1, 1, Basins.Antarctic, self.box) + self.psi = AreaMoc(self.data_manager, '20000101', 1, 1, self.basins.Atlantic, self.box) + def fake_parse(self, value): + if type(value) is Basin: + return value + if value == 'atl': + value = 'Atlantic' + else: + value = 'Global' + + return Basin(value) + + @patch.object(Basins, 'parse', fake_parse) def test_generate_jobs(self): jobs = AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, Basins.Global, self.box)) - self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, Basins.Global, self.box)) + self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, self.basins.Global, self.box)) + self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, self.basins.Global, self.box)) jobs = AreaMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', 'atl']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, Basins.Atlantic, self.box)) - self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, Basins.Atlantic, self.box)) + self.assertEqual(jobs[0], AreaMoc(self.data_manager, '20010101', 0, 0, self.basins.Atlantic, self.box)) + self.assertEqual(jobs[1], AreaMoc(self.data_manager, '20010101', 0, 1, self.basins.Atlantic, self.box)) with self.assertRaises(Exception): AreaMoc.generate_jobs(self.diags, ['diagnostic']) @@ -40,4 +54,4 @@ class TestAreaMoc(TestCase): def test_str(self): self.assertEquals(str(self.psi), 'Area MOC Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0 ' - 'Basin: Antarctic_Ocean') + 'Basin: Atlantic') diff --git a/test/unit/test_averagesection.py b/test/unit/ocean/test_averagesection.py similarity index 60% rename from test/unit/test_averagesection.py rename to test/unit/ocean/test_averagesection.py index 7a454c4a97c2b13ccfe099169a9dfc12d2548b79..d3be4b2bba95b3f66515ee07af45eab51d42b1d5 100644 --- a/test/unit/test_averagesection.py +++ b/test/unit/ocean/test_averagesection.py @@ -3,7 +3,8 @@ from unittest import TestCase from earthdiagnostics.box import Box from earthdiagnostics.ocean.averagesection import AverageSection -from mock import Mock +from mock import Mock, patch +from earthdiagnostics.diagnostic import DiagnosticVariableOption from earthdiagnostics.modelingrealm import ModelingRealms @@ -21,28 +22,34 @@ class TestAverageSection(TestCase): self.box.max_lon = 0 self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) - self.psi = AverageSection(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', self.box) + def fake_parse(self, value): + return value + + # noinspection PyUnresolvedReferences + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): - jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'var', '0', '0', '0', '0']) + jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - self.box)) + self.box, '')) self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - self.box)) + self.box, '')) - jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'var', '0', '0', '0', '0', 'ocean']) + jobs = AverageSection.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '0', '0', '0', 'grid']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - self.box)) + self.box, 'grid')) self.assertEqual(jobs[1], AverageSection(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - self.box)) + self.box, 'grid')) with self.assertRaises(Exception): AverageSection.generate_jobs(self.diags, ['diagnostic']) with self.assertRaises(Exception): - AverageSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + AverageSection.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', '0', '0', '0', '0', 'grid', + 'extra']) def test_str(self): - self.assertEquals(str(self.psi), 'Average section Startdate: 20000101 Member: 1 Chunk: 1 Box: 0N0E ' - 'Variable: ocean:var') + diag = AverageSection(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', self.box, 'grid') + self.assertEquals(str(diag), 'Average section Startdate: 20010101 Member: 0 Chunk: 0 Box: 0N0E ' + 'Variable: ocean:var Grid: grid') diff --git a/test/unit/test_convectionsites.py b/test/unit/ocean/test_convectionsites.py similarity index 100% rename from test/unit/test_convectionsites.py rename to test/unit/ocean/test_convectionsites.py diff --git a/test/unit/test_cutsection.py b/test/unit/ocean/test_cutsection.py similarity index 83% rename from test/unit/test_cutsection.py rename to test/unit/ocean/test_cutsection.py index 170d04265c6a55a8816deb494416f04d3237abaa..a658b06d5524c96b2e2bf4f73e32958fcffcef4d 100644 --- a/test/unit/test_cutsection.py +++ b/test/unit/ocean/test_cutsection.py @@ -1,9 +1,10 @@ # coding=utf-8 from unittest import TestCase +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError from earthdiagnostics.box import Box from earthdiagnostics.ocean.cutsection import CutSection -from mock import Mock +from mock import Mock, patch from earthdiagnostics.modelingrealm import ModelingRealms @@ -23,6 +24,12 @@ class TestCutSection(TestCase): self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.psi = CutSection(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', True, 0) + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): jobs = CutSection.generate_jobs(self.diags, ['diagnostic', 'var', 'true', '10']) self.assertEqual(len(jobs), 2) @@ -38,9 +45,9 @@ class TestCutSection(TestCase): self.assertEqual(jobs[1], CutSection(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', False, 0)) - with self.assertRaises(Exception): + with self.assertRaises(DiagnosticOptionError): CutSection.generate_jobs(self.diags, ['diagnostic']) - with self.assertRaises(Exception): + with self.assertRaises(DiagnosticOptionError): CutSection.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): diff --git a/test/unit/test_gyres.py b/test/unit/ocean/test_gyres.py similarity index 88% rename from test/unit/test_gyres.py rename to test/unit/ocean/test_gyres.py index becc4e7d331754aff23baae244d82154252eaa6e..8a0255835c0373f317d9885864a2e5df14a10610 100644 --- a/test/unit/test_gyres.py +++ b/test/unit/ocean/test_gyres.py @@ -11,7 +11,7 @@ class TestGyres(TestCase): self.data_manager = Mock() self.diags = Mock() - self.diags.model_version = 'model_version' + self.diags.config.experiment.model_version = 'model_version' self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.gyres = Gyres(self.data_manager, '20000101', 1, 1, 'model_version') @@ -26,4 +26,4 @@ class TestGyres(TestCase): Gyres.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): - self.assertEquals(str(self.gyres), 'Gyres Startdate: 20000101 Member: 1 Chunk: 1') + self.assertEquals(str(self.gyres), 'Gyres Startdate: 20000101 Member: 1 Chunk: 1 Model version: model_version') diff --git a/test/unit/ocean/test_heatcontent.py b/test/unit/ocean/test_heatcontent.py new file mode 100644 index 0000000000000000000000000000000000000000..c58d1217a9338e0dd422a21a3d3d3fb6926ef124 --- /dev/null +++ b/test/unit/ocean/test_heatcontent.py @@ -0,0 +1,47 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.box import Box +from earthdiagnostics.constants import Basins +from earthdiagnostics.ocean.heatcontent import HeatContent +from mock import Mock, patch + + +# noinspection PyUnusedLocal +def _get_levels_from_meters_mock(cls, box): + return 20, 10 + + +class TestHeatContent(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + self.box = Box(True) + self.box.min_depth = 0 + self.box.max_depth = 100 + + @patch('earthdiagnostics.ocean.heatcontent.HeatContent._get_levels_from_meters') + def test_generate_jobs(self, levels_mock): + levels_mock.return_value = (1, 20) + jobs = HeatContent.generate_jobs(self.diags, ['diagnostic', 'Global', '-1', '0', '100']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], HeatContent(self.data_manager, '20010101', 0, 0, Basins().Global, -1, + self.box, 0, 0)) + self.assertEqual(jobs[1], HeatContent(self.data_manager, '20010101', 0, 1, Basins().Global, -1, + self.box, 0, 0)) + + with self.assertRaises(Exception): + HeatContent.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(Exception): + HeatContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + + def test_str(self): + diag = HeatContent(self.data_manager, '20010101', 0, 0, Basins().Global, -1, self.box, 1, 20) + self.assertEquals(str(diag), 'Heat content Startdate: 20010101 Member: 0 Chunk: 0 Mixed layer: -1 Box: 0-100m ' + 'Basin: Global') diff --git a/test/unit/test_heatcontentlayer.py b/test/unit/ocean/test_heatcontentlayer.py similarity index 73% rename from test/unit/test_heatcontentlayer.py rename to test/unit/ocean/test_heatcontentlayer.py index 60b6dd8bd46abf050ee4b83c5ae996b2da94196a..bf8135c65ed2a39b633660b410e14874f058766c 100644 --- a/test/unit/test_heatcontentlayer.py +++ b/test/unit/ocean/test_heatcontentlayer.py @@ -20,7 +20,6 @@ class TestHeatContentLayer(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.psi = HeatContentLayer(self.data_manager, '20000101', 1, 1, self.box, self.weight, 0, 10) - def test_str(self): - self.assertEquals(str(self.psi), 'Heat content layer Startdate: 20000101 Member: 1 Chunk: 1 Box: 0m-100m') + diag = HeatContentLayer(self.data_manager, '20000101', 1, 1, self.box, self.weight, 0, 10) + self.assertEquals(str(diag), 'Heat content layer Startdate: 20000101 Member: 1 Chunk: 1 Box: 0-100m') diff --git a/test/unit/ocean/test_interpolate.py b/test/unit/ocean/test_interpolate.py new file mode 100644 index 0000000000000000000000000000000000000000..4d19949ff4b0d09a33057008119c9e14c33f7493 --- /dev/null +++ b/test/unit/ocean/test_interpolate.py @@ -0,0 +1,76 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.interpolate import Interpolate +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.diagnostic import DiagnosticVariableListOption + + +class TestInterpolate(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.model_version = 'model_version' + + def fake_parse(self, value): + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid', + 'model_version', False, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var1-var2']) + self.assertEqual(len(jobs), 4) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var1', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var1', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[2], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var2', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[3], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var2', 'grid', + 'model_version', False, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', False, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', + 'model_version', False, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, '')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, '')) + + jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true', 'original_grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, 'original_grid')) + self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, 'original_grid')) + + with self.assertRaises(Exception): + Interpolate.generate_jobs(self.diags, ['interp']) + + with self.assertRaises(Exception): + Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true', 'original_grid', 'extra']) + + def test_str(self): + diag = Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', + 'model_version', True, 'original_grid') + self.assertEquals(str(diag), 'Interpolate Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var ' + 'Target grid: grid Invert lat: True Model: model_version ' + 'Original grid: original_grid') diff --git a/test/unit/ocean/test_interpolatecdo.py b/test/unit/ocean/test_interpolatecdo.py new file mode 100644 index 0000000000000000000000000000000000000000..04b085521407b34572a6edc95f238906de21b8b0 --- /dev/null +++ b/test/unit/ocean/test_interpolatecdo.py @@ -0,0 +1,89 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.interpolatecdo import InterpolateCDO +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError + + +class TestInterpolate(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + self.diags.config.experiment.model_version = 'model_version' + self.diags.config.experiment.atmos_grid = 'atmos_grid' + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value.split('-') + + @patch('earthdiagnostics.ocean.interpolatecdo.InterpolateCDO._compute_weights') + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + def test_generate_jobs(self, mock_weights): + mock_weights.return_value = None + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'atmos_grid', 'model_version', True, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'atmos_grid', 'model_version', True, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', True, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic', + 'false']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, '', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, '', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic', + 'false', 'orig']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + + jobs = InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'target_grid', 'bicubic', + 'false', 'orig', 'false']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + self.assertEqual(jobs[1], InterpolateCDO(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 'target_grid', 'model_version', False, 'orig', None)) + + with self.assertRaises(DiagnosticOptionError): + InterpolateCDO.generate_jobs(self.diags, ['interp']) + + with self.assertRaises(DiagnosticOptionError): + InterpolateCDO.generate_jobs(self.diags, ['interpcdo', 'ocean', 'var', 'bicubic', 'false', 'orig', 'false', + 'extra']) + + def test_str(self): + diag = InterpolateCDO(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 'atmos_grid', 'model_version', False, 'orig', None) + self.assertEquals(str(diag), 'Interpolate with CDO Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var ' + 'Target grid: atmos_grid Original grid: orig Mask ocean: False ' + 'Model: model_version') diff --git a/test/unit/ocean/test_maskland.py b/test/unit/ocean/test_maskland.py new file mode 100644 index 0000000000000000000000000000000000000000..ede009114fda18e2582a6ce7ba7372520d436192 --- /dev/null +++ b/test/unit/ocean/test_maskland.py @@ -0,0 +1,65 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.diagnostic import DiagnosticVariableListOption, DiagnosticOptionError +from earthdiagnostics.box import Box +from earthdiagnostics.ocean.mask_land import MaskLand +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestMaskLand(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + + self.box = Box() + self.box.min_lat = 0 + self.box.max_lat = 0 + self.box.min_lon = 0 + self.box.max_lon = 0 + + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value.split('-') + + @patch.object(DiagnosticVariableListOption, 'parse', fake_parse) + @patch('earthdiagnostics.ocean.mask_land.MaskLand._get_mask') + def test_generate_jobs(self, get_mask_mock): + get_mask_mock.return_value = None + jobs = MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 't', '')) + self.assertEqual(jobs[1], MaskLand(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 't', '')) + + for mask in ('t', 'u', 'v', 'f', 'w'): + jobs = MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', mask]) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + mask, '')) + self.assertEqual(jobs[1], MaskLand(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + mask, '')) + + with self.assertRaises(DiagnosticOptionError): + MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'BAD']) + + jobs = MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 't', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + 't', 'grid')) + self.assertEqual(jobs[1], MaskLand(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + 't', 'grid')) + + with self.assertRaises(DiagnosticOptionError): + MaskLand.generate_jobs(self.diags, ['diagnostic']) + with self.assertRaises(DiagnosticOptionError): + MaskLand.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 't', 'grid', 'extra']) + + def test_str(self): + diag = MaskLand(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 't', 'grid') + self.assertEquals(str(diag), 'Land mask Startdate: 20010101 Member: 0 Chunk: 0 Variable: ocean:var Grid: grid') diff --git a/test/unit/test_maxmoc.py b/test/unit/ocean/test_maxmoc.py similarity index 70% rename from test/unit/test_maxmoc.py rename to test/unit/ocean/test_maxmoc.py index f9542b2996ae9c8cf4d5670b1dcbc128827184a1..f6f12b3468a050c5ad118ff86664b664ceae29c9 100644 --- a/test/unit/test_maxmoc.py +++ b/test/unit/ocean/test_maxmoc.py @@ -2,15 +2,18 @@ from unittest import TestCase from earthdiagnostics.box import Box -from earthdiagnostics.constants import Basins +from earthdiagnostics.constants import Basins, Basin from earthdiagnostics.ocean.maxmoc import MaxMoc -from mock import Mock +from mock import Mock, patch class TestMaxMoc(TestCase): def setUp(self): self.data_manager = Mock() + self.basins = Mock() + self.basins.Global = Basin('Global') + self.basins.Atlantic = Basin('Atlantic') self.box = Box(True) self.box.min_lat = 0.0 @@ -18,8 +21,19 @@ class TestMaxMoc(TestCase): self.box.min_depth = 0.0 self.box.max_depth = 0.0 - self.maxmoc = MaxMoc(self.data_manager, '20000101', 1, 2000, Basins.Global, self.box) + self.maxmoc = MaxMoc(self.data_manager, '20000101', 1, 2000, self.basins.Global, self.box) + def fake_parse(self, value): + if type(value) is Basin: + return value + if value == 'atl': + value = 'Atlantic' + else: + value = 'Global' + + return Basin(value) + + @patch.object(Basins, 'parse', fake_parse) def test_generate_jobs(self): self.diags = Mock() self.diags.model_version = 'model_version' @@ -29,13 +43,13 @@ class TestMaxMoc(TestCase): jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, Basins.Global, self.box)) - self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, Basins.Global, self.box)) + self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, self.basins.Global, self.box)) + self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, self.basins.Global, self.box)) jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', 'atl']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, Basins.Atlantic, self.box)) - self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, Basins.Atlantic, self.box)) + self.assertEqual(jobs[0], MaxMoc(self.data_manager, '20010101', 0, 2000, self.basins.Atlantic, self.box)) + self.assertEqual(jobs[1], MaxMoc(self.data_manager, '20010101', 0, 2001, self.basins.Atlantic, self.box)) self.diags.config.experiment.get_full_years.return_value = list() jobs = MaxMoc.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0']) @@ -49,4 +63,4 @@ class TestMaxMoc(TestCase): def test_str(self): self.assertEquals(str(self.maxmoc), 'Max moc Startdate: 20000101 Member: 1 Year: 2000 ' - 'Box: 0N0 Basin: Global_Ocean') + 'Box: 0.0N0.0m Basin: Global') diff --git a/test/unit/test_mixedlayerheatcontent.py b/test/unit/ocean/test_mixedlayerheatcontent.py similarity index 100% rename from test/unit/test_mixedlayerheatcontent.py rename to test/unit/ocean/test_mixedlayerheatcontent.py diff --git a/test/unit/test_mixedlayersaltcontent.py b/test/unit/ocean/test_mixedlayersaltcontent.py similarity index 100% rename from test/unit/test_mixedlayersaltcontent.py rename to test/unit/ocean/test_mixedlayersaltcontent.py diff --git a/test/unit/test_moc.py b/test/unit/ocean/test_moc.py similarity index 100% rename from test/unit/test_moc.py rename to test/unit/ocean/test_moc.py diff --git a/test/unit/ocean/test_mxl.py b/test/unit/ocean/test_mxl.py new file mode 100644 index 0000000000000000000000000000000000000000..ead3fbbccfb2bfc40b6075089b8dc166326dcba8 --- /dev/null +++ b/test/unit/ocean/test_mxl.py @@ -0,0 +1,28 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.ocean.mxl import Mxl +from mock import Mock + + +class TestMxl(TestCase): + + def setUp(self): + self.data_manager = Mock() + + self.diags = Mock() + self.diags.model_version = 'model_version' + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def test_generate_jobs(self): + jobs = Mxl.generate_jobs(self.diags, ['diagnostic']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Mxl(self.data_manager, '20010101', 0, 0)) + self.assertEqual(jobs[1], Mxl(self.data_manager, '20010101', 0, 1)) + + with self.assertRaises(Exception): + Mxl.generate_jobs(self.diags, ['diagnostic', 'extra']) + + def test_str(self): + diag = Mxl(self.data_manager, '20010101', 0, 0) + self.assertEquals(str(diag), 'Mixed layer Startdate: 20010101 Member: 0 Chunk: 0') diff --git a/test/unit/test_psi.py b/test/unit/ocean/test_psi.py similarity index 100% rename from test/unit/test_psi.py rename to test/unit/ocean/test_psi.py diff --git a/test/unit/ocean/test_region_mean.py b/test/unit/ocean/test_region_mean.py new file mode 100644 index 0000000000000000000000000000000000000000..e527feda328b3103615680a4387d155ef29d3f63 --- /dev/null +++ b/test/unit/ocean/test_region_mean.py @@ -0,0 +1,100 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.ocean.regionmean import RegionMean +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption +from mock import Mock, patch + + +class TestRegionMean(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + box = Box() + box.min_depth = 0 + box.max_depth = 0 + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'T', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'T', + box, True, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, True, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, False, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, False, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', + 'True']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, '')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, '')) + + jobs = RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', + 'True', 'grid']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, 'grid')) + self.assertEqual(jobs[1], RegionMean(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'U', + box, False, Basins().Global, True, 'grid')) + + with self.assertRaises(DiagnosticOptionError): + RegionMean.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + RegionMean.generate_jobs(self.diags, ['diagnostic', 'ocean', 'var', 'U', 'global', '1', '10', 'false', + 'True', 'grid', 'extra']) + + def test_str(self): + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + diag = RegionMean(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'U', box, False, + Basins().Global, True, 'grid') + self.assertEquals(str(diag), 'Region mean Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Grid point: U ' + 'Box: 1-10 Save 3D: False Save variance: True Original grid: grid') diff --git a/test/unit/test_siasiesiv.py b/test/unit/ocean/test_siasiesiv.py similarity index 89% rename from test/unit/test_siasiesiv.py rename to test/unit/ocean/test_siasiesiv.py index 2ed284261464ec4341067576c9e2a89019d2fd49..00f7c68ca53e22c4ca2f10d90b1d4d89e4176236 100644 --- a/test/unit/test_siasiesiv.py +++ b/test/unit/ocean/test_siasiesiv.py @@ -15,7 +15,7 @@ class TestSiasiesiv(TestCase): self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) self.mask = Mock() - self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins.Global, self.mask) + self.psi = Siasiesiv(self.data_manager, '20000101', 1, 1, Basins().Global, self.mask) def test_str(self): - self.assertEquals(str(self.psi), 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global_Ocean') + self.assertEquals(str(self.psi), 'Siasiesiv Startdate: 20000101 Member: 1 Chunk: 1 Basin: Global') diff --git a/test/unit/ocean/test_vertical_gradient.py b/test/unit/ocean/test_vertical_gradient.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d9d22d32358059c73899c1c7aee834400dddab --- /dev/null +++ b/test/unit/ocean/test_vertical_gradient.py @@ -0,0 +1,65 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.ocean.verticalgradient import VerticalGradient +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins +from earthdiagnostics.box import Box +from earthdiagnostics.diagnostic import DiagnosticOptionError, DiagnosticVariableOption +from mock import Mock, patch + + +class TestVerticalGradient(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) + + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + + box = Box() + box.min_depth = 1 + box.max_depth = 2 + + jobs = VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box)) + self.assertEqual(jobs[1], VerticalGradient(self.data_manager, '20010101', 0, 1, 'var', box)) + + box = Box() + box.min_depth = 2 + box.max_depth = 2 + + jobs = VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var', '2']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box)) + self.assertEqual(jobs[1], VerticalGradient(self.data_manager, '20010101', 0, 1, 'var', box)) + + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + jobs = VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var', '1', 10]) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box)) + self.assertEqual(jobs[1], VerticalGradient(self.data_manager, '20010101', 0, 1, 'var', box)) + + with self.assertRaises(DiagnosticOptionError): + VerticalGradient.generate_jobs(self.diags, ['diagnostic']) + + with self.assertRaises(DiagnosticOptionError): + VerticalGradient.generate_jobs(self.diags, ['diagnostic', 'var', '1', '10', 'extra']) + + def test_str(self): + box = Box() + box.min_depth = 1 + box.max_depth = 10 + + diag = VerticalGradient(self.data_manager, '20010101', 0, 0, 'var', box) + self.assertEquals(str(diag), 'Vertical gradient Startdate: 20010101 Member: 0 Chunk: 0 Variable: var Box: 1-10') diff --git a/test/unit/test_verticalmean.py b/test/unit/ocean/test_verticalmean.py similarity index 81% rename from test/unit/test_verticalmean.py rename to test/unit/ocean/test_verticalmean.py index 59d0fb501a35cabcba02d01cdc189d3dc2a302be..fc501afda0a4b1c3842c8fd60b16d4fb88b03b9f 100644 --- a/test/unit/test_verticalmean.py +++ b/test/unit/ocean/test_verticalmean.py @@ -1,9 +1,10 @@ # coding=utf-8 from unittest import TestCase +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError from earthdiagnostics.box import Box from earthdiagnostics.ocean.verticalmean import VerticalMean -from mock import Mock +from mock import Mock, patch class TestVerticalMean(TestCase): @@ -21,6 +22,12 @@ class TestVerticalMean(TestCase): self.mixed = VerticalMean(self.data_manager, '20000101', 1, 1, 'var', self.box) + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): jobs = VerticalMean.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) self.assertEqual(len(jobs), 2) @@ -39,11 +46,11 @@ class TestVerticalMean(TestCase): self.assertEqual(jobs[0], VerticalMean(self.data_manager, '20010101', 0, 0, 'var', Box())) self.assertEqual(jobs[1], VerticalMean(self.data_manager, '20010101', 0, 1, 'var', Box())) - with self.assertRaises(Exception): + with self.assertRaises(DiagnosticOptionError): VerticalMean.generate_jobs(self.diags, ['diagnostic']) - with self.assertRaises(Exception): - VerticalMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) + with self.assertRaises(DiagnosticOptionError): + VerticalMean.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Vertical mean Startdate: 20000101 Member: 1 Chunk: 1 Variable: var ' diff --git a/test/unit/test_verticalmeanmeters.py b/test/unit/ocean/test_verticalmeanmeters.py similarity index 64% rename from test/unit/test_verticalmeanmeters.py rename to test/unit/ocean/test_verticalmeanmeters.py index 39a86c3a66c21fc7de7d2d1232402e2ff8c9ca55..f696d20d4adc81e59cb62d700cd2fcba7a0f79d0 100644 --- a/test/unit/test_verticalmeanmeters.py +++ b/test/unit/ocean/test_verticalmeanmeters.py @@ -1,10 +1,11 @@ # coding=utf-8 from unittest import TestCase +from earthdiagnostics.diagnostic import DiagnosticVariableOption, DiagnosticOptionError from earthdiagnostics.box import Box from earthdiagnostics.ocean.verticalmeanmeters import VerticalMeanMeters from earthdiagnostics.modelingrealm import ModelingRealms -from mock import Mock +from mock import Mock, patch class TestVerticalMeanMeters(TestCase): @@ -20,38 +21,44 @@ class TestVerticalMeanMeters(TestCase): self.box.min_depth = 0 self.box.max_depth = 100 - self.mixed = VerticalMeanMeters(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', self.box) + self.mixed = VerticalMeanMeters(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', self.box, 'T') + def fake_parse(self, value): + if not value: + raise DiagnosticOptionError + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) def test_generate_jobs(self): jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var', '0', '100']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', - self.box)) + self.box, 'T')) self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', - self.box)) + self.box, 'T')) jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var', '0']) box = Box(True) box.min_depth = 0 self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, 'var', ModelingRealms.ocean, - box)) - self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, 'var', ModelingRealms.ocean, - box)) + self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + box, 'T')) + self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + box, 'T')) jobs = VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', 'var']) self.assertEqual(len(jobs), 2) - self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, 'var', ModelingRealms.ocean, - Box(True))) - self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, 'var', ModelingRealms.ocean, - Box(True))) + self.assertEqual(jobs[0], VerticalMeanMeters(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', + Box(True), 'T')) + self.assertEqual(jobs[1], VerticalMeanMeters(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', + Box(True), 'T')) - with self.assertRaises(Exception): + with self.assertRaises(DiagnosticOptionError): VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic']) - with self.assertRaises(Exception): + with self.assertRaises(DiagnosticOptionError): VerticalMeanMeters.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) def test_str(self): self.assertEquals(str(self.mixed), 'Vertical mean meters Startdate: 20000101 Member: 1 Chunk: 1 ' - 'Variable: ocean:var Box: 0m-100m') + 'Variable: ocean:var Box: 0-100m') diff --git a/test/unit/statistics/__init__.py b/test/unit/statistics/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad5790a5799b96f2e164d825c0b1f8ec0c2dfb --- /dev/null +++ b/test/unit/statistics/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/test/unit/statistics/test_climatologicalpercentile.py b/test/unit/statistics/test_climatologicalpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..12752caab2bd8fcc7322b7e80130a939f4652029 --- /dev/null +++ b/test/unit/statistics/test_climatologicalpercentile.py @@ -0,0 +1,42 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestClimatologicalPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.data_manager.variable_list.get_variable.return_value = None + + self.diags = Mock() + self.diags.data_manager = self.data_manager + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', 'ocean', 'var', '2000', '2001', '11']) + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0], ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', + 2000, 2001, 11, + self.diags.config.experiment)) + + with self.assertRaises(Exception): + ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent']) + with self.assertRaises(Exception): + ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', 'ocean', 'var', '2000', '2001', '11', + 'extra']) + + def test_str(self): + diagnostic = ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', + 2000, 2001, 11, self.diags.config.experiment) + + self.assertEquals(str(diagnostic), 'Climatological percentile Variable: ocean:var Period: 2000-2001 ' + 'Forecast month: 11') diff --git a/test/unit/statistics/test_daysoverpercentile.py b/test/unit/statistics/test_daysoverpercentile.py new file mode 100644 index 0000000000000000000000000000000000000000..cc225cd64a589cb1f453e2582777841cc6b2c2ff --- /dev/null +++ b/test/unit/statistics/test_daysoverpercentile.py @@ -0,0 +1,33 @@ +# coding=utf-8 +from unittest import TestCase + +from mock import Mock + +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.statistics.daysoverpercentile import DaysOverPercentile + + +class TestDaysOverPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.diags = Mock() + self.diags.config.experiment.get_chunk_list.return_value = (('20011101', 0, 0), ('20011101', 0, 1)) + + def test_generate_jobs(self): + jobs = DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001', '11']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, + 2000, 11)) + self.assertEqual(jobs[1], DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, + 2001, 11)) + + with self.assertRaises(Exception): + DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001']) + with self.assertRaises(Exception): + DaysOverPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '2000', '2001', '11', 'extra']) + + def test_str(self): + diagnostic = DaysOverPercentile(self.data_manager, ModelingRealms.ocean, 'var', 2000, 2001, '20001101', 11) + self.assertEquals(str(diagnostic), 'Days over percentile Startdate: 20001101 Variable: ocean:var ' + 'Climatology: 2000-2001') diff --git a/test/unit/statistics/test_discretize.py b/test/unit/statistics/test_discretize.py new file mode 100644 index 0000000000000000000000000000000000000000..402a3772aff95aa7a6cf001cb049d2034be8847a --- /dev/null +++ b/test/unit/statistics/test_discretize.py @@ -0,0 +1,60 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.statistics.discretize import Discretize +from earthdiagnostics.diagnostic import DiagnosticVariableOption +from mock import Mock, patch + +from earthdiagnostics.modelingrealm import ModelingRealms + + +class TestClimatologicalPercentile(TestCase): + + def setUp(self): + self.data_manager = Mock() + self.data_manager.variable_list.get_variable.return_value = None + + self.diags = Mock() + self.diags.data_manager = self.data_manager + self.diags.config.experiment.startdates = ('20000101', '20010101') + + def fake_parse(self, value): + return value + + @patch.object(DiagnosticVariableOption, 'parse', fake_parse) + def test_generate_jobs(self): + jobs = Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var']) + self.assertEqual(len(jobs), 2) + self.assertEqual(jobs[0], Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', + 2000, float('nan'), float('nan'))) + self.assertEqual(jobs[1], Discretize(self.data_manager, '20010101', ModelingRealms.ocean, 'var', + 2000, float('nan'), float('nan'))) + + jobs = Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var', '500']) + self.assertEqual(jobs[0], Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', + 500, float('nan'), float('nan'))) + self.assertEqual(jobs[1], Discretize(self.data_manager, '20010101', ModelingRealms.ocean, 'var', + 500, float('nan'), float('nan'))) + + jobs = Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var', '500', '0', '40']) + self.assertEqual(jobs[0], Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', + 500, 0, 40)) + self.assertEqual(jobs[1], Discretize(self.data_manager, '20010101', ModelingRealms.ocean, 'var', + 500, 0, 40)) + + with self.assertRaises(Exception): + Discretize.generate_jobs(self.diags, ['discretize']) + with self.assertRaises(Exception): + Discretize.generate_jobs(self.diags, ['discretize', 'ocean', 'var', '500', '0', '40', 'extra']) + + def test_str(self): + diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, 10, 40) + + self.assertEquals(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' + 'Range: [10, 40]') + + diagnostic = Discretize(self.data_manager, '20000101', ModelingRealms.ocean, 'var', 2000, + float('nan'), float('nan')) + + self.assertEquals(str(diagnostic), 'Discretizing variable: ocean:var Startdate: 20000101 Bins: 2000 ' + 'Range: [None, None]') diff --git a/test/unit/test_monthlypercentile.py b/test/unit/statistics/test_monthlypercentile.py similarity index 97% rename from test/unit/test_monthlypercentile.py rename to test/unit/statistics/test_monthlypercentile.py index 4b9bbfb09ed3b03bb0aa2b4cd68a373972c844a7..a902ec89098794bdca2baabce94be5d1aa3d4107 100644 --- a/test/unit/test_monthlypercentile.py +++ b/test/unit/statistics/test_monthlypercentile.py @@ -24,7 +24,7 @@ class TestMonthlyPercentile(TestCase): self.diagnostic = MonthlyPercentile(self.data_manager, '20000101', 1, 1, ModelingRealms.ocean, 'var', [10, 90]) def test_generate_jobs(self): - jobs = MonthlyPercentile.generate_jobs(self.diags, ['monpercent', 'var', 'ocean', '10-90']) + jobs = MonthlyPercentile.generate_jobs(self.diags, ['monpercent', 'ocean', 'var', '10-90']) self.assertEqual(len(jobs), 2) self.assertEqual(jobs[0], MonthlyPercentile(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', [10, 90])) diff --git a/test/unit/test_cdftools.py b/test/unit/test_cdftools.py index 9ebdc65f9289ea0a217f046800e258cde95904e6..28de4ac34e9b60726ddbcac9e1d7d3e7f38a8d1f 100644 --- a/test/unit/test_cdftools.py +++ b/test/unit/test_cdftools.py @@ -7,38 +7,59 @@ from earthdiagnostics.cdftools import CDFTools import mock +# noinspection PyUnusedLocal +def bad_file(path, access=None): + return not os.path.basename(path).startswith('bad') + + class TestCDFTools(TestCase): - def setUp(self): + + # noinspection PyUnusedLocal + @mock.patch('os.path.isfile', side_effect=bad_file) + @mock.patch('os.access', side_effect=bad_file) + @mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') + def test_run(self, mock_path, mock_exists, execute_mock): + self.cdftools = CDFTools('/test/path') + execute_mock.return_value = ['Command output'] + with self.assertRaises(ValueError): + self.cdftools.run('badcommand', input='input_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='badinput_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='input_file', output='input_file') + with self.assertRaises(Exception): + self.cdftools.run('command', input='input_file', output='badoutput_file') + + self.cdftools.run('command', input='input_file', output='output_file') + self.cdftools.run('command', input='input_file') + self.cdftools.run('command', input=None) + self.cdftools.run('command', input=['input_file', 'input_file2']) + self.cdftools.run('command', input='input_file', options='-o -p') + self.cdftools.run('command', input='input_file', options=('-o', '-p')) + + # noinspection PyUnusedLocal + @mock.patch('os.path.isfile', side_effect=bad_file) + @mock.patch('os.access', side_effect=bad_file) + @mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') + def test_run(self, mock_path, mock_exists, execute_mock): self.cdftools = CDFTools('') - mock.patch('os.path.join') - - # noinspection PyTypeChecker - def test_run(self): - # noinspection PyUnusedLocal - def mock_exists(path, access=None): - return not os.path.basename(path.startswith('bad')) - - with mock.patch('os.path.exists') as exists_mock: - with mock.patch('os.access') as access_mock: - exists_mock.side_effect = mock_exists - access_mock.side_effect = mock_exists - - with mock.patch('earthdiagnostics.utils.Utils.execute_shell_command') as execute_mock: - execute_mock.return_value = ['Command output'] - with self.assertRaises(ValueError): - self.cdftools.run('badcommand', input='input_file', output='output_file') - with self.assertRaises(ValueError): - self.cdftools.run('command', input='badinput_file', output='output_file') - with self.assertRaises(ValueError): - self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') - with self.assertRaises(ValueError): - self.cdftools.run('command', input='input_file', output='input_file') - with self.assertRaises(Exception): - self.cdftools.run('command', input='input_file', output='badoutput_file') - - self.cdftools.run('command', input='input_file', output='output_file') - self.cdftools.run('command', input='input_file') - self.cdftools.run('command', input=None) - self.cdftools.run('command', input=['input_file', 'input_file2']) - self.cdftools.run('command', input='input_file', options='-o -p') - self.cdftools.run('command', input='input_file', options=('-o', '-p')) + execute_mock.return_value = ['Command output'] + with self.assertRaises(ValueError): + self.cdftools.run('badcommand', input='input_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='badinput_file', output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input=['input_file', 'badinput_file'], output='output_file') + with self.assertRaises(ValueError): + self.cdftools.run('command', input='input_file', output='input_file') + with self.assertRaises(Exception): + self.cdftools.run('command', input='input_file', output='badoutput_file') + + self.cdftools.run('command', input='input_file', output='output_file') + self.cdftools.run('command', input='input_file') + self.cdftools.run('command', input=None) + self.cdftools.run('command', input=['input_file', 'input_file2']) + self.cdftools.run('command', input='input_file', options='-o -p') + self.cdftools.run('command', input='input_file', options=('-o', '-p')) diff --git a/test/unit/test_climatologicalpercentile.py b/test/unit/test_climatologicalpercentile.py deleted file mode 100644 index 95afc38bb081bf3cdee76a6f64521c688817158d..0000000000000000000000000000000000000000 --- a/test/unit/test_climatologicalpercentile.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding=utf-8 -from unittest import TestCase - -from earthdiagnostics.statistics.climatologicalpercentile import ClimatologicalPercentile -from mock import Mock - -from earthdiagnostics.modelingrealm import ModelingRealms - - -class TestClimatologicalPercentile(TestCase): - - def setUp(self): - self.data_manager = Mock() - self.data_manager.variable_list.get_variable.return_value = None - - self.diags = Mock() - self.diags.data_manager = self.data_manager - - self.diagnostic = ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', - [10, 90], 1000, self.diags.config.experiment) - - def test_generate_jobs(self): - jobs = ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', 'ocean', 'var', '1-2', '1000']) - self.assertEqual(len(jobs), 1) - self.assertEqual(jobs[0], ClimatologicalPercentile(self.data_manager, ModelingRealms.ocean, 'var', [1, 2], - 1000, self.diags.config.experiment)) - - with self.assertRaises(Exception): - ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent']) - with self.assertRaises(Exception): - ClimatologicalPercentile.generate_jobs(self.diags, ['climpercent', '0', '0', '0', '0', '0', '0', '0']) - - def test_str(self): - self.assertEquals(str(self.diagnostic), 'Climatological percentile Variable: ocean:var Leadtimes: [10, 90] ' - 'Bins: 1000') diff --git a/test/unit/test_config.py b/test/unit/test_config.py new file mode 100644 index 0000000000000000000000000000000000000000..419be475362582c7df34169f4a640748ad298e9e --- /dev/null +++ b/test/unit/test_config.py @@ -0,0 +1,319 @@ +# coding=utf-8 +from unittest import TestCase + +from earthdiagnostics.config import CMORConfig, ConfigException, THREDDSConfig, ReportConfig, ExperimentConfig +from earthdiagnostics.frequency import Frequencies +from earthdiagnostics.modelingrealm import ModelingRealms + + +class VariableMock(object): + def __init__(self): + self.domain = ModelingRealms.ocean + self.short_name = 'tos' + + def __eq__(self, other): + return self.domain == other.domain and self.short_name == other.short_name + + +class VariableManagerMock(object): + def get_variable(self, alias, silent=False): + if alias == 'bad': + return None + var = VariableMock() + var.short_name = alias + return var + + +class ParserMock(object): + + def __init__(self): + self._values = {} + + def add_value(self, section, var, value): + self._values[self.get_var_string(section, var)] = value + + def get_var_string(self, section, var): + return '{0}:{1}'.format(section, var) + + def get_value(self, section, var, default): + try: + return self._values[self.get_var_string(section, var)] + except KeyError: + return default + + def get_bool_option(self, section, var, default): + return self.get_value(section, var, default) + + def get_path_option(self, section, var, default): + return self.get_value(section, var, default) + + def get_int_option(self, section, var, default=0): + return self.get_value(section, var, default) + + + def get_int_list_option(self, section, var, default=list(), separator=' '): + try: + return [int(val) for val in self._values[self.get_var_string(section, var)].split(separator)] + except KeyError: + return default + + def get_list_option(self, section, var, default=list(), separator=' '): + try: + return [val for val in self._values[self.get_var_string(section, var)].split(separator)] + except KeyError: + return default + + def get_option(self, section, var, default=None): + return self.get_value(section, var, default) + + +class TestCMORConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + self.var_manager = VariableManagerMock() + + def test_basic_config(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.ocean, True) + self.assertEquals(config.atmosphere, True) + self.assertEquals(config.force, False) + self.assertEquals(config.force_untar, False) + self.assertEquals(config.use_grib, True) + self.assertEquals(config.activity, 'CMIP') + self.assertEquals(config.associated_experiment, 'to be filled') + self.assertEquals(config.associated_model, 'to be filled') + self.assertEquals(config.initialization_description, 'to be filled') + self.assertEquals(config.initialization_method, '1') + self.assertEquals(config.initialization_number, 1) + self.assertEquals(config.source, 'to be filled') + self.assertEquals(config.version, '') + self.assertEquals(config.physics_version, '1') + self.assertEquals(config.physics_description, 'to be filled') + self.assertEquals(config.filter_files, '') + self.assertEquals(config.default_atmos_grid, 'gr') + self.assertEquals(config.default_ocean_grid, 'gn') + self.assertEquals(config.min_cmorized_vars, 10) + + def test_cmorize(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + self.assertTrue(config.cmorize(None)) + + def test_cmorize_list(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:thetao ocean:tos') + + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + + thetao_mock = VariableMock() + thetao_mock.domain = ModelingRealms.ocean + thetao_mock.short_name = 'thetao' + self.assertTrue(config.cmorize(thetao_mock)) + + def test_bad_list(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', '#ocean:tos') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'atmos:tos') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:bad') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + def test_not_cmorize(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos') + + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + + self.assertFalse(config.cmorize(None)) + + tas_mock = VariableMock() + tas_mock.domain = ModelingRealms.atmos + tas_mock.short_name = 'tas' + self.assertFalse(config.cmorize(tas_mock)) + + thetao_mock = VariableMock() + thetao_mock.domain = ModelingRealms.ocean + thetao_mock.short_name = 'thetao' + self.assertFalse(config.cmorize(thetao_mock)) + + def test_comment(self): + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos #ocean:thetao ') + + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.cmorize(VariableMock())) + + thetao_mock = VariableMock() + thetao_mock.domain = ModelingRealms.ocean + thetao_mock.short_name = 'thetao' + self.assertFalse(config.cmorize(thetao_mock)) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', '#ocean:tos ocean:thetao ') + with self.assertRaises(ConfigException): + CMORConfig(self.mock_parser, self.var_manager) + + def test_cmorization_chunk(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.chunk_cmorization_requested(1)) + + def test_cmorize_only_some_chunks(self): + self.mock_parser.add_value('CMOR', 'CHUNKS', '3 5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.chunk_cmorization_requested(3)) + self.assertTrue(config.chunk_cmorization_requested(5)) + self.assertFalse(config.chunk_cmorization_requested(1)) + self.assertFalse(config.chunk_cmorization_requested(4)) + self.assertFalse(config.chunk_cmorization_requested(6)) + + def test_any_required(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.any_required(['tos'])) + + self.mock_parser.add_value('CMOR', 'VARIABLE_LIST', 'ocean:tos ocean:thetao') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertTrue(config.any_required(['tos', 'thetao', 'tas'])) + self.assertTrue(config.any_required(['tos', 'tas'])) + self.assertTrue(config.any_required(['thetao'])) + + self.assertFalse(config.any_required(['tas'])) + + def test_hourly_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.six_hourly), {}) + + self.mock_parser.add_value('CMOR', 'ATMOS_HOURLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.six_hourly), {128: None, + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) + + self.assertEquals(config.get_levels(Frequencies.six_hourly, 128), None) + self.assertEquals(config.get_levels(Frequencies.six_hourly, 129), '1') + self.assertEquals(config.get_levels(Frequencies.six_hourly, 130), '1,2') + self.assertEquals(config.get_levels(Frequencies.six_hourly, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEquals(config.get_levels(Frequencies.six_hourly, 132), '0,5') + + def test_daily_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.daily), {}) + + self.mock_parser.add_value('CMOR', 'ATMOS_DAILY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.daily), {128: None, + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) + + self.assertEquals(config.get_levels(Frequencies.daily, 128), None) + self.assertEquals(config.get_levels(Frequencies.daily, 129), '1') + self.assertEquals(config.get_levels(Frequencies.daily, 130), '1,2') + self.assertEquals(config.get_levels(Frequencies.daily, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEquals(config.get_levels(Frequencies.daily, 132), '0,5') + + def test_monthly_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.monthly), {}) + + self.mock_parser.add_value('CMOR', 'ATMOS_MONTHLY_VARS', '128,129:1,130:1-2,131:1:10,132:0:10:5') + config = CMORConfig(self.mock_parser, self.var_manager) + self.assertEquals(config.get_variables(Frequencies.monthly), {128: None, + 129: '1', + 130: '1,2', + 131: '1,2,3,4,5,6,7,8,9', + 132: '0,5'}) + + self.assertEquals(config.get_levels(Frequencies.monthly, 128), None) + self.assertEquals(config.get_levels(Frequencies.monthly, 129), '1') + self.assertEquals(config.get_levels(Frequencies.monthly, 130), '1,2') + self.assertEquals(config.get_levels(Frequencies.monthly, 131), '1,2,3,4,5,6,7,8,9',) + self.assertEquals(config.get_levels(Frequencies.monthly, 132), '0,5') + + def test_bad_frequency_vars(self): + config = CMORConfig(self.mock_parser, self.var_manager) + with self.assertRaises(ValueError): + self.assertEquals(config.get_variables(Frequencies.climatology), {}) + + +class TestTHREDDSConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + + def test_basic_config(self): + config = THREDDSConfig(self.mock_parser) + self.assertEquals(config.server_url, '') + + def test_url(self): + self.mock_parser.add_value('THREDDS', 'SERVER_URL', 'test_url') + config = THREDDSConfig(self.mock_parser) + self.assertEquals(config.server_url, 'test_url') + + +class TestReportConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + + def test_basic_config(self): + config = ReportConfig(self.mock_parser) + self.assertEquals(config.path, '') + self.assertEquals(config.maximum_priority, 10) + + def test_path(self): + self.mock_parser.add_value('REPORT', 'PATH', 'new_path') + config = ReportConfig(self.mock_parser) + self.assertEquals(config.path, 'new_path') + + def test_priority(self): + config = ReportConfig(self.mock_parser) + self.assertEquals(config.maximum_priority, 3) + + +class TestExperimentConfig(TestCase): + + def setUp(self): + self.mock_parser = ParserMock() + + def test_basic_config(self): + config = ExperimentConfig(self.mock_parser) + + self.assertEquals(config.startdates, []) + self.assertEquals(config.members, []) + self.assertEquals(config.chunk_size, 0) + self.assertEquals(config.num_chunks, 0) + + self.assertEquals(config.atmos_grid, '') + self.assertEquals(config.atmos_timestep, 6) + self.assertEquals(config.ocean_timestep, 6) + + def test_cmor_version_required(self): + self.mock_parser.add_value('CMOR', 'VERSION', '20001101') + self.mock_parser.add_value('EXPERIMENT', 'DATA_CONVENTION', 'Primavera') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.path, 'new_path') + + def test_startdates(self): + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '20001101 20011101') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200(0|1)1101') + config = ExperimentConfig(self.mock_parser) + self.assertEquals(config.startdates, ['20001101', '20011101']) + + self.mock_parser.add_value('EXPERIMENT', 'STARTDATES', '200[0-2](02|05|08|11)01') + config = ExperimentConfig(self.mock_parser) + print(config.startdates) + self.assertEquals(config.startdates, [u'20000201', u'20000501', u'20000801', u'20001101', u'20010201', + u'20010501', u'20010801', u'20011101', u'20020201', u'20020501', + u'20020801', u'20021101']) + + diff --git a/test/unit/test_diagnostic.py b/test/unit/test_diagnostic.py index 31ba6fa9c1562d05ed669a618f76a8f0b8925583..ae9921fa920affc0833b87dffbdf5d5899c6b806 100644 --- a/test/unit/test_diagnostic.py +++ b/test/unit/test_diagnostic.py @@ -3,56 +3,7 @@ from earthdiagnostics.diagnostic import * from unittest import TestCase from earthdiagnostics.modelingrealm import ModelingRealms - - -class TestDiagnostic(TestCase): - - # noinspection PyMissingOrEmptyDocstring - class MockDiag(Diagnostic): - def compute(self): - pass - - @classmethod - def generate_jobs(cls, diags, options): - pass - - alias = 'mockdiag' - - def setUp(self): - self.diagnostic = Diagnostic(None) - Diagnostic.register(TestDiagnostic.MockDiag) - - def test_register(self): - with self.assertRaises(ValueError): - # noinspection PyTypeChecker - Diagnostic.register(str) - with self.assertRaises(ValueError): - Diagnostic.register(Diagnostic) - Diagnostic.register(TestDiagnostic.MockDiag) - - def test_get_diagnostic(self): - self.assertIsNone(Diagnostic.get_diagnostic('none')) - self.assertIs(TestDiagnostic.MockDiag, Diagnostic.get_diagnostic('mockdiag')) - - def test_generate_jobs(self): - with self.assertRaises(NotImplementedError): - Diagnostic.generate_jobs(None, ['']) - - def test_compute(self): - with self.assertRaises(NotImplementedError): - self.diagnostic.compute() - - def test_str(self): - self.assertEquals('Developer must override base class __str__ method', str(self.diagnostic)) - - def test_repr(self): - self.assertEquals(self.diagnostic.__repr__(), str(self.diagnostic)) - - def test_empty_process_options(self): - self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) - - # def test_empty_process_options(self): - # self.assertEqual(len(cls.process_options(('diag_name', ), tuple())), 0) +from mock import patch, Mock class TestDiagnosticOption(TestCase): @@ -251,9 +202,167 @@ class TestDiagnosticListIntOption(TestCase): diag = DiagnosticListIntOption('option') self.assertEqual([3], diag.parse('3')) + def test_too_low(self): + diag = DiagnosticListIntOption('option', min_limit=5) + with self.assertRaises(DiagnosticOptionError): + diag.parse('3') + + def test_too_high(self): + diag = DiagnosticListIntOption('option', max_limit=5) + with self.assertRaises(DiagnosticOptionError): + diag.parse('8') + def test_parse_bad_value(self): diag = DiagnosticListIntOption('option') with self.assertRaises(ValueError): diag.parse('3.5') +class TestDiagnosticChoiceOption(TestCase): + + def test_choice_value(self): + diag = DiagnosticChoiceOption('option', ('a', 'b')) + self.assertEqual('a', diag.parse('a')) + + def test_choice_default_value(self): + diag = DiagnosticChoiceOption('option', ('a', 'b'), default_value='a') + self.assertEqual('a', diag.parse('')) + + def test_bad_default_value(self): + with self.assertRaises(DiagnosticOptionError): + DiagnosticChoiceOption('option', ('a', 'b'), default_value='c') + + def test_ignore_case_value(self): + diag = DiagnosticChoiceOption('option', ('a', 'b')) + self.assertEqual('b', diag.parse('b')) + self.assertEqual('b', diag.parse('B')) + + diag = DiagnosticChoiceOption('option', ('a', 'b'), ignore_case=False) + self.assertEqual('b', diag.parse('b')) + with self.assertRaises(DiagnosticOptionError): + self.assertEqual('b', diag.parse('B')) + + +class TestDiagnosticVariableOption(TestCase): + + def get_var_mock(self, name): + mock = Mock() + mock.short_name = name + return mock + + def test_parse(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = self.get_var_mock('var1') + + diag = DiagnosticVariableOption(var_manager_mock) + self.assertEqual('var1', diag.parse('var1')) + + def test_not_recognized(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = None + + diag = DiagnosticVariableOption(var_manager_mock) + self.assertEqual('var1', diag.parse('var1')) + + +class TestDiagnosticVariableListOption(TestCase): + + def test_parse_multiple(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.side_effect = (self.get_var_mock('var1'), self.get_var_mock('var2')) + diag = DiagnosticVariableListOption(var_manager_mock, 'variables') + self.assertEqual(['var1', 'var2'], diag.parse('var1-var2')) + + def test_parse_one(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = self.get_var_mock('var1') + diag = DiagnosticVariableListOption(var_manager_mock, 'variables') + self.assertEqual(['var1'], diag.parse('var1')) + + def test_not_recognized(self): + var_manager_mock = Mock() + var_manager_mock.get_variable.return_value = None + diag = DiagnosticVariableListOption(var_manager_mock, 'variables') + self.assertEqual(['var1'], diag.parse('var1')) + + def get_var_mock(self, name): + mock = Mock() + mock.short_name = name + return mock + + +class TestDiagnostic(TestCase): + + def setUp(self): + class MockDiag(Diagnostic): + @classmethod + def generate_jobs(cls, diags, options): + pass + + def declare_data_generated(self): + pass + + def request_data(self): + pass + + def compute(self): + pass + + self.MockDiag = MockDiag + + def test_str(self): + self.assertEqual(str(Diagnostic(None)), 'Developer must override base class __str__ method') + + def test_compute_is_virtual(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).compute() + + def test_declare_data_generated_is_virtual(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).declare_data_generated() + + def test_request_data_is_virtual(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).request_data() + + @patch.object(Diagnostic, 'dispatch') + def test_set_status_call_dispatch(self, dispatch_mock): + diag = Diagnostic(None) + diag.status = DiagnosticStatus.FAILED + dispatch_mock.assert_called_once_with(diag) + + @patch.object(Diagnostic, 'dispatch') + def test_set_status_call_dispatch(self, dispatch_mock): + diag = Diagnostic(None) + diag.status = diag.status + assert not dispatch_mock.called, 'Dispatch should not have been called' + + def test_register(self): + with self.assertRaises(ValueError): + Diagnostic.register(TestDiagnostic) + + with self.assertRaises(ValueError): + Diagnostic.register(self.MockDiag) + + self.MockDiag.alias = 'mock' + Diagnostic.register(self.MockDiag) + + def test_get_diagnostic(self): + self.assertIsNone(Diagnostic.get_diagnostic('none')) + self.MockDiag.alias = 'mock' + Diagnostic.register(self.MockDiag) + self.assertIs(self.MockDiag, Diagnostic.get_diagnostic('mock')) + + def test_generate_jobs(self): + with self.assertRaises(NotImplementedError): + Diagnostic.generate_jobs(None, ['']) + + def test_compute(self): + with self.assertRaises(NotImplementedError): + Diagnostic(None).compute() + + def test_repr(self): + self.assertEquals(Diagnostic(None).__repr__(), str(Diagnostic(None))) + + def test_empty_process_options(self): + self.assertEqual(len(Diagnostic.process_options(('diag_name',), tuple())), 0) diff --git a/test/unit/test_earthdiags.py b/test/unit/test_earthdiags.py index 5dc657b81b4e6af7b491b7b037a1f9012e23a75d..122793ec6e24f2c02de20bfbb1c7f2d8132d9d4f 100644 --- a/test/unit/test_earthdiags.py +++ b/test/unit/test_earthdiags.py @@ -1,8 +1,8 @@ -# coding=utf-8 -from unittest import TestCase - -from earthdiagnostics.earthdiags import EarthDiags - - -class TestEarthDiags(TestCase): - pass +# # coding=utf-8 +# from unittest import TestCase +# +# from earthdiagnostics.earthdiags import EarthDiags +# +# +# class TestEarthDiags(TestCase): +# pass diff --git a/test/unit/test_frequency.py b/test/unit/test_frequency.py index 845dfe6ad336a4923da92ec4fd80663a65541076..34e32a3695d7f2f364100573dfbbbad5fe7cda7e 100644 --- a/test/unit/test_frequency.py +++ b/test/unit/test_frequency.py @@ -24,7 +24,10 @@ class TestFrequency(TestCase): self.assertEqual(Frequency('d').folder_name(VariableType.STATISTIC), 'daily_statistics') def test_get_6hourlymean(self): - self.assertEqual(Frequency('6hr').folder_name(VariableType.STATISTIC), '6hourly') + self.assertEqual(Frequency('6hr').folder_name(VariableType.MEAN), '6hourly') + + def test_get_6hourlystatistics(self): + self.assertEqual(Frequency('6hr').folder_name(VariableType.STATISTIC), '6hourly_statistics') def test_get_climatology(self): self.assertEqual(Frequency('clim').folder_name(VariableType.STATISTIC), 'clim') diff --git a/test/unit/test_heatcontent.py b/test/unit/test_heatcontent.py deleted file mode 100644 index a98d0c8651966e83bb1e251918051ab1fe7b9d70..0000000000000000000000000000000000000000 --- a/test/unit/test_heatcontent.py +++ /dev/null @@ -1,39 +0,0 @@ -# # coding=utf-8 -# from unittest import TestCase -# -# from earthdiagnostics.box import Box -# from earthdiagnostics.constants import Basins -# from earthdiagnostics.ocean.heatcontent import HeatContent -# from mock import Mock -# -# -# class TestHeatContent(TestCase): -# -# def setUp(self): -# self.data_manager = Mock() -# -# self.diags = Mock() -# self.diags.model_version = 'model_version' -# self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) -# -# self.box = Box(False) -# self.box.min_depth = 0 -# self.box.max_depth = 100 -# -# self.heat_content = HeatContent(self.data_manager, '20000101', 1, 1, Basins().Global, 1, self.box) -# -# def test_generate_jobs(self): -# jobs = HeatContent.generate_jobs(self.diags, ['diagnostic', 'atl', '-1', '0', '100']) -# self.assertEqual(len(jobs), 2) -# self.assertEqual(jobs[0], HeatContent(self.data_manager, '20010101', 0, 0, Basins().Atlantic, -1, self.box)) -# self.assertEqual(jobs[1], HeatContent(self.data_manager, '20010101', 0, 1, Basins().Atlantic, -1, self.box)) -# -# with self.assertRaises(Exception): -# HeatContent.generate_jobs(self.diags, ['diagnostic']) -# -# with self.assertRaises(Exception): -# HeatContent.generate_jobs(self.diags, ['diagnostic', '0', '0', '0', '0', '0', '0', '0']) -# -# def test_str(self): -# self.assertEquals(str(self.heat_content), 'Heat content Startdate: 20000101 Member: 1 Chunk: 1 Mixed layer: 1 ' -# 'Box: 0-100 Basin: Global_Ocean') diff --git a/test/unit/test_interpolate.py b/test/unit/test_interpolate.py deleted file mode 100644 index 3ca2ba52c1e78b44412834ca6459718fbdf2367d..0000000000000000000000000000000000000000 --- a/test/unit/test_interpolate.py +++ /dev/null @@ -1,54 +0,0 @@ -# # coding=utf-8 -# from unittest import TestCase -# -# from earthdiagnostics.ocean.interpolate import Interpolate -# from mock import Mock -# -# from earthdiagnostics.modelingrealm import ModelingRealms -# -# -# class TestInterpolate(TestCase): -# -# def setUp(self): -# self.data_manager = Mock() -# -# self.diags = Mock() -# self.diags.model_version = 'model_version' -# self.diags.config.experiment.get_chunk_list.return_value = (('20010101', 0, 0), ('20010101', 0, 1)) -# self.diags.config.experiment.model_version = 'model_version' -# -# self.interpolate = Interpolate(self.data_manager, '20000101', 1, 1, ModelingRealms.atmos, 'var', 'grid', -# 'model_version', False) -# -# def test_generate_jobs(self): -# jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var']) -# self.assertEqual(len(jobs), 2) -# self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.ocean, 'var', 'grid', -# 'model_version', False)) -# self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.ocean, 'var', 'grid', -# 'model_version', False)) -# -# jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos']) -# self.assertEqual(len(jobs), 2) -# self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', -# 'model_version', False)) -# self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', -# 'model_version', False)) -# -# jobs = Interpolate.generate_jobs(self.diags, ['interp', 'grid', 'var', 'atmos', 'true']) -# self.assertEqual(len(jobs), 2) -# self.assertEqual(jobs[0], Interpolate(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'var', 'grid', -# 'model_version', True)) -# self.assertEqual(jobs[1], Interpolate(self.data_manager, '20010101', 0, 1, ModelingRealms.atmos, 'var', 'grid', -# 'model_version', True)) -# -# with self.assertRaises(Exception): -# Interpolate.generate_jobs(self.diags, ['interp']) -# -# with self.assertRaises(Exception): -# Interpolate.generate_jobs(self.diags, ['interp', '0', '0', '0', '0', '0', '0', '0']) -# -# def test_str(self): -# self.assertEquals(str(self.interpolate), 'Interpolate Startdate: 20000101 Member: 1 Chunk: 1 ' -# 'Variable: atmos:var Target grid: grid Invert lat: False ' -# 'Model: model_version') diff --git a/test/unit/test_modelling_realm.py b/test/unit/test_modelling_realm.py index 2d44e6a6ddb03a17835afd1f3b5bb1bc881623f0..eec9d0cec9a9055ef6df139f05bf97bd307aedf8 100644 --- a/test/unit/test_modelling_realm.py +++ b/test/unit/test_modelling_realm.py @@ -18,19 +18,19 @@ class TestModellingRealms(TestCase): class TestModellingRealm(TestCase): def setUp(self): - self.basin = ModelingRealm('ocean') + self.realm = ModelingRealm('ocean') def test_constructor_fail_on_bad_realm(self): with self.assertRaises(ValueError): ModelingRealm('badrealm') def test_comparison(self): - self.assertEqual(ModelingRealm('ocean'), self.basin) - self.assertNotEqual(ModelingRealm('OCEAN'), self.basin) - self.assertNotEqual(ModelingRealm('atmos'), self.basin) + self.assertEqual(ModelingRealm('ocean'), self.realm) + self.assertNotEqual(ModelingRealm('OCEAN'), self.realm) + self.assertNotEqual(ModelingRealm('atmos'), self.realm) def test_get_omon(self): - self.assertEqual(self.basin.get_table_name(Frequencies.monthly, 'specs'), 'Omon') + self.assertEqual(self.realm.get_table_name(Frequencies.monthly, 'specs'), 'Omon') def test_get_oimon(self): self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'specs'), 'OImon') diff --git a/test/unit/test_publisher.py b/test/unit/test_publisher.py new file mode 100644 index 0000000000000000000000000000000000000000..5fe325ff819c7840370abbab9e7cc41d8e09d326 --- /dev/null +++ b/test/unit/test_publisher.py @@ -0,0 +1,37 @@ +# coding=utf-8 +from unittest import TestCase +from earthdiagnostics.publisher import Publisher +from mock import Mock + + +class TestPublisher(TestCase): + + def test_suscribe(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber, callback=suscriber.callback) + self.assertIn(suscriber, pub.suscribers) + + def test_suscribe_default(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber) + self.assertTrue(hasattr(suscriber, 'update')) + self.assertIn(suscriber, pub.suscribers) + + def test_unsuscribe(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber, callback=suscriber.callback) + pub.unsubscribe(suscriber) + + self.assertNotIn(suscriber, pub.suscribers) + + def test_dispatch(self): + suscriber = Mock() + pub = Publisher() + pub.subscribe(suscriber, callback=suscriber.callback) + + pub.dispatch(1, 2, 3) + suscriber.callback.assert_called_with(1, 2, 3) + diff --git a/test/unit/test_variable.py b/test/unit/test_variable.py index 07199ab822962e8d8c43a6685dddc30ddb4ff96f..c53baa633654b4b31525c08e25dad727b52f863e 100644 --- a/test/unit/test_variable.py +++ b/test/unit/test_variable.py @@ -2,7 +2,10 @@ from mock import Mock from unittest import TestCase -from earthdiagnostics.variable import CMORTable, VariableAlias +from earthdiagnostics.variable import CMORTable, VariableAlias, Variable, VariableJsonException +from earthdiagnostics.modelingrealm import ModelingRealms +from earthdiagnostics.constants import Basins +from earthdiagnostics.frequency import Frequencies class TestCMORTable(TestCase): @@ -25,3 +28,157 @@ class TestVariableAlias(TestCase): self.assertEquals(str(alias), 'alias Basin: basin Grid: grid') +class TestVariable(TestCase): + + def test_parse_json(self): + var = Variable() + json = {'out_name': 'out_name', + 'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + } + var.parse_json(json, 'out_name') + + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.priority, 1) + + self.assertEqual(var.domain, ModelingRealms.ocean) + + def test_parse_json_no_out_name(self): + var = Variable() + json = {'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + } + with self.assertRaises(VariableJsonException): + var.parse_json(json, 'out_name') + + def test_parse_json_with_priority(self): + var = Variable() + json = {'out_name': 'out_name', + 'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + 'priority': '2', + } + var.parse_json(json, 'out_name') + + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.priority, 2) + + self.assertEqual(var.domain, ModelingRealms.ocean) + + def test_parse_json_with_primavera_priority(self): + var = Variable() + json = {'out_name': 'out_name', + 'standard_name': 'standard_name', + 'long_name': 'long_name', + 'modeling_realm': 'ocean', + 'valid_min': 'valid_min', + 'valid_max': 'valid_max', + 'units': 'units', + 'primavera_priority': '2', + } + var.parse_json(json, 'out_name') + + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.priority, 2) + + self.assertEqual(var.domain, ModelingRealms.ocean) + + def test_get_modelling_realm(self): + var = Variable() + domain = var.get_modelling_realm(('ocean',)) + self.assertEqual(ModelingRealms.ocean, domain) + + domain = var.get_modelling_realm(('ocean', 'atmos')) + self.assertEqual(ModelingRealms.ocean, domain) + + domain = var.get_modelling_realm(('ocean', 'ocnBgchem')) + self.assertEqual(ModelingRealms.ocnBgchem, domain) + + domain = var.get_modelling_realm(('ocean', 'seaIce')) + self.assertEqual(ModelingRealms.seaIce, domain) + + domain = var.get_modelling_realm(('atmos', 'atmosChem')) + self.assertEqual(ModelingRealms.atmosChem, domain) + + domain = var.get_modelling_realm(('land', 'landIce')) + self.assertEqual(ModelingRealms.landIce, domain) + + domain = var.get_modelling_realm(tuple()) + self.assertIsNone(domain) + + def test_parse_csv(self): + var = Variable() + var.parse_csv(['not_used', 'out_name', 'standard_name', 'long_name', 'ocean', 'global', 'units', + 'valid_min', 'valid_max', 'grid', 'Amon: ']) + self.assertEqual(var.short_name, 'out_name') + self.assertEqual(var.standard_name, 'standard_name') + self.assertEqual(var.long_name, 'long_name') + + self.assertEqual(var.valid_min, 'valid_min') + self.assertEqual(var.valid_max, 'valid_max') + self.assertEqual(var.units, 'units') + self.assertEqual(var.grid, 'grid') + + self.assertEqual(var.domain, ModelingRealms.ocean) + self.assertEqual(var.basin, Basins().Global) + + def test_get_table(self): + var = Variable() + var.domain = ModelingRealms.atmos + table = var.get_table(Frequencies.monthly, 'specs') + self.assertEqual(table.frequency, Frequencies.monthly) + self.assertEqual(table.name, 'Amon') + self.assertEqual(table.date, 'December 2013') + + def test_get_table_added(self): + var = Variable() + var.domain = ModelingRealms.atmos + var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) + table = var.get_table(Frequencies.monthly, 'specs') + self.assertEqual(table.frequency, Frequencies.monthly) + self.assertEqual(table.name, 'Amon') + self.assertEqual(table.date, 'December 2013') + + def test_get_table_not_added(self): + var = Variable() + var.domain = ModelingRealms.atmos + var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013')) + table = var.get_table(Frequencies.daily, 'specs') + self.assertEqual(table.frequency, Frequencies.daily) + self.assertEqual(table.name, 'day') + self.assertEqual(table.date, 'December 2013') + + def test_get_table_not_matching(self): + var = Variable() + with self.assertRaises(ValueError): + var.get_table(Frequencies.daily, 'specs')