diff --git a/VERSION b/VERSION index 13d22bb6857324e2a20bd9229c94eb193a70db43..ea82d6cc91680129741851af5b1de7cee01edade 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.0.0b18 +3.0.0b19 diff --git a/diags.conf b/diags.conf index 7f0695a3e98c4eecff46440ba77f1dfffae04f85..149ce4212a9954e0afd5f17f1811e58afe802be8 100644 --- a/diags.conf +++ b/diags.conf @@ -11,13 +11,13 @@ CON_FILES = /esnas/autosubmit/con_files/ # Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or # an alias defined in the ALIAS section (see more below). If you are using the diagnpostics just to CMORize, leave it # empty -DIAGS = climpercent,ocean,tas,2 +DIAGS = monpercent,atmos,tas,90 monpercent,atmos,tas,10 monpercent,atmos,sfcWind,90 monpercent,atmos,sfcWind,10 # DIAGS = OHC # Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores # its results at monthly frequency (obvious) and has a parameter to specify input's frequency. FREQUENCY = 6hr # Path to CDFTOOLS binaries -CDFTOOLS_PATH = ~/CDFTOOLS/bin +CDFTOOLS_PATH = # If true, copies the mesh files regardless of presence in scratch dir RESTORE_MESHES = False # Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available) @@ -33,9 +33,6 @@ ATMOSPHERE_FILES = True # You can specify the variable to cmorize, in the way domain:var domain:var2 domain2:var VARIABLE_LIST = ocean:tos -[THREDDS] -SERVER_URL = http://earth.bsc.es/thredds - # Variables to be CMORized from the grib atmospheric files, separated by comma. # You can also specify the levels to extract using the following syntax # VARIABLE_CODE, VARIABLE_CODE:LEVEL, VARIABLE_CODE:LEVEL1-LEVEL2, VARIABLE_CODE:MIN_LEVEL:MAX_LEVEL:STEP @@ -61,6 +58,9 @@ ATMOS_MONTHLY_VARS = 167, 201, 202, 165, 166, 151, 144, 228, 205, 182, 164, 146, # ASSOCIATED_MODEL = # SOURCE = 'EC-Earthv2.3.0, ocean: Nemo3.1, ifs31r1, lim2 +[THREDDS] +SERVER_URL = http://earth.bsc.es/thredds + [EXPERIMENT] # Experiments parameters as defined in CMOR standard INSTITUTE = ecmwf @@ -68,9 +68,9 @@ MODEL = system4_m1 # Model version: Available versions MODEL_VERSION =Ec2.3_O1L46 # Atmospheric output timestep in hours -ATMOS_TIMESTEP = 3 +ATMOS_TIMESTEP = 6 # Ocean output timestep in hours -OCEAN_TIMESTEP = 3 +OCEAN_TIMESTEP = 6 # For those who use Autosubmit, this will be easy # EXPID is the unique identifier of the experiment. @@ -80,11 +80,11 @@ OCEAN_TIMESTEP = 3 # if 2, fc00 # CHUNK_SIZE is the size of each data file, given in months # CHUNKS is the number of chunks. You can specify less chunks than present on the experiment -EXPID = i03i -STARTDATES = 19811101 19821101 19831101 +EXPID = resilience +STARTDATES = 19811101 MEMBERS = 0 MEMBER_DIGITS = 1 -CHUNK_SIZE = 4 +CHUNK_SIZE = 7 CHUNKS = 1 # CHUNKS = 1 diff --git a/doc/source/conf.py b/doc/source/conf.py index 27204143eaa8e5e6db6d572b037fe51a7139f86d..de252a5c547e44024ec7e14df5571cc1973e055c 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -64,7 +64,7 @@ copyright = u'2016, BSC-CNS Earth Sciences Department' # The short X.Y version. version = '3.0b' # The full version, including alpha/beta/rc tags. -release = '3.0.0b18' +release = '3.0.0b19' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/earthdiagnostics/EarthDiagnostics.pdf b/earthdiagnostics/EarthDiagnostics.pdf index 9128d4f49f9b5c64569b424ad307a124a897a2c1..e75ffc2163c2bcc599212feb08d17d64fa4fa004 100644 Binary files a/earthdiagnostics/EarthDiagnostics.pdf and b/earthdiagnostics/EarthDiagnostics.pdf differ diff --git a/earthdiagnostics/cmor_table.csv b/earthdiagnostics/cmor_table.csv index fc4d16e93de36a9071d88ae6826285dfd86e6e2a..4e8090b10eb0ed92449b49081a8e23238cf3afd8 100644 --- a/earthdiagnostics/cmor_table.csv +++ b/earthdiagnostics/cmor_table.csv @@ -261,7 +261,7 @@ scsshtst,zostoga,snthic,Global average thermosteric sea level change ,ocean,,,,, heatc,ohc,ocean_heat_content,Ocean heat content,ocean,,J,,, ohcsum,ohcsum,total_ocean_heat_content,Total Ocean heat content,ocean,,J,,, ohcvmean,ohcvmean,average_ocean_heat_content,Average Ocean heat content,ocean,,J m-3,,, -ohc,ohc,ocean_heat_content,Ocean heat content,ocean,,J,,, +ohc,ohc,ocean_heat_content,Ocean heat content,ocean,,J m-2,,, transix,transix,sea_ice_x_transport,X-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,, transiy,transiy,sea_ice_y_transport,Y-Component of Sea Ice Mass Transport,seaIce,,kg s-1,,, windsp,sfcWind,wind_speed,Near-Surface Wind Speed,atmos,,,,, diff --git a/earthdiagnostics/cmormanager.py b/earthdiagnostics/cmormanager.py index ea033ea72799af469a500e4eb134a6cd85c4d5eb..2382c62a4978b71c93490d707e0306e5105b5702 100644 --- a/earthdiagnostics/cmormanager.py +++ b/earthdiagnostics/cmormanager.py @@ -9,7 +9,7 @@ from autosubmit.date.chunk_date_lib import parse_date, chunk_start_date, chunk_e from earthdiagnostics.cmorizer import Cmorizer from earthdiagnostics.datamanager import DataManager, NetCDFFile from earthdiagnostics.utils import TempFile, Utils -from earthdiagnostics.variable import Variable +from earthdiagnostics.variable import Variable, VarType class CMORManager(DataManager): @@ -28,7 +28,8 @@ class CMORManager(DataManager): if not self.config.data_dir: raise Exception('Can not find model data') - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None): + def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VarType.MEAN): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy @@ -161,11 +162,11 @@ class CMORManager(DataManager): if not frequency: frequency = self.config.frequency filepath = self.get_file_path(startdate, member, domain, var, chunk, frequency, grid, str(year), date_str) - self._create_link(domain, filepath, frequency, var, grid, move_old) + self._create_link(domain, filepath, frequency, var, grid, move_old, vartype) def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False): + diagnostic=None, cmorized=False, vartype=VarType.MEAN): """ Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge with already existing ones as needed @@ -228,7 +229,7 @@ class CMORManager(DataManager): 'using the CMORManager') netcdf_file.send() - self._create_link(domain, filepath, frequency, var, grid, move_old) + self._create_link(domain, filepath, frequency, var, grid, move_old, vartype) def get_year(self, domain, var, startdate, member, year, grid=None, box=None): """ @@ -395,10 +396,11 @@ class CMORManager(DataManager): for name in os.listdir(os.path.join(path, freq, domain, var, member)): filepath = os.path.join(path, freq, domain, var, member, name) if os.path.isfile(filepath): - self._create_link(domain, filepath, freq, var, "", False) + self._create_link(domain, filepath, freq, var, "", False, vartype=VarType.MEAN) else: for filename in os.listdir(filepath): - self._create_link(domain, os.path.join(filepath, filename), freq, var, "", False) + self._create_link(domain, os.path.join(filepath, filename), freq, var, "", False, + vartype=VarType.MEAN) Log.info('Creating lings for CMOR files') def _get_startdate_path(self, startdate): diff --git a/earthdiagnostics/datamanager.py b/earthdiagnostics/datamanager.py index 7b3b453a3a9aea09b9563a71ff6989ac4ad720e7..b837d50eb72f7f6ff2ab07b8e165cec702f283e8 100644 --- a/earthdiagnostics/datamanager.py +++ b/earthdiagnostics/datamanager.py @@ -10,7 +10,7 @@ import re from cfunits import Units from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Variable, Domains +from earthdiagnostics.variable import Variable, Domains, VarType class DataManager(object): @@ -29,7 +29,8 @@ class DataManager(object): self.lock = threading.Lock() self.cmor_path = os.path.join(self.config.data_dir, self.experiment.expid, 'cmorfiles') - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None): + def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VarType.MEAN): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy @@ -56,7 +57,7 @@ class DataManager(object): def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False): + diagnostic=None, cmorized=False, vartype=VarType.MEAN): """ Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge with already existing ones as needed @@ -133,8 +134,8 @@ class DataManager(object): else: return '{0}_f{1}h'.format(var, self.experiment.atmos_timestep) - def _create_link(self, domain, filepath, frequency, var, grid, move_old): - freq_str = self.frequency_folder_name(frequency) + def _create_link(self, domain, filepath, frequency, var, grid, move_old, vartype): + freq_str = self.frequency_folder_name(frequency, vartype) if not grid: grid = 'original' @@ -183,9 +184,9 @@ class DataManager(object): self.lock.release() @staticmethod - def frequency_folder_name(frequency): + def frequency_folder_name(frequency, vartype): if frequency in ('d', 'daily', 'day'): - freq_str = 'daily_mean' + freq_str = 'daily_{0}'.format(VarType.to_str(vartype)) elif frequency == 'clim': freq_str = 'clim' elif frequency.endswith('hr'): @@ -372,6 +373,7 @@ class NetCDFFile(object): except ValueError: factor, offset = UnitConversion.get_conversion_factor_offset(var_handler.units, self.cmor_var.units) + var_handler[:] = var_handler[:] * factor + offset if 'valid_min' in var_handler.ncattrs(): var_handler.valid_min = float(var_handler.valid_min) * factor + offset diff --git a/earthdiagnostics/diagnostic.py b/earthdiagnostics/diagnostic.py index af71dea66e4b25be3a672f8c119ef640cb518e4c..5eac342437954275e5147ff98d09f7a253432aa2 100644 --- a/earthdiagnostics/diagnostic.py +++ b/earthdiagnostics/diagnostic.py @@ -1,4 +1,7 @@ # coding=utf-8 +from earthdiagnostics.variable import VarType + + class Diagnostic(object): """ Base class for the diagnostics. Provides a common interface for them and also @@ -52,7 +55,8 @@ class Diagnostic(object): return None def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, - box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False): + box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, + vartype=VarType.MEAN): """ :param filetosend: @@ -73,7 +77,8 @@ class Diagnostic(object): :return: """ self.data_manager.send_file(filetosend, domain, var, startdate, member, chunk, grid, region, - box, rename_var, frequency, year, date_str, move_old, diagnostic=self) + box, rename_var, frequency, year, date_str, move_old, diagnostic=self, + vartype=vartype) def compute(self): """ diff --git a/earthdiagnostics/statistics/climatologicalpercentile.py b/earthdiagnostics/statistics/climatologicalpercentile.py index 470b0c303ad25b9c1a00d2f9d5da143339719f1b..fb8c635ff6705d468a305b8fe4d9f74d7f8191c1 100644 --- a/earthdiagnostics/statistics/climatologicalpercentile.py +++ b/earthdiagnostics/statistics/climatologicalpercentile.py @@ -3,7 +3,7 @@ from autosubmit.config.log import Log from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain, Variable +from earthdiagnostics.variable import Domain, Variable, VarType import numpy as np @@ -111,7 +111,8 @@ class ClimatologicalPercentile(Diagnostic): handler.close() - self.send_file(temp, self.domain, self.variable + 'percent', None, None, frequency='clim', rename_var='percent') + self.send_file(temp, self.domain, self.variable + 'percent', None, None, frequency='clim', rename_var='percent', + vartype=VarType.STATISTIC) def _calculate_percentiles(self, distribution): Log.debug('Calculating percentiles') diff --git a/earthdiagnostics/statistics/monthlypercentile.py b/earthdiagnostics/statistics/monthlypercentile.py index 7ac103083172c8080b02fa45cda494edd0eee07b..06aa9d83f7ddb96e69a435fd7466d94b76e15982 100644 --- a/earthdiagnostics/statistics/monthlypercentile.py +++ b/earthdiagnostics/statistics/monthlypercentile.py @@ -1,7 +1,10 @@ # coding=utf-8 +import shutil + from earthdiagnostics.diagnostic import Diagnostic from earthdiagnostics.utils import Utils, TempFile -from earthdiagnostics.variable import Domain +from earthdiagnostics.variable import Domain, VarType +from calendar import monthrange class MonthlyPercentile(Diagnostic): @@ -76,11 +79,35 @@ class MonthlyPercentile(Diagnostic): """ variable_file = self.data_manager.get_file(self.domain, self.variable, self.startdate, self.member, self.chunk) temp = TempFile.get() + handler = Utils.openCdf(variable_file) + datetimes = Utils.get_datetime_from_netcdf(handler) + handler.close() + + start_index = 0 + while datetimes[start_index].day != 1 and start_index < handler.size: + start_index += 1 + if start_index == datetimes.size: + raise Exception('No complete month for diagnostic {0}'.format(self)) + + end_index = datetimes.size - 1 + while datetimes[end_index].day != monthrange(datetimes[end_index].year, datetimes[end_index].month)[1] \ + and end_index >= 0: + end_index -= 1 + + if end_index < 0: + raise Exception('No complete month for diagnostic {0}'.format(self)) + + if start_index != 0 or end_index != datetimes.size - 1: + start_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[start_index]) + end_date = '{0.year}-{0.month}-{0.day}'.format(datetimes[end_index]) + Utils.cdo.seldate('{0},{1}'.format(start_date, end_date), input=variable_file, output=temp) + shutil.move(temp, variable_file) + Utils.cdo.monpctl(str(self.percentile), input=[variable_file, '-monmin ' + variable_file, '-monmax ' + variable_file], output=temp) Utils.rename_variable(temp, 'lev', 'ensemble', False, True) - self.send_file(temp, self.domain, '{0}p{1}'.format(self.variable, self.percentile), self.startdate, self.member, - self.chunk, frequency='mon', rename_var=self.variable) + self.send_file(temp, self.domain, '{0}_q{1}'.format(self.variable, self.percentile), self.startdate, self.member, + self.chunk, frequency='mon', rename_var=self.variable, vartype=VarType.STATISTIC) diff --git a/earthdiagnostics/threddsmanager.py b/earthdiagnostics/threddsmanager.py index d10fbc10f77a6f7f0706312617eff00e419ee50e..ded8b5571b4a2ef202fd6c977646111597410040 100644 --- a/earthdiagnostics/threddsmanager.py +++ b/earthdiagnostics/threddsmanager.py @@ -1,6 +1,4 @@ # coding=utf-8 -import csv - import os from autosubmit.date.chunk_date_lib import parse_date, add_months @@ -8,7 +6,7 @@ from earthdiagnostics.datamanager import DataManager, NetCDFFile from earthdiagnostics.utils import TempFile, Utils import urllib -from earthdiagnostics.variable import Variable +from earthdiagnostics.variable import Variable, VarType class THREDDSManager(DataManager): @@ -29,10 +27,10 @@ class THREDDSManager(DataManager): if not self.config.data_dir: raise Exception('Can not find model data') - def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None): + def get_leadtimes(self, domain, variable, startdate, member, leadtimes, frequency=None, vartype=VarType.MEAN): if not frequency: frequency = self.config.frequency - aggregation_path = self.get_var_url(variable, startdate, frequency, None, False) + aggregation_path = self.get_var_url(variable, startdate, frequency, None, False, vartype) temp = TempFile.get() startdate = parse_date(startdate) selected_months = ','.join([str(add_months(startdate, i, 'standard').month) for i in leadtimes]) @@ -41,7 +39,8 @@ class THREDDSManager(DataManager): Utils.cdo.selyear(selected_years, input=select_months, output=temp) return temp - def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None): + def get_file(self, domain, var, startdate, member, chunk, grid=None, box=None, frequency=None, + vartype=VarType.MEAN): """ Copies a given file from the CMOR repository to the scratch folder and returns the path to the scratch's copy @@ -66,7 +65,7 @@ class THREDDSManager(DataManager): """ if not frequency: frequency = self.config.frequency - aggregation_path = self.get_var_url(var, startdate, frequency, box, True) + aggregation_path = self.get_var_url(var, startdate, frequency, box, True, vartype) temp = TempFile.get() urllib.urlretrieve(aggregation_path, temp) if not Utils.check_netcdf_file(temp): @@ -75,7 +74,7 @@ class THREDDSManager(DataManager): def send_file(self, filetosend, domain, var, startdate, member, chunk=None, grid=None, region=None, box=None, rename_var=None, frequency=None, year=None, date_str=None, move_old=False, - diagnostic=None, cmorized=False): + diagnostic=None, cmorized=False, vartype=VarType.MEAN): """ Copies a given file to the CMOR repository. It also automatically converts to netCDF 4 if needed and can merge with already existing ones as needed @@ -129,8 +128,7 @@ class THREDDSManager(DataManager): if not frequency: frequency = self.config.frequency - filepath = self.get_file_path(startdate, domain, var, frequency, box, - grid) + filepath = self.get_file_path(startdate, domain, var, frequency, vartype, box, grid) netcdf_file = NetCDFFile(filepath, filetosend, domain, var, cmor_var) if diagnostic: netcdf_file.add_diagnostic_history(diagnostic) @@ -138,7 +136,7 @@ class THREDDSManager(DataManager): raise ValueError('You must provide a diagnostic to store data using the THREDDSmanager') netcdf_file.send() - def get_file_path(self, startdate, domain, var, frequency, + def get_file_path(self, startdate, domain, var, frequency, vartype, box=None, grid=None): """ Returns the path to a concrete file @@ -161,7 +159,7 @@ class THREDDSManager(DataManager): frequency = self.config.frequency var = self._get_final_var_name(box, var) - folder_path = self._get_folder_path(frequency, domain, var, grid) + folder_path = self._get_folder_path(frequency, domain, var, grid, vartype) if startdate: file_name = '{0}_{1}.nc'.format(var, startdate) else: @@ -170,11 +168,11 @@ class THREDDSManager(DataManager): filepath = os.path.join(folder_path, file_name) return filepath - def _get_folder_path(self, frequency, domain, variable, grid): + def _get_folder_path(self, frequency, domain, variable, grid, vartype): folder_path = os.path.join(self.config.data_dir, self.experiment.institute.lower(), self.experiment.model.lower(), - self.frequency_folder_name(frequency), + self.frequency_folder_name(frequency, vartype), self.get_varfolder(domain, variable, grid)) return folder_path @@ -198,14 +196,14 @@ class THREDDSManager(DataManager): :return: """ - def get_var_url(self, var, startdate, frequency, box, fileserver): + def get_var_url(self, var, startdate, frequency, box, fileserver, vartype): var = self._get_final_var_name(box, var) if fileserver: protocol = 'fileServer' else: protocol = 'dodsC' return os.path.join(self.server_url, protocol, 'exp', self.experiment.institute, - self.experiment.model, self.frequency_folder_name(frequency), + self.experiment.model, self.frequency_folder_name(frequency, vartype), var, '{0}_{1}.nc'.format(var, startdate)) def link_file(self, domain, var, startdate, member, chunk=None, grid=None, box=None, diff --git a/earthdiagnostics/variable.py b/earthdiagnostics/variable.py index a187081242955b94bf54cb5bcbb8e5c39a667348..e6d0e9b004f7b27500fbecfd66a65b82a12f9203 100644 --- a/earthdiagnostics/variable.py +++ b/earthdiagnostics/variable.py @@ -113,3 +113,18 @@ class Domains(object): atmos = Domain('seaice') land = Domain('land') + +class VarType(object): + MEAN = 1 + STATISTIC = 2 + + @staticmethod + def to_str(vartype): + if vartype == VarType.MEAN: + return 'mean' + elif vartype == VarType.STATISTIC: + return 'statistic' + else: + raise ValueError('Variable type {0} not supported'.format(vartype)) + +