Commit 7fc69265 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Fixed cmorization

parent 905a187a
......@@ -16,7 +16,7 @@ CON_FILES = /esnas/autosubmit/con_files/
# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or
# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it
# empty
DIAGS = discretize,atmos,sfcWind,2000,0,40 climpercent,atmos,sfcWind,1984,1984,1
DIAGS = discretize,atmos,sfcWind,2000,0,40 climpercent,atmos,sfcWind,1991,1992,11
# DIAGS = OHC
# Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores
# its results at monthly frequency (obvious) and has a parameter to specify input's frequency.
......@@ -25,8 +25,8 @@ FREQUENCY = 6hr
CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin
# If true, copies the mesh files regardless of presence in scratch dir
RESTORE_MESHES = False
# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available)
MAX_CORES = 2
# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available)z
# MAX_CORES = 2
[CMOR]
# If true, recreates CMOR files regardless of presence. Default = False
......@@ -87,7 +87,7 @@ OCEAN_TIMESTEP = 6
# CHUNK_SIZE is the size of each data file, given in months
# CHUNKS is the number of chunks. You can specify less chunks than present on the experiment
EXPID = testing_erainterim
STARTDATES = 19911101 19921101
STARTDATES = 19931101 19941101 19951101 19961101 19971101 19981101 19991101 20001101 20011101 20021101 20031101
MEMBERS = 0
MEMBER_DIGITS = 1
CHUNK_SIZE = 1
......
......@@ -277,13 +277,12 @@ class Cmorizer(object):
Log.info('Unpacking... ')
# remap on regular Gauss grid
if grid == 'SH':
Utils.cdo.splitparam(input='-sp2gpl {0}'.format(full_file), output=gribfile + '_',
options='-f nc4')
Utils.cdo.splitparam(input=['-sp2gpl {0}'.format(full_file), gribfile + '_'], options='-f nc4')
else:
Utils.cdo.splitparam(input=full_file, output=gribfile + '_', options='-R -f nc4')
Utils.cdo.splitparam(input=[full_file, gribfile + '_'], options='-R -f nc4')
# total precipitation (remove negative values)
Utils.cdo.setcode(228, input='-setmisstoc,0 -setvrange,0,Inf -add '
'{0}_{{142,143}}.128.nc'.format(gribfile),
Utils.cdo.setcode(228,
input='-setmisstoc,0 -setvrange,0,Inf -add {0}_{{142,143}}.128.nc'.format(gribfile),
output='{0}_228.128.nc'.format(gribfile))
Utils.remove_file('ICM')
......@@ -645,12 +644,15 @@ class Cmorizer(object):
gribfiles = glob.glob(grb_path)
return len(gribfiles) > 0
def cmorization_required(self, chunk, domain):
def cmorization_required(self, chunk, domains):
if not self.config.cmor.chunk_cmorization_requested(chunk):
return False
if self.config.cmor.force:
return True
return not self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain)
for domain in domains:
if self.data_manager.is_cmorized(self.startdate, self.member, chunk, domain):
return False
return True
class CMORException(Exception):
......
......@@ -377,46 +377,45 @@ class CMORManager(DataManager):
if not self._unpack_cmor_files(startdate, member):
self._cmorize_member(startdate, member)
def is_cmorized(self, startdate, member, chunk):
def is_cmorized(self, startdate, member, chunk, domain):
identifier = (startdate, member, chunk)
if identifier not in self._dic_cmorized:
self._dic_cmorized[identifier] = self._is_cmorized(startdate, member, chunk)
if self._dic_cmorized[identifier]:
return True
return False
self._dic_cmorized[identifier] = {}
self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain)
elif domain not in self._dic_cmorized[identifier]:
self._dic_cmorized[identifier][domain] = self._is_cmorized(startdate, member, chunk, domain)
return self._dic_cmorized[identifier][domain]
def _is_cmorized(self, startdate, member, chunk):
def _is_cmorized(self, startdate, member, chunk, domain):
startdate_path = self._get_startdate_path(startdate)
if not os.path.isdir(startdate_path):
return False
if self.config.data_convention == 'specs':
for freq in os.listdir(startdate_path):
for domain in (ModelingRealms.ocean, ModelingRealms.ocnBgchem, ModelingRealms.ocnBgchem,
ModelingRealms.atmos):
domain_path = os.path.join(startdate_path, freq,
domain.name)
if os.path.isdir(domain_path):
for var in os.listdir(domain_path):
cmor_var = self.variable_list.get_variable(var, True)
var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk,
Frequency(freq))
if os.path.isfile(var_path):
return True
domain_path = os.path.join(startdate_path, freq,
domain.name)
if os.path.isdir(domain_path):
for var in os.listdir(domain_path):
cmor_var = self.variable_list.get_variable(var, True)
var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk,
Frequency(freq))
if os.path.isfile(var_path):
return True
else:
member_path = os.path.join(startdate_path, self._get_member_str(member))
if not os.path.isdir(member_path):
return False
for table, domain, freq in (('Amon', ModelingRealms.atmos, Frequencies.monthly),
('Omon', ModelingRealms.ocean, Frequencies.monthly),
('SImon', ModelingRealms.seaIce, Frequencies.monthly)):
table_dir = os.path.join(member_path, table)
if not os.path.isdir(table_dir):
continue
for var in os.listdir(table_dir):
cmor_var = self.variable_list.get_variable(var, True)
var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq)
if os.path.isfile(var_path):
return True
freq = Frequencies.monthly
table = domain.get_table(freq, self.config.data_convention)
table_dir = os.path.join(member_path, table)
if not os.path.isdir(table_dir):
return False
for var in os.listdir(table_dir):
cmor_var = self.variable_list.get_variable(var, True)
var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq)
if os.path.isfile(var_path):
return True
return False
def _cmorize_member(self, startdate, member):
......@@ -436,7 +435,8 @@ class CMORManager(DataManager):
cmorized = False
if not self.config.cmor.force_untar:
while self.is_cmorized(startdate, member, chunk):
while self.is_cmorized(startdate, member, chunk, ModelingRealms.atmos) or \
self.is_cmorized(startdate, member, chunk, ModelingRealms.ocean):
chunk += 1
while self._unpack_chunk(startdate, member, chunk):
......
#!/usr/bin/env bash
#SBATCH -n 8
#SBATCH --time 2:00:00
#SBATCH --time 7-00:00:00
#SBATCH --error=job.%J.err
#SBATCH --output=job.%J.out
PATH_TO_CONF_FILE=~vtorralb/diags-erai.conf
PATH_TO_CONF_FILE=~jvegas/earthdiagnostics/diags.conf
PATH_TO_DIAGNOSTICS=~jvegas/earthdiagnostics
PATH_TO_CONDAENV=~jvegas/anaconda/envs/earthdiags/bin
......@@ -22,4 +22,4 @@ source ${PATH_TO_CONDAENV}/activate earthdiags
export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH}
cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/
./earthdiags.py -f ${PATH_TO_CONF_FILE}
./earthdiags.py -lc DEBUG -f ${PATH_TO_CONF_FILE}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment