Commit e512b9a3 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Add memory tracker for discretize diagnostic

parent e1dbf8f6
......@@ -16,7 +16,8 @@ CON_FILES = /esnas/autosubmit/con_files/
# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or
# an alias defined in the ALIAS section (see more below). If you are using the diagnostics just to CMORize, leave it
# empty
DIAGS = discretize,atmos,sfcWind,2000,0,40 climpercent,atmos,sfcWind,1991,1992,11
DIAGS = discretize,atmos,sfcWind,2000,0,40
# climpercent,atmos,sfcWind,1991,1992,11
# DIAGS = OHC
# Frequency of the data you want to use by default. Some diagnostics do not use this value: i.e. monmean always stores
# its results at monthly frequency (obvious) and has a parameter to specify input's frequency.
......@@ -26,7 +27,7 @@ CDFTOOLS_PATH = ~jvegas/CDFTOOLS/bin
# If true, copies the mesh files regardless of presence in scratch dir
RESTORE_MESHES = False
# Limits the maximum amount of threads used. Default: 0 (no limitation, one per virtual core available)z
# MAX_CORES = 2
MAX_CORES = 1
[CMOR]
# If true, recreates CMOR files regardless of presence. Default = False
......@@ -87,7 +88,8 @@ OCEAN_TIMESTEP = 6
# CHUNK_SIZE is the size of each data file, given in months
# CHUNKS is the number of chunks. You can specify less chunks than present on the experiment
EXPID = testing_erainterim
STARTDATES = 19931101 19941101 19951101 19961101 19971101 19981101 19991101 20001101 20011101 20021101 20031101
# STARTDATES = 19931101 19941101 19951101 19961101 19971101 19981101 19991101 20001101 20011101 20021101 20031101
STARTDATES = 19840101 19850101
MEMBERS = 0
MEMBER_DIGITS = 1
CHUNK_SIZE = 1
......
# coding=utf-8
import six
from bscearth.utils.date import parse_date, add_months
from bscearth.utils.log import Log
from earthdiagnostics.diagnostic import Diagnostic, DiagnosticVariableOption, DiagnosticDomainOption, \
DiagnosticIntOption, DiagnosticListIntOption, DiagnosticFloatOption
from earthdiagnostics.frequency import Frequencies
DiagnosticIntOption, DiagnosticFloatOption
from earthdiagnostics.utils import Utils, TempFile
from earthdiagnostics.variable_type import VariableType
import numpy as np
......@@ -16,6 +14,8 @@ from iris.time import PartialDateTime
import iris.exceptions
import iris.coords
import math
import psutil
import six
class Discretize(Diagnostic):
......@@ -65,6 +65,11 @@ class Discretize(Diagnostic):
self.max_value = None
self.check_max_value = True
self.process = psutil.Process()
def print_memory_used(self):
Log.user_warning('Memory: {0:.2f} GB'.format(self.process.memory_info().rss / 1024.0**3))
@property
def bins(self):
if self._bins is None:
......@@ -121,12 +126,20 @@ class Discretize(Diagnostic):
"""
Runs the diagnostic
"""
self.print_memory_used()
iris.FUTURE.netcdf_promote = True
self._load_cube()
self.print_memory_used()
self._get_value_interval()
self.print_memory_used()
Log.info('Range: [{0}, {1}]', self.min_value, self.max_value)
self._get_distribution()
self.print_memory_used()
self._save_results()
self.print_memory_used()
del self.distribution
del self.data_cube
self.print_memory_used()
def _load_cube(self):
......@@ -171,7 +184,7 @@ class Discretize(Diagnostic):
cubes = iris.cube.CubeList()
for leadtime, distribution in self.distribution.iteritems():
for leadtime, distribution in six.iteritems(self.distribution):
leadtime_cube = Cube(distribution.astype(np.uint32), var_name=self.data_cube.var_name,
standard_name=self.data_cube.standard_name, units='1')
leadtime_cube.add_dim_coord(bins_coord, 0)
......@@ -191,9 +204,9 @@ class Discretize(Diagnostic):
for leadtime in set(self.data_cube.coord('leadtime').points):
Log.debug('Discretizing leadtime {0}', leadtime)
leadtime_cube = self.data_cube.extract(iris.Constraint(leadtime=leadtime))
for realization in self.data_cube.coord('realization').points:
Log.debug('Discretizing realization {0}', realization)
self.print_memory_used()
try:
realization_cube = leadtime_cube.extract(iris.Constraint(realization=realization))
except iris.exceptions.CoordinateNotFoundError:
......
#!/usr/bin/env bash
#SBATCH -n 8
#SBATCH -n 1
#SBATCH --time 7-00:00:00
#SBATCH --error=job.%J.err
#SBATCH --output=job.%J.out
......@@ -9,16 +9,17 @@
PATH_TO_CONF_FILE=~jvegas/earthdiagnostics/diags.conf
PATH_TO_DIAGNOSTICS=~jvegas/earthdiagnostics
PATH_TO_CONDAENV=~jvegas/anaconda/envs/earthdiags/bin
PATH_TO_CONDAENV=diags
module purge
module load NCO/4.5.4-foss-2015a
module load CDO/1.7.2-foss-2015a
module load CDFTOOLS/3.0a8-foss-2015a
module load Miniconda2
set -xv
source ${PATH_TO_CONDAENV}/activate earthdiags
source activate diags
export PYTHONPATH=${PATH_TO_DIAGNOSTICS}:${PYTHONPATH}
cd ${PATH_TO_DIAGNOSTICS}/earthdiagnostics/
......
......@@ -26,7 +26,7 @@ setup(
keywords=['climate', 'weather', 'diagnostic'],
setup_requires=['pyproj'],
install_requires=['numpy', 'netCDF4', 'bscearth.utils', 'cdo>=1.3.4', 'nco>=0.0.3', 'iris>=1.12.0', 'coverage',
'pygrib', 'openpyxl', 'mock', 'futures', 'cf_units', 'cfunits', 'xxhash'],
'pygrib', 'openpyxl', 'mock', 'futures', 'cf_units', 'cfunits', 'xxhash', 'six', 'psutil'],
packages=find_packages(),
include_package_data=True,
scripts=['bin/earthdiags']
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment