diff --git a/doc/source/diagnostic_list.rst b/doc/source/diagnostic_list.rst index 1679fef9c01b338b5a4ce8604edded373418524f..de450a4f55f7b23e2695e6e1b91409d2452ade91 100644 --- a/doc/source/diagnostic_list.rst +++ b/doc/source/diagnostic_list.rst @@ -55,12 +55,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge Options: ******** -1. Variable: - Variable name - -2. Domain: +1. Domain: Variable domain +2. Variable: + Variable name + 3. Original frequency: Original frequency to use @@ -102,58 +102,19 @@ Calculates the monthly mean for a given variable. See :class:`~earthdiagnostics. Options: ******** -1. Variable: - Variable name - -2. Domain: +1. Domain: Variable domain -3. Original frequency = daily: - Original frequency to use - -4. Grid = '': - Variable grid. Only required in case that you want to use - interpolated data. - -relink -~~~~~~ - -Regenerates the links created in the monthly_mean, daily_mean, etc folders -for a given varible. - -See :class:`~earthdiagnostics.general.relink.Relink` - -Options: -******** - -1. Variable: +2. Variable: Variable name -2. Domain: - Variable domain - -3. Move old = - True: If True, any data founded in the target directory will be moved - to another folder (called FOLDER_NAME_old) instead of deleted. +3. Original frequency = daily: + Original frequency to use 4. Grid = '': Variable grid. Only required in case that you want to use interpolated data. - -relinkall -~~~~~~~~~ - -Regenerates the links created in the monthly_mean, daily_mean, etc folders -for all variables - -See :class:`~earthdiagnostics.general.relinkall.RelinkAll` - -Options: -******** - -This diagnostic has no options - rewrite: ~~~~~~~~ @@ -229,8 +190,8 @@ Options: 1. Domain: Variable domain -2. Variable: - Variable name +2. Variables: + List of variables to simplify, separated by ':' 5. Grid = '': Variable grid. Only required in case that you want to use interpolated data. @@ -247,12 +208,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge Options: ******** -1. Variable: - Variable name - -2. Domain: +1. Domain: Variable domain +2. Variable: + Variable name + 3. Original frequency: Original frequency to use diff --git a/earthdiagnostics/datafile.py b/earthdiagnostics/datafile.py index ec70195b076821cc5baf5295e10d3bdebf6726c3..d29956c48ea4597cb83cc6c54effc2cac34397b7 100644 --- a/earthdiagnostics/datafile.py +++ b/earthdiagnostics/datafile.py @@ -17,11 +17,6 @@ from earthdiagnostics.publisher import Publisher from earthdiagnostics.variable import VariableType import netCDF4 -<< << << < HEAD - -== == == = ->>>>>> > Append regions without spaces - class LocalStatus(object): diff --git a/earthdiagnostics/general/relink.py b/earthdiagnostics/general/relink.py deleted file mode 100644 index bb277ce3ea23cda947feb14ec43bc4e442578c8c..0000000000000000000000000000000000000000 --- a/earthdiagnostics/general/relink.py +++ /dev/null @@ -1,136 +0,0 @@ -# coding=utf-8 -"""Create links for a variable""" -from earthdiagnostics.diagnostic import ( - Diagnostic, - DiagnosticOption, - DiagnosticDomainOption, - DiagnosticBoolOption, - DiagnosticVariableOption, -) - - -class Relink(Diagnostic): - """ - Recreates the links for the variable specified - - :original author: Javier Vegas-Regidor - - :created: September 2016 - - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - :param member: member number - :type member: int - :param chunk: chunk's number - :type chunk: int - :param variable: variable's name - :type variable: str - :param domain: variable's domain - :type domain: ModelingRealm - :param move_old: if true, looks for files following the old convention - and moves to avoid collisions - :type move_old: bool - """ - - alias = "relink" - "Diagnostic alias for the configuration file" - - def __init__( - self, - data_manager, - startdate, - member, - chunk, - domain, - variable, - move_old, - grid, - ): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.move_old = move_old - self.grid = grid - self.var_manager = data_manager.config.var_manager - - def __str__(self): - return ( - "Relink output Startdate: {0.startdate} Member: {0.member} " - "Chunk: {0.chunk} Move old: {0.move_old} " - "Variable: {0.domain}:{0.variable} Grid: {0.grid}".format(self) - ) - - def __hash__(self): - return hash(str(self)) - - def __eq__(self, other): - if self._different_type(other): - return False - return ( - self.startdate == other.startdate - and self.member == other.member - and self.chunk == other.chunk - and self.domain == other.domain - and self.variable == other.variable - and self.move_old == other.move_old - and self.grid == other.grid - ) - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, move_old=False - :type options: list[str] - :return: - """ - options_available = ( - DiagnosticDomainOption(), - DiagnosticVariableOption(diags.data_manager.config.var_manager), - DiagnosticBoolOption("move_old", True), - DiagnosticOption("grid", ""), - ) - options = cls.process_options(options, options_available) - job_list = list() - chunk_list = diags.config.experiment.get_chunk_list() - for startdate, member, chunk in chunk_list: - job_list.append( - Relink( - diags.data_manager, - startdate, - member, - chunk, - options["domain"], - options["variable"], - options["move_old"], - options["grid"], - ) - ) - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - - def compute(self): - """Run the diagnostic""" - self.data_manager.link_file( - self.domain, - self.variable, - self.var_manager.get_variable(self.variable), - self.startdate, - self.member, - self.chunk, - move_old=self.move_old, - grid=self.grid, - ) diff --git a/earthdiagnostics/general/relinkall.py b/earthdiagnostics/general/relinkall.py deleted file mode 100644 index 107fa290068fa68c5adcee0348539b98637b9031..0000000000000000000000000000000000000000 --- a/earthdiagnostics/general/relinkall.py +++ /dev/null @@ -1,64 +0,0 @@ -# coding=utf-8 -"""Create links for all variables in a startdate""" -from earthdiagnostics.diagnostic import Diagnostic - - -class RelinkAll(Diagnostic): - """ - Recreates the links for the variable specified - - :original author: Javier Vegas-Regidor - - :created: September 2016 - - :param data_manager: data management object - :type data_manager: DataManager - :param startdate: startdate - :type startdate: str - """ - - alias = "relinkall" - "Diagnostic alias for the configuration file" - - def __init__(self, data_manager, startdate): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - - def __str__(self): - return "Relink all output Startdate: {0}".format(self.startdate) - - def __hash__(self): - return hash(str(self)) - - def __eq__(self, other): - if self._different_type(other): - return False - return self.startdate == other.startdate - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: variable, domain, move_old=False - :type options: list[str] - :return: - """ - if len(options) > 1: - raise Exception("The Relink All diagnostic has no options") - job_list = list() - for startdate in diags.config.experiment.startdates: - job_list.append(RelinkAll(diags.data_manager, startdate)) - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - - def declare_data_generated(self): - """Declare data to be generated by the diagnostic""" - - def compute(self): - """Run the diagnostic""" - self.data_manager.convention.create_links(self.startdate) diff --git a/earthdiagnostics/general/select_levels.py b/earthdiagnostics/general/select_levels.py deleted file mode 100644 index a5c99a4fcda2830e0e0e047d5f3e79da126e8b32..0000000000000000000000000000000000000000 --- a/earthdiagnostics/general/select_levels.py +++ /dev/null @@ -1,188 +0,0 @@ -# coding=utf-8 -"""Extract levels from variable""" -from earthdiagnostics.box import Box -from earthdiagnostics.diagnostic import ( - Diagnostic, - DiagnosticOption, - DiagnosticDomainOption, - DiagnosticVariableListOption, - DiagnosticIntOption, -) -from earthdiagnostics.utils import Utils, TempFile - - -class SelectLevels(Diagnostic): - """ - Extract levels from file - - Parameters - ---------- - data_manager: DataManager - startdate: str - member: int - chunk: init - domain: ModellingRealm - variable: str - grid: str or None - first_level: int - last_level: int - """ - - alias = "selev" - "Diagnostic alias for the configuration file" - - def __init__( - self, - data_manager, - startdate, - member, - chunk, - domain, - variable, - grid, - first_level, - last_level, - ): - Diagnostic.__init__(self, data_manager) - self.startdate = startdate - self.member = member - self.chunk = chunk - self.variable = variable - self.domain = domain - self.grid = grid - self.box = Box(False) - self.box.min_depth = first_level - self.box.max_depth = last_level - - self.variable_file = None - self.result = None - - def __str__(self): - return ( - "Select levels Startdate: {0.startdate} Member: {0.member} " - "Chunk: {0.chunk} Variable: {0.domain}:{0.variable} " - "Levels: {0.box.min_depth}-{0.box.max_depth} " - "Grid: {0.grid}".format(self) - ) - - def __eq__(self, other): - if self._different_type(other): - return False - return ( - self.startdate == other.startdate - and self.member == other.member - and self.chunk == other.chunk - and self.domain == other.domain - and self.variable == other.variable - and self.box == other.box - and self.grid == self.grid - ) - - @classmethod - def generate_jobs(cls, diags, options): - """ - Create a job for each chunk to compute the diagnostic - - :param diags: Diagnostics manager class - :type diags: Diags - :param options: domain,variables,grid - :type options: list[str] - :return: - """ - options_available = ( - DiagnosticDomainOption(), - DiagnosticVariableListOption( - diags.data_manager.config.var_manager, "variables" - ), - DiagnosticIntOption("first_level"), - DiagnosticIntOption("last_level"), - DiagnosticOption("grid", ""), - ) - options = cls.process_options(options, options_available) - job_list = list() - variables = options["variables"] - for var in variables: - chunk_list = diags.config.experiment.get_chunk_list() - for startdate, member, chunk in chunk_list: - job_list.append( - SelectLevels( - diags.data_manager, - startdate, - member, - chunk, - options["domain"], - var, - options["grid"], - options["first_level"], - options["last_level"], - ) - ) - return job_list - - def request_data(self): - """Request data required by the diagnostic""" - self.variable_file = self.request_chunk( - self.domain, - self.variable, - self.startdate, - self.member, - self.chunk, - grid=self.grid, - to_modify=True, - ) - - def declare_data_generated(self): - """Request data required by the diagnostic""" - self.result = self.declare_chunk( - self.domain, - self.variable, - self.startdate, - self.member, - self.chunk, - grid=self.grid, - ) - - def compute(self): - """Run the diagnostic""" - temp = TempFile.get() - handler = Utils.open_cdf(self.variable_file.local_file) - var_name = "" - for var in ("lev", "plev"): - if var in handler.variables: - var_name = var - continue - handler.close() - - Utils.nco().ncks( - input=self.variable_file.local_file, - output=temp, - options="-O -d {1},{0.min_depth},{0.max_depth}".format( - self.box, var_name - ), - ) - self.result.set_local_file(temp) - - @staticmethod - def _create_var(var_name, var_values, source, destiny): - old_var = source.variables[var_name] - new_var = destiny.createVariable( - var_name, old_var.dtype, dimensions=(var_name,) - ) - new_var[:] = var_values - Utils.copy_attributes(new_var, old_var) - - vertices_name = "{0}_vertices".format(var_name) - - if vertices_name in source.variables: - var_vertices = source.variables[vertices_name] - if var_name == "lon": - vertices_values = var_vertices[0:1, ...] - else: - vertices_values = var_vertices[:, 0:1, :] - new_lat_vertices = destiny.createVariable( - vertices_name, - var_vertices.dtype, - dimensions=(var_name, "vertices"), - ) - new_lat_vertices[:] = vertices_values - Utils.copy_attributes(new_lat_vertices, var_vertices) diff --git a/earthdiagnostics/work_manager.py b/earthdiagnostics/work_manager.py index 048388f740d18a03544c9e13cba191a4a4c0b861..c46821fbcdea2453c3c4d1c78ab64051da7e38ed 100644 --- a/earthdiagnostics/work_manager.py +++ b/earthdiagnostics/work_manager.py @@ -362,8 +362,6 @@ class WorkManager(object): ) from earthdiagnostics.general.module import Module from earthdiagnostics.general.rewrite import Rewrite - from earthdiagnostics.general.relink import Relink - from earthdiagnostics.general.relinkall import RelinkAll from earthdiagnostics.general.scale import Scale from earthdiagnostics.general.verticalmeanmetersiris import ( VerticalMeanMetersIris, @@ -376,8 +374,6 @@ class WorkManager(object): Diagnostic.register(MonthlyMean) Diagnostic.register(YearlyMean) Diagnostic.register(Rewrite) - Diagnostic.register(Relink) - Diagnostic.register(RelinkAll) Diagnostic.register(Scale) Diagnostic.register(Attribute) Diagnostic.register(Module) diff --git a/test/unit/general/test_relink.py b/test/unit/general/test_relink.py deleted file mode 100644 index f3c87dcab7a74163e1a1e33c843bdbaa47993b66..0000000000000000000000000000000000000000 --- a/test/unit/general/test_relink.py +++ /dev/null @@ -1,147 +0,0 @@ -# coding=utf-8 -from unittest import TestCase - -from earthdiagnostics.diagnostic import DiagnosticVariableOption -from earthdiagnostics.box import Box -from earthdiagnostics.general.relink import Relink -from mock import Mock, patch - -from earthdiagnostics.modelingrealm import ModelingRealms - - -class TestRelink(TestCase): - def setUp(self): - self.data_manager = Mock() - - self.diags = Mock() - self.diags.model_version = "model_version" - self.diags.config.experiment.get_chunk_list.return_value = ( - ("20010101", 0, 0), - ("20010101", 0, 1), - ) - - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 - - def fake_parse(self, value): - return value - - @patch.object(DiagnosticVariableOption, "parse", fake_parse) - def test_generate_jobs(self): - - jobs = Relink.generate_jobs(self.diags, ["diagnostic", "atmos", "var"]) - self.assertEqual(len(jobs), 2) - self.assertEqual( - jobs[0], - Relink( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.atmos, - "var", - True, - "", - ), - ) - self.assertEqual( - jobs[1], - Relink( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.atmos, - "var", - True, - "", - ), - ) - - jobs = Relink.generate_jobs( - self.diags, ["diagnostic", "seaIce", "var", "False"] - ) - self.assertEqual(len(jobs), 2) - self.assertEqual( - jobs[0], - Relink( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.seaIce, - "var", - False, - "", - ), - ) - self.assertEqual( - jobs[1], - Relink( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.seaIce, - "var", - False, - "", - ), - ) - - jobs = Relink.generate_jobs( - self.diags, ["diagnostic", "ocean", "var", "True", "grid"] - ) - self.assertEqual(len(jobs), 2) - self.assertEqual( - jobs[0], - Relink( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.ocean, - "var", - True, - "grid", - ), - ) - self.assertEqual( - jobs[1], - Relink( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.ocean, - "var", - True, - "grid", - ), - ) - - with self.assertRaises(Exception): - Relink.generate_jobs(self.diags, ["diagnostic"]) - - with self.assertRaises(Exception): - Relink.generate_jobs( - self.diags, ["diagnostic", "0", "0", "0", "0", "0", "0", "0"] - ) - - def test_str(self): - mixed = Relink( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.ocean, - "var", - True, - "grid", - ) - self.assertEqual( - str(mixed), - "Relink output Startdate: 20010101 Member: 0 Chunk: 0 " - "Move old: True Variable: ocean:var Grid: grid", - ) diff --git a/test/unit/general/test_relinkall.py b/test/unit/general/test_relinkall.py deleted file mode 100644 index b49595e0ab00a0f20643133bbed0f5c113f29a14..0000000000000000000000000000000000000000 --- a/test/unit/general/test_relinkall.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding=utf-8 -from unittest import TestCase - -from earthdiagnostics.diagnostic import DiagnosticVariableOption -from earthdiagnostics.box import Box -from earthdiagnostics.general.relinkall import RelinkAll -from mock import Mock, patch - - -class TestRelinkAll(TestCase): - def setUp(self): - self.data_manager = Mock() - - self.diags = Mock() - self.diags.config.experiment.get_chunk_list.return_value = ( - ("20010101", 0, 0), - ("20010101", 0, 1), - ) - self.diags.config.experiment.startdates = [ - "20010101", - ] - - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 - - def fake_parse(self, value): - return value - - @patch.object(DiagnosticVariableOption, "parse", fake_parse) - def test_generate_jobs(self): - jobs = RelinkAll.generate_jobs(self.diags, ["diagnostic"]) - self.assertEqual(len(jobs), 1) - self.assertEqual(jobs[0], RelinkAll(self.data_manager, "20010101")) - - with self.assertRaises(Exception): - RelinkAll.generate_jobs(self.diags, ["diagnostic", "0"]) - - def test_str(self): - mixed = RelinkAll(self.data_manager, "20010101") - self.assertEqual(str(mixed), "Relink all output Startdate: 20010101") diff --git a/test/unit/general/test_select_levels.py b/test/unit/general/test_select_levels.py deleted file mode 100644 index f05b0a2f70abbf11846958d128604c3248661e88..0000000000000000000000000000000000000000 --- a/test/unit/general/test_select_levels.py +++ /dev/null @@ -1,198 +0,0 @@ -# coding=utf-8 -from unittest import TestCase -import os -from tempfile import mktemp - -from earthdiagnostics.diagnostic import ( - DiagnosticVariableListOption, - DiagnosticOptionError, -) -from earthdiagnostics.box import Box -from earthdiagnostics.general.select_levels import SelectLevels -from earthdiagnostics.frequency import Frequencies -from mock import Mock, patch - -from earthdiagnostics.modelingrealm import ModelingRealms - - -class TestSelectLevels(TestCase): - def setUp(self): - self.data_manager = Mock() - - self.diags = Mock() - self.diags.config.experiment.get_chunk_list.return_value = ( - ("20010101", 0, 0), - ("20010101", 0, 1), - ) - self.diags.config.experiment.startdates = [ - "20010101", - ] - self.diags.config.frequency = Frequencies.monthly - - self.box = Box() - self.box.min_depth = 0 - self.box.max_depth = 100 - self.var_file = mktemp(".nc") - - def tearDown(self): - if os.path.exists(self.var_file): - os.remove(self.var_file) - - def fake_parse(self, value): - return value.split("-") - - @patch.object(DiagnosticVariableListOption, "parse", fake_parse) - def test_generate_jobs(self): - jobs = SelectLevels.generate_jobs( - self.diags, ["diagnostic", "atmos", "var", "0", "20"] - ) - self.assertEqual(len(jobs), 2) - self.assertEqual( - jobs[0], - SelectLevels( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.atmos, - "var", - "", - 0, - 20, - ), - ) - self.assertEqual( - jobs[1], - SelectLevels( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.atmos, - "var", - "", - 0, - 20, - ), - ) - - jobs = SelectLevels.generate_jobs( - self.diags, ["diagnostic", "atmos", "var1-var2", "0", "20"] - ) - self.assertEqual(len(jobs), 4) - self.assertEqual( - jobs[0], - SelectLevels( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.atmos, - "var1", - "", - 0, - 20, - ), - ) - self.assertEqual( - jobs[1], - SelectLevels( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.atmos, - "var1", - "", - 0, - 20, - ), - ) - self.assertEqual( - jobs[2], - SelectLevels( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.atmos, - "var2", - "", - 0, - 20, - ), - ) - self.assertEqual( - jobs[3], - SelectLevels( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.atmos, - "var2", - "", - 0, - 20, - ), - ) - - jobs = SelectLevels.generate_jobs( - self.diags, ["diagnostic", "atmos", "var", "0", "20", "grid"] - ) - self.assertEqual(len(jobs), 2) - self.assertEqual( - jobs[0], - SelectLevels( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.atmos, - "var", - "grid", - 0, - 20, - ), - ) - self.assertEqual( - jobs[1], - SelectLevels( - self.data_manager, - "20010101", - 0, - 1, - ModelingRealms.atmos, - "var", - "grid", - 0, - 20, - ), - ) - - with self.assertRaises(DiagnosticOptionError): - SelectLevels.generate_jobs(self.diags, ["diagnostic"]) - - with self.assertRaises(DiagnosticOptionError): - SelectLevels.generate_jobs( - self.diags, - ["diagnostic", "atmos", "var", "0", "20", "grid", "extra"], - ) - - def test_str(self): - select = SelectLevels( - self.data_manager, - "20010101", - 0, - 0, - ModelingRealms.atmos, - "var", - "grid", - 0, - 20, - ) - self.assertEqual( - str(select), - "Select levels Startdate: 20010101 Member: 0 Chunk: 0 " - "Variable: atmos:var Levels: 0-20 Grid: grid", - )