Commit 3e793ea5 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Update doc and remove unneeded diagnsotics

parent ec577c43
...@@ -55,12 +55,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge ...@@ -55,12 +55,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge
Options: Options:
******** ********
1. Variable: 1. Domain:
Variable name
2. Domain:
Variable domain Variable domain
2. Variable:
Variable name
3. Original frequency: 3. Original frequency:
Original frequency to use Original frequency to use
...@@ -102,58 +102,19 @@ Calculates the monthly mean for a given variable. See :class:`~earthdiagnostics. ...@@ -102,58 +102,19 @@ Calculates the monthly mean for a given variable. See :class:`~earthdiagnostics.
Options: Options:
******** ********
1. Variable: 1. Domain:
Variable name
2. Domain:
Variable domain Variable domain
3. Original frequency = daily: 2. Variable:
Original frequency to use
4. Grid = '':
Variable grid. Only required in case that you want to use
interpolated data.
relink
~~~~~~
Regenerates the links created in the monthly_mean, daily_mean, etc folders
for a given varible.
See :class:`~earthdiagnostics.general.relink.Relink`
Options:
********
1. Variable:
Variable name Variable name
2. Domain: 3. Original frequency = daily:
Variable domain Original frequency to use
3. Move old =
True: If True, any data founded in the target directory will be moved
to another folder (called FOLDER_NAME_old) instead of deleted.
4. Grid = '': 4. Grid = '':
Variable grid. Only required in case that you want to use Variable grid. Only required in case that you want to use
interpolated data. interpolated data.
relinkall
~~~~~~~~~
Regenerates the links created in the monthly_mean, daily_mean, etc folders
for all variables
See :class:`~earthdiagnostics.general.relinkall.RelinkAll`
Options:
********
This diagnostic has no options
rewrite: rewrite:
~~~~~~~~ ~~~~~~~~
...@@ -229,8 +190,8 @@ Options: ...@@ -229,8 +190,8 @@ Options:
1. Domain: 1. Domain:
Variable domain Variable domain
2. Variable: 2. Variables:
Variable name List of variables to simplify, separated by ':'
5. Grid = '': 5. Grid = '':
Variable grid. Only required in case that you want to use interpolated data. Variable grid. Only required in case that you want to use interpolated data.
...@@ -247,12 +208,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge ...@@ -247,12 +208,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge
Options: Options:
******** ********
1. Variable: 1. Domain:
Variable name
2. Domain:
Variable domain Variable domain
2. Variable:
Variable name
3. Original frequency: 3. Original frequency:
Original frequency to use Original frequency to use
......
...@@ -17,11 +17,6 @@ from earthdiagnostics.publisher import Publisher ...@@ -17,11 +17,6 @@ from earthdiagnostics.publisher import Publisher
from earthdiagnostics.variable import VariableType from earthdiagnostics.variable import VariableType
import netCDF4 import netCDF4
<< << << < HEAD
== == == =
>>>>>> > Append regions without spaces
class LocalStatus(object): class LocalStatus(object):
......
# coding=utf-8
"""Create links for a variable"""
from earthdiagnostics.diagnostic import (
Diagnostic,
DiagnosticOption,
DiagnosticDomainOption,
DiagnosticBoolOption,
DiagnosticVariableOption,
)
class Relink(Diagnostic):
"""
Recreates the links for the variable specified
:original author: Javier Vegas-Regidor<javier.vegas@bsc.es>
:created: September 2016
:param data_manager: data management object
:type data_manager: DataManager
:param startdate: startdate
:type startdate: str
:param member: member number
:type member: int
:param chunk: chunk's number
:type chunk: int
:param variable: variable's name
:type variable: str
:param domain: variable's domain
:type domain: ModelingRealm
:param move_old: if true, looks for files following the old convention
and moves to avoid collisions
:type move_old: bool
"""
alias = "relink"
"Diagnostic alias for the configuration file"
def __init__(
self,
data_manager,
startdate,
member,
chunk,
domain,
variable,
move_old,
grid,
):
Diagnostic.__init__(self, data_manager)
self.startdate = startdate
self.member = member
self.chunk = chunk
self.variable = variable
self.domain = domain
self.move_old = move_old
self.grid = grid
self.var_manager = data_manager.config.var_manager
def __str__(self):
return (
"Relink output Startdate: {0.startdate} Member: {0.member} "
"Chunk: {0.chunk} Move old: {0.move_old} "
"Variable: {0.domain}:{0.variable} Grid: {0.grid}".format(self)
)
def __hash__(self):
return hash(str(self))
def __eq__(self, other):
if self._different_type(other):
return False
return (
self.startdate == other.startdate
and self.member == other.member
and self.chunk == other.chunk
and self.domain == other.domain
and self.variable == other.variable
and self.move_old == other.move_old
and self.grid == other.grid
)
@classmethod
def generate_jobs(cls, diags, options):
"""
Create a job for each chunk to compute the diagnostic
:param diags: Diagnostics manager class
:type diags: Diags
:param options: variable, domain, move_old=False
:type options: list[str]
:return:
"""
options_available = (
DiagnosticDomainOption(),
DiagnosticVariableOption(diags.data_manager.config.var_manager),
DiagnosticBoolOption("move_old", True),
DiagnosticOption("grid", ""),
)
options = cls.process_options(options, options_available)
job_list = list()
chunk_list = diags.config.experiment.get_chunk_list()
for startdate, member, chunk in chunk_list:
job_list.append(
Relink(
diags.data_manager,
startdate,
member,
chunk,
options["domain"],
options["variable"],
options["move_old"],
options["grid"],
)
)
return job_list
def request_data(self):
"""Request data required by the diagnostic"""
def declare_data_generated(self):
"""Declare data to be generated by the diagnostic"""
def compute(self):
"""Run the diagnostic"""
self.data_manager.link_file(
self.domain,
self.variable,
self.var_manager.get_variable(self.variable),
self.startdate,
self.member,
self.chunk,
move_old=self.move_old,
grid=self.grid,
)
# coding=utf-8
"""Create links for all variables in a startdate"""
from earthdiagnostics.diagnostic import Diagnostic
class RelinkAll(Diagnostic):
"""
Recreates the links for the variable specified
:original author: Javier Vegas-Regidor<javier.vegas@bsc.es>
:created: September 2016
:param data_manager: data management object
:type data_manager: DataManager
:param startdate: startdate
:type startdate: str
"""
alias = "relinkall"
"Diagnostic alias for the configuration file"
def __init__(self, data_manager, startdate):
Diagnostic.__init__(self, data_manager)
self.startdate = startdate
def __str__(self):
return "Relink all output Startdate: {0}".format(self.startdate)
def __hash__(self):
return hash(str(self))
def __eq__(self, other):
if self._different_type(other):
return False
return self.startdate == other.startdate
@classmethod
def generate_jobs(cls, diags, options):
"""
Create a job for each chunk to compute the diagnostic
:param diags: Diagnostics manager class
:type diags: Diags
:param options: variable, domain, move_old=False
:type options: list[str]
:return:
"""
if len(options) > 1:
raise Exception("The Relink All diagnostic has no options")
job_list = list()
for startdate in diags.config.experiment.startdates:
job_list.append(RelinkAll(diags.data_manager, startdate))
return job_list
def request_data(self):
"""Request data required by the diagnostic"""
def declare_data_generated(self):
"""Declare data to be generated by the diagnostic"""
def compute(self):
"""Run the diagnostic"""
self.data_manager.convention.create_links(self.startdate)
# coding=utf-8
"""Extract levels from variable"""
from earthdiagnostics.box import Box
from earthdiagnostics.diagnostic import (
Diagnostic,
DiagnosticOption,
DiagnosticDomainOption,
DiagnosticVariableListOption,
DiagnosticIntOption,
)
from earthdiagnostics.utils import Utils, TempFile
class SelectLevels(Diagnostic):
"""
Extract levels from file
Parameters
----------
data_manager: DataManager
startdate: str
member: int
chunk: init
domain: ModellingRealm
variable: str
grid: str or None
first_level: int
last_level: int
"""
alias = "selev"
"Diagnostic alias for the configuration file"
def __init__(
self,
data_manager,
startdate,
member,
chunk,
domain,
variable,
grid,
first_level,
last_level,
):
Diagnostic.__init__(self, data_manager)
self.startdate = startdate
self.member = member
self.chunk = chunk
self.variable = variable
self.domain = domain
self.grid = grid
self.box = Box(False)
self.box.min_depth = first_level
self.box.max_depth = last_level
self.variable_file = None
self.result = None
def __str__(self):
return (
"Select levels Startdate: {0.startdate} Member: {0.member} "
"Chunk: {0.chunk} Variable: {0.domain}:{0.variable} "
"Levels: {0.box.min_depth}-{0.box.max_depth} "
"Grid: {0.grid}".format(self)
)
def __eq__(self, other):
if self._different_type(other):
return False
return (
self.startdate == other.startdate
and self.member == other.member
and self.chunk == other.chunk
and self.domain == other.domain
and self.variable == other.variable
and self.box == other.box
and self.grid == self.grid
)
@classmethod
def generate_jobs(cls, diags, options):
"""
Create a job for each chunk to compute the diagnostic
:param diags: Diagnostics manager class
:type diags: Diags
:param options: domain,variables,grid
:type options: list[str]
:return:
"""
options_available = (
DiagnosticDomainOption(),
DiagnosticVariableListOption(
diags.data_manager.config.var_manager, "variables"
),
DiagnosticIntOption("first_level"),
DiagnosticIntOption("last_level"),
DiagnosticOption("grid", ""),
)
options = cls.process_options(options, options_available)
job_list = list()
variables = options["variables"]
for var in variables:
chunk_list = diags.config.experiment.get_chunk_list()
for startdate, member, chunk in chunk_list:
job_list.append(
SelectLevels(
diags.data_manager,
startdate,
member,
chunk,
options["domain"],
var,
options["grid"],
options["first_level"],
options["last_level"],
)
)
return job_list
def request_data(self):
"""Request data required by the diagnostic"""
self.variable_file = self.request_chunk(
self.domain,
self.variable,
self.startdate,
self.member,
self.chunk,
grid=self.grid,
to_modify=True,
)
def declare_data_generated(self):
"""Request data required by the diagnostic"""
self.result = self.declare_chunk(
self.domain,
self.variable,
self.startdate,
self.member,
self.chunk,
grid=self.grid,
)
def compute(self):
"""Run the diagnostic"""
temp = TempFile.get()
handler = Utils.open_cdf(self.variable_file.local_file)
var_name = ""
for var in ("lev", "plev"):
if var in handler.variables:
var_name = var
continue
handler.close()
Utils.nco().ncks(
input=self.variable_file.local_file,
output=temp,
options="-O -d {1},{0.min_depth},{0.max_depth}".format(
self.box, var_name
),
)
self.result.set_local_file(temp)
@staticmethod
def _create_var(var_name, var_values, source, destiny):
old_var = source.variables[var_name]
new_var = destiny.createVariable(
var_name, old_var.dtype, dimensions=(var_name,)
)
new_var[:] = var_values
Utils.copy_attributes(new_var, old_var)
vertices_name = "{0}_vertices".format(var_name)
if vertices_name in source.variables:
var_vertices = source.variables[vertices_name]
if var_name == "lon":
vertices_values = var_vertices[0:1, ...]
else:
vertices_values = var_vertices[:, 0:1, :]
new_lat_vertices = destiny.createVariable(
vertices_name,
var_vertices.dtype,
dimensions=(var_name, "vertices"),
)
new_lat_vertices[:] = vertices_values
Utils.copy_attributes(new_lat_vertices, var_vertices)
...@@ -362,8 +362,6 @@ class WorkManager(object): ...@@ -362,8 +362,6 @@ class WorkManager(object):
) )
from earthdiagnostics.general.module import Module from earthdiagnostics.general.module import Module
from earthdiagnostics.general.rewrite import Rewrite from earthdiagnostics.general.rewrite import Rewrite
from earthdiagnostics.general.relink import Relink
from earthdiagnostics.general.relinkall import RelinkAll
from earthdiagnostics.general.scale import Scale from earthdiagnostics.general.scale import Scale
from earthdiagnostics.general.verticalmeanmetersiris import ( from earthdiagnostics.general.verticalmeanmetersiris import (
VerticalMeanMetersIris, VerticalMeanMetersIris,
...@@ -376,8 +374,6 @@ class WorkManager(object): ...@@ -376,8 +374,6 @@ class WorkManager(object):
Diagnostic.register(MonthlyMean) Diagnostic.register(MonthlyMean)
Diagnostic.register(YearlyMean) Diagnostic.register(YearlyMean)
Diagnostic.register(Rewrite) Diagnostic.register(Rewrite)
Diagnostic.register(Relink)
Diagnostic.register(RelinkAll)
Diagnostic.register(Scale) Diagnostic.register(Scale)
Diagnostic.register(Attribute) Diagnostic.register(Attribute)
Diagnostic.register(Module) Diagnostic.register(Module)
......
# coding=utf-8
from unittest import TestCase
from earthdiagnostics.diagnostic import DiagnosticVariableOption
from earthdiagnostics.box import Box
from earthdiagnostics.general.relink import Relink
from mock import Mock, patch
from earthdiagnostics.modelingrealm import ModelingRealms
class TestRelink(TestCase):
def setUp(self):
self.data_manager = Mock()
self.diags = Mock()
self.diags.model_version = "model_version"
self.diags.config.experiment.get_chunk_list.return_value = (
("20010101", 0, 0),
("20010101", 0, 1),
)
self.box = Box()
self.box.min_depth = 0
self.box.max_depth = 100
def fake_parse(self, value):
return value
@patch.object(DiagnosticVariableOption, "parse", fake_parse)
def test_generate_jobs(self):
jobs = Relink.generate_jobs(self.diags, ["diagnostic", "atmos", "var"])
self.assertEqual(len(jobs), 2)
self.assertEqual(
jobs[0],
Relink(
self.data_manager,
"20010101",
0,
0,
ModelingRealms.atmos,
"var",