Commit 3e793ea5 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Update doc and remove unneeded diagnsotics

parent ec577c43
......@@ -55,12 +55,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge
Options:
********
1. Variable:
Variable name
2. Domain:
1. Domain:
Variable domain
2. Variable:
Variable name
3. Original frequency:
Original frequency to use
......@@ -102,58 +102,19 @@ Calculates the monthly mean for a given variable. See :class:`~earthdiagnostics.
Options:
********
1. Variable:
Variable name
2. Domain:
1. Domain:
Variable domain
3. Original frequency = daily:
Original frequency to use
4. Grid = '':
Variable grid. Only required in case that you want to use
interpolated data.
relink
~~~~~~
Regenerates the links created in the monthly_mean, daily_mean, etc folders
for a given varible.
See :class:`~earthdiagnostics.general.relink.Relink`
Options:
********
1. Variable:
2. Variable:
Variable name
2. Domain:
Variable domain
3. Move old =
True: If True, any data founded in the target directory will be moved
to another folder (called FOLDER_NAME_old) instead of deleted.
3. Original frequency = daily:
Original frequency to use
4. Grid = '':
Variable grid. Only required in case that you want to use
interpolated data.
relinkall
~~~~~~~~~
Regenerates the links created in the monthly_mean, daily_mean, etc folders
for all variables
See :class:`~earthdiagnostics.general.relinkall.RelinkAll`
Options:
********
This diagnostic has no options
rewrite:
~~~~~~~~
......@@ -229,8 +190,8 @@ Options:
1. Domain:
Variable domain
2. Variable:
Variable name
2. Variables:
List of variables to simplify, separated by ':'
5. Grid = '':
Variable grid. Only required in case that you want to use interpolated data.
......@@ -247,12 +208,12 @@ Calculates the daily mean for a given variable. See :class:`~earthdiagnostics.ge
Options:
********
1. Variable:
Variable name
2. Domain:
1. Domain:
Variable domain
2. Variable:
Variable name
3. Original frequency:
Original frequency to use
......
......@@ -17,11 +17,6 @@ from earthdiagnostics.publisher import Publisher
from earthdiagnostics.variable import VariableType
import netCDF4
<< << << < HEAD
== == == =
>>>>>> > Append regions without spaces
class LocalStatus(object):
......
# coding=utf-8
"""Create links for a variable"""
from earthdiagnostics.diagnostic import (
Diagnostic,
DiagnosticOption,
DiagnosticDomainOption,
DiagnosticBoolOption,
DiagnosticVariableOption,
)
class Relink(Diagnostic):
"""
Recreates the links for the variable specified
:original author: Javier Vegas-Regidor<javier.vegas@bsc.es>
:created: September 2016
:param data_manager: data management object
:type data_manager: DataManager
:param startdate: startdate
:type startdate: str
:param member: member number
:type member: int
:param chunk: chunk's number
:type chunk: int
:param variable: variable's name
:type variable: str
:param domain: variable's domain
:type domain: ModelingRealm
:param move_old: if true, looks for files following the old convention
and moves to avoid collisions
:type move_old: bool
"""
alias = "relink"
"Diagnostic alias for the configuration file"
def __init__(
self,
data_manager,
startdate,
member,
chunk,
domain,
variable,
move_old,
grid,
):
Diagnostic.__init__(self, data_manager)
self.startdate = startdate
self.member = member
self.chunk = chunk
self.variable = variable
self.domain = domain
self.move_old = move_old
self.grid = grid
self.var_manager = data_manager.config.var_manager
def __str__(self):
return (
"Relink output Startdate: {0.startdate} Member: {0.member} "
"Chunk: {0.chunk} Move old: {0.move_old} "
"Variable: {0.domain}:{0.variable} Grid: {0.grid}".format(self)
)
def __hash__(self):
return hash(str(self))
def __eq__(self, other):
if self._different_type(other):
return False
return (
self.startdate == other.startdate
and self.member == other.member
and self.chunk == other.chunk
and self.domain == other.domain
and self.variable == other.variable
and self.move_old == other.move_old
and self.grid == other.grid
)
@classmethod
def generate_jobs(cls, diags, options):
"""
Create a job for each chunk to compute the diagnostic
:param diags: Diagnostics manager class
:type diags: Diags
:param options: variable, domain, move_old=False
:type options: list[str]
:return:
"""
options_available = (
DiagnosticDomainOption(),
DiagnosticVariableOption(diags.data_manager.config.var_manager),
DiagnosticBoolOption("move_old", True),
DiagnosticOption("grid", ""),
)
options = cls.process_options(options, options_available)
job_list = list()
chunk_list = diags.config.experiment.get_chunk_list()
for startdate, member, chunk in chunk_list:
job_list.append(
Relink(
diags.data_manager,
startdate,
member,
chunk,
options["domain"],
options["variable"],
options["move_old"],
options["grid"],
)
)
return job_list
def request_data(self):
"""Request data required by the diagnostic"""
def declare_data_generated(self):
"""Declare data to be generated by the diagnostic"""
def compute(self):
"""Run the diagnostic"""
self.data_manager.link_file(
self.domain,
self.variable,
self.var_manager.get_variable(self.variable),
self.startdate,
self.member,
self.chunk,
move_old=self.move_old,
grid=self.grid,
)
# coding=utf-8
"""Create links for all variables in a startdate"""
from earthdiagnostics.diagnostic import Diagnostic
class RelinkAll(Diagnostic):
"""
Recreates the links for the variable specified
:original author: Javier Vegas-Regidor<javier.vegas@bsc.es>
:created: September 2016
:param data_manager: data management object
:type data_manager: DataManager
:param startdate: startdate
:type startdate: str
"""
alias = "relinkall"
"Diagnostic alias for the configuration file"
def __init__(self, data_manager, startdate):
Diagnostic.__init__(self, data_manager)
self.startdate = startdate
def __str__(self):
return "Relink all output Startdate: {0}".format(self.startdate)
def __hash__(self):
return hash(str(self))
def __eq__(self, other):
if self._different_type(other):
return False
return self.startdate == other.startdate
@classmethod
def generate_jobs(cls, diags, options):
"""
Create a job for each chunk to compute the diagnostic
:param diags: Diagnostics manager class
:type diags: Diags
:param options: variable, domain, move_old=False
:type options: list[str]
:return:
"""
if len(options) > 1:
raise Exception("The Relink All diagnostic has no options")
job_list = list()
for startdate in diags.config.experiment.startdates:
job_list.append(RelinkAll(diags.data_manager, startdate))
return job_list
def request_data(self):
"""Request data required by the diagnostic"""
def declare_data_generated(self):
"""Declare data to be generated by the diagnostic"""
def compute(self):
"""Run the diagnostic"""
self.data_manager.convention.create_links(self.startdate)
# coding=utf-8
"""Extract levels from variable"""
from earthdiagnostics.box import Box
from earthdiagnostics.diagnostic import (
Diagnostic,
DiagnosticOption,
DiagnosticDomainOption,
DiagnosticVariableListOption,
DiagnosticIntOption,
)
from earthdiagnostics.utils import Utils, TempFile
class SelectLevels(Diagnostic):
"""
Extract levels from file
Parameters
----------
data_manager: DataManager
startdate: str
member: int
chunk: init
domain: ModellingRealm
variable: str
grid: str or None
first_level: int
last_level: int
"""
alias = "selev"
"Diagnostic alias for the configuration file"
def __init__(
self,
data_manager,
startdate,
member,
chunk,
domain,
variable,
grid,
first_level,
last_level,
):
Diagnostic.__init__(self, data_manager)
self.startdate = startdate
self.member = member
self.chunk = chunk
self.variable = variable
self.domain = domain
self.grid = grid
self.box = Box(False)
self.box.min_depth = first_level
self.box.max_depth = last_level
self.variable_file = None
self.result = None
def __str__(self):
return (
"Select levels Startdate: {0.startdate} Member: {0.member} "
"Chunk: {0.chunk} Variable: {0.domain}:{0.variable} "
"Levels: {0.box.min_depth}-{0.box.max_depth} "
"Grid: {0.grid}".format(self)
)
def __eq__(self, other):
if self._different_type(other):
return False
return (
self.startdate == other.startdate
and self.member == other.member
and self.chunk == other.chunk
and self.domain == other.domain
and self.variable == other.variable
and self.box == other.box
and self.grid == self.grid
)
@classmethod
def generate_jobs(cls, diags, options):
"""
Create a job for each chunk to compute the diagnostic
:param diags: Diagnostics manager class
:type diags: Diags
:param options: domain,variables,grid
:type options: list[str]
:return:
"""
options_available = (
DiagnosticDomainOption(),
DiagnosticVariableListOption(
diags.data_manager.config.var_manager, "variables"
),
DiagnosticIntOption("first_level"),
DiagnosticIntOption("last_level"),
DiagnosticOption("grid", ""),
)
options = cls.process_options(options, options_available)
job_list = list()
variables = options["variables"]
for var in variables:
chunk_list = diags.config.experiment.get_chunk_list()
for startdate, member, chunk in chunk_list:
job_list.append(
SelectLevels(
diags.data_manager,
startdate,
member,
chunk,
options["domain"],
var,
options["grid"],
options["first_level"],
options["last_level"],
)
)
return job_list
def request_data(self):
"""Request data required by the diagnostic"""
self.variable_file = self.request_chunk(
self.domain,
self.variable,
self.startdate,
self.member,
self.chunk,
grid=self.grid,
to_modify=True,
)
def declare_data_generated(self):
"""Request data required by the diagnostic"""
self.result = self.declare_chunk(
self.domain,
self.variable,
self.startdate,
self.member,
self.chunk,
grid=self.grid,
)
def compute(self):
"""Run the diagnostic"""
temp = TempFile.get()
handler = Utils.open_cdf(self.variable_file.local_file)
var_name = ""
for var in ("lev", "plev"):
if var in handler.variables:
var_name = var
continue
handler.close()
Utils.nco().ncks(
input=self.variable_file.local_file,
output=temp,
options="-O -d {1},{0.min_depth},{0.max_depth}".format(
self.box, var_name
),
)
self.result.set_local_file(temp)
@staticmethod
def _create_var(var_name, var_values, source, destiny):
old_var = source.variables[var_name]
new_var = destiny.createVariable(
var_name, old_var.dtype, dimensions=(var_name,)
)
new_var[:] = var_values
Utils.copy_attributes(new_var, old_var)
vertices_name = "{0}_vertices".format(var_name)
if vertices_name in source.variables:
var_vertices = source.variables[vertices_name]
if var_name == "lon":
vertices_values = var_vertices[0:1, ...]
else:
vertices_values = var_vertices[:, 0:1, :]
new_lat_vertices = destiny.createVariable(
vertices_name,
var_vertices.dtype,
dimensions=(var_name, "vertices"),
)
new_lat_vertices[:] = vertices_values
Utils.copy_attributes(new_lat_vertices, var_vertices)
......@@ -362,8 +362,6 @@ class WorkManager(object):
)
from earthdiagnostics.general.module import Module
from earthdiagnostics.general.rewrite import Rewrite
from earthdiagnostics.general.relink import Relink
from earthdiagnostics.general.relinkall import RelinkAll
from earthdiagnostics.general.scale import Scale
from earthdiagnostics.general.verticalmeanmetersiris import (
VerticalMeanMetersIris,
......@@ -376,8 +374,6 @@ class WorkManager(object):
Diagnostic.register(MonthlyMean)
Diagnostic.register(YearlyMean)
Diagnostic.register(Rewrite)
Diagnostic.register(Relink)
Diagnostic.register(RelinkAll)
Diagnostic.register(Scale)
Diagnostic.register(Attribute)
Diagnostic.register(Module)
......
# coding=utf-8
from unittest import TestCase
from earthdiagnostics.diagnostic import DiagnosticVariableOption
from earthdiagnostics.box import Box
from earthdiagnostics.general.relink import Relink
from mock import Mock, patch
from earthdiagnostics.modelingrealm import ModelingRealms
class TestRelink(TestCase):
def setUp(self):
self.data_manager = Mock()
self.diags = Mock()
self.diags.model_version = "model_version"
self.diags.config.experiment.get_chunk_list.return_value = (
("20010101", 0, 0),
("20010101", 0, 1),
)
self.box = Box()
self.box.min_depth = 0
self.box.max_depth = 100
def fake_parse(self, value):
return value
@patch.object(DiagnosticVariableOption, "parse", fake_parse)
def test_generate_jobs(self):
jobs = Relink.generate_jobs(self.diags, ["diagnostic", "atmos", "var"])
self.assertEqual(len(jobs), 2)
self.assertEqual(
jobs[0],
Relink(
self.data_manager,
"20010101",
0,
0,
ModelingRealms.atmos,
"var",
True,
"",
),
)
self.assertEqual(
jobs[1],
Relink(
self.data_manager,
"20010101",
0,
1,
ModelingRealms.atmos,
"var",
True,
"",
),
)
jobs = Relink.generate_jobs(
self.diags, ["diagnostic", "seaIce", "var", "False"]
)
self.assertEqual(len(jobs), 2)
self.assertEqual(
jobs[0],
Relink(
self.data_manager,
"20010101",
0,
0,
ModelingRealms.seaIce,
"var",
False,
"",
),
)
self.assertEqual(
jobs[1],