Commit 1598c138 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Merge branch 'master' into 'production'

Master

See merge request !50
parents 3cfddd94 6501c4b8
......@@ -705,13 +705,13 @@ class Cmorizer(object):
handler.physics_description = cmor.physics_description
handler.model_id = experiment.model
handler.associated_model = cmor.associated_model
handler.project_id = self.config.data_convention.upper()
handler.project_id = self.config.data_convention.name.upper()
handler.realization = str(self.member + 1)
handler.source = cmor.source
handler.startdate = 'S{0}'.format(self.startdate)
handler.tracking_id = str(uuid.uuid1())
handler.title = "{0} model output prepared for {2} {1}".format(experiment.model, experiment.experiment_name,
self.config.data_convention.upper())
self.config.data_convention.name.upper())
def _gribfiles_available(self):
grb_path = os.path.join(self.original_files_path, '*.grb')
......
......@@ -23,6 +23,18 @@ class Basin(object):
def __ne__(self, other):
return not self == other
def __gt__(self, other):
return self.name > other.name
def __lt__(self, other):
return self.name < other.name
def __ge__(self, other):
return self.name >= other.name
def __le__(self, other):
return self.name <= other.name
def __str__(self):
return self._name
......
......@@ -681,8 +681,8 @@ class NetCDFFile(DataFile):
def create_link(self):
"""Create a link from the original data in the <frequency>_<var_type> folder"""
try:
self.data_manager.create_link(self.domain, self.remote_file, self.frequency, self.final_name,
self.grid, True, self.var_type)
self.data_convention.create_link(self.domain, self.remote_file, self.frequency, self.final_name,
self.grid, True, self.var_type)
except (ValueError, Exception) as ex:
Log.error('Can not create link to {1}: {0}'.format(ex, self.remote_file))
......
......@@ -76,7 +76,7 @@ class ModelingRealm(object):
"""
if self.name == 'seaIce':
if data_convention in ('specs', 'preface'):
if data_convention.name in ('specs', 'preface'):
prefix = 'OI'
else:
prefix = 'SI'
......@@ -88,8 +88,8 @@ class ModelingRealm(object):
if frequency == Frequencies.six_hourly:
table_name = '6hrPlev'
else:
if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention not in ('specs',
'preface'):
if (frequency in (Frequencies.monthly, Frequencies.climatology)) or data_convention.name not in ('specs',
'preface'):
table_name = prefix + str(frequency)
else:
table_name = frequency.frequency
......
......@@ -79,12 +79,14 @@ class Siasiesiv(Diagnostic):
DiagnosticBoolOption('omit_volume', False))
options = cls.process_options(options, options_available)
if not options['basins']:
basins = options['basins']
if not basins:
Log.error('Basins not recognized')
return ()
masks = {}
for basin in options['basins']:
basins.sort()
for basin in basins:
masks[basin] = Utils.get_mask(basin)
job_list = list()
......@@ -121,12 +123,12 @@ class Siasiesiv(Diagnostic):
def _declare_var(self, var_name):
self.generated[var_name] = self.declare_chunk(ModelingRealms.seaIce, var_name,
self.startdate, self.member, self.chunk,
region=self.masks.keys())
self.startdate, self.member, self.chunk)
def compute(self):
"""Run the diagnostic"""
coordinates = ' '.join(('time', 'leadtime', 'time_centered', 'latitude', 'longitude'))
coordinates = ' '.join(('time', 'leadtime', 'time_centered',
self.data_convention.lon_name, self.data_convention.lat_name))
handler = Utils.open_cdf(self.sic.local_file)
handler.variables[self.sic_varname].coordinates = coordinates
handler.close()
......@@ -179,9 +181,9 @@ class Siasiesiv(Diagnostic):
results.append(result)
if not results:
continue
self._save_file(results.merge_cube(), self.generated[var], basins)
self._save_file(results.merge_cube(), self.generated[var])
def _save_file(self, data, generated_file, basins):
def _save_file(self, data, generated_file):
temp = TempFile.get()
region = data.coord('region').points
data.remove_coord('region')
......@@ -192,5 +194,4 @@ class Siasiesiv(Diagnostic):
var[...] = region
handler.close()
Utils.rename_variable(temp, 'region2', 'region', True)
generated_file.set_local_file(temp, region=basins.keys())
generated_file.set_local_file(temp)
# coding=utf-8
"""Earthdiagnostics workflow manager"""
import time
import datetime
import operator
import sys
......@@ -144,9 +145,9 @@ class WorkManager(object):
else:
step = 0.5
for _ in range(int(interval / step)):
if lock.acquire(blocking=False):
if lock.acquire(False):
return
sys.sleep(step)
time.sleep(step)
Log.info('Current status:')
Log.info('===============')
......
from earthdiagnostics.cmorizer import Cmorizer
from earthdiagnostics.utils import TempFile, Utils
from earthdiagnostics.data_convention import DataConvention
from bscearth.utils import log
from unittest import TestCase
from mock import Mock
from mock import Mock, create_autospec
import os
import tempfile
import shutil
......@@ -49,7 +50,8 @@ class TestCmorizer(TestCase):
self.data_manager.config.data_dir = os.path.join(self.tmp_dir, 'data')
self.data_manager.config.scratch_dir = os.path.join(self.tmp_dir, 'scratch')
TempFile.scratch_folder = self.data_manager.config.scratch_dir
self.data_manager.config.data_convention = 'data_convention'
self.data_manager.config.data_convention = create_autospec(DataConvention)
self.data_manager.config.data_convention.name = 'data_convention'
self.data_manager.config.var_manager.get_variable_and_alias = self._get_variable_and_alias
self.data_manager.config.var_manager.get_variable = self._get_variable
......
......@@ -19,6 +19,10 @@ class TestBasin(TestCase):
def test__str__(self):
self.assertEqual(str(self.basin), 'Basin')
def test_order(self):
self.assertTrue(self.basin < Basin('Vasin'))
self.assertTrue(self.basin > Basin('Asin'))
class TestBasins(TestCase):
......
......@@ -20,6 +20,8 @@ class TestModellingRealm(TestCase):
def setUp(self):
self.realm = ModelingRealm('ocean')
self.data_convention = Mock()
self.data_convention.name = 'specs'
def test__repr__(self):
self.assertEqual(str(self.realm), repr(self.realm))
......@@ -37,19 +39,20 @@ class TestModellingRealm(TestCase):
self.assertEqual(self.realm.get_table_name(Frequencies.monthly, 'specs'), 'Omon')
def test_get_oimon(self):
self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'specs'), 'OImon')
self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, self.data_convention), 'OImon')
def test_get_simon(self):
self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, 'cmip6'), 'SImon')
self.data_convention.name = 'cmip6'
self.assertEqual(ModelingRealm('seaIce').get_table_name(Frequencies.monthly, self.data_convention), 'SImon')
def test_get_limon(self):
self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, 'specs'), 'LImon')
self.assertEqual(ModelingRealm('landIce').get_table_name(Frequencies.monthly, self.data_convention), 'LImon')
def test_get_day(self):
self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, 'specs'), 'day')
self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.daily, self.data_convention), 'day')
def test_get_6hrplev(self):
self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, 'specs'), '6hrPlev')
self.assertEqual(ModelingRealm('atmos').get_table_name(Frequencies.six_hourly, self.data_convention), '6hrPlev')
def test_get_varfolder(self):
self.assertEqual(ModelingRealm('ocean').get_varfolder('var', 6, 3), 'var_f6h')
......
......@@ -184,27 +184,33 @@ class TestVariable(TestCase):
self.assertEqual(table.date, 'December 2013')
def test_get_table_added(self):
convention = Mock()
convention.name = 'specs'
var = Variable()
var.domain = ModelingRealms.atmos
var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013'))
table = var.get_table(Frequencies.monthly, 'specs')
table = var.get_table(Frequencies.monthly, convention)
self.assertEqual(table.frequency, Frequencies.monthly)
self.assertEqual(table.name, 'Amon')
self.assertEqual(table.date, 'December 2013')
def test_get_table_not_added(self):
convention = Mock()
convention.name = 'specs'
var = Variable()
var.domain = ModelingRealms.atmos
var.add_table(CMORTable('Amon', Frequencies.monthly, 'December 2013'))
table = var.get_table(Frequencies.daily, 'specs')
table = var.get_table(Frequencies.daily, convention)
self.assertEqual(table.frequency, Frequencies.daily)
self.assertEqual(table.name, 'day')
self.assertEqual(table.date, 'December 2013')
def test_get_table_not_matching(self):
convention = Mock()
convention.name = 'specs'
var = Variable()
with self.assertRaises(ValueError):
var.get_table(Frequencies.daily, 'specs')
var.get_table(Frequencies.daily, convention)
class TestVariableManager(TestCase):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment