Commit f5c74aa7 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Merge branch 'master' into 'production'

Master

See merge request !53
parents 638dd6e9 a2b61e20
......@@ -5,9 +5,6 @@
engines:
coverage:
enabled: true
exclude_paths: [
'tests',
]
metrics:
enabled: true
duplication:
......@@ -20,6 +17,5 @@ engines:
exclude_paths: [
'doc/**',
'test/**',
'earthdiagnostics/cmor_tables/**',
]
[run]
branch = True
source = earthdiagnostics
source = earthdiagnostics test
[html]
title = Coverage report for EarthDiagnostics
......
Variable,Shortname,Name,Long name,Domain,Basin,Units,Valid min,Valid max,Grid,Tables
hfxout,hfxout,,Ocean surface heat flux,ocean,,W m-2,,,,,
iiceages:siage:iice_otd,ageice,age_of_sea_ice,Age of sea ice,seaIce,,,,,,
al,al,surface_albedo,Albedo,atmos,,,,,,
bgfrcsal,bgfrcsal,change_over_time_in_heat_content_from_forcing,Change over time in salt content from forcing,ocean,,,,,,
......@@ -345,4 +346,4 @@ zqsb,hfsso,surface_downward_sensible_heat_flux,Surface Downward Sensible Heat Fl
zqlw,rlntds,surface_net_downward_longwave_flux,Surface Net Downward Longwave Radiation,ocean,,W m-2,,,,
var78,tclw,total_column_liquid_water,Total column liquid water,atmos,,kg m-2,,,,
var79,tciw,total_column_ice_water,Total column ice water,atmos,,kg m-2,,,,
rho,rhopoto,sea_water_potential_density,Sea Water Potential Density,ocean,,kg m-3,,,,
\ No newline at end of file
rho,rhopoto,sea_water_potential_density,Sea Water Potential Density,ocean,,kg m-3,,,,
......@@ -50,23 +50,21 @@ class Cmorizer(object):
def __init__(self, data_manager, startdate, member):
self.data_manager = data_manager
self.startdate = startdate
self.member = member
self.config = data_manager.config
self.experiment = self.config.experiment
self.cmor = self.config.cmor
self.convetion = self.config.data_convention
self.member_str = self.experiment.get_member_str(member)
self.original_files_path = os.path.join(self.config.data_dir, self.experiment.expid, 'original_files',
self.startdate, self.member_str, 'outputs')
self.atmos_timestep = None
self.cmor_scratch = str(os.path.join(self.config.scratch_dir, 'CMOR', self.startdate, self.member_str))
if self.config.data_convention in ('primavera', 'cmip6'):
self.lon_name = 'longitude'
self.lat_name = 'latitude'
else:
self.lon_name = 'lon'
self.lat_name = 'lat'
self.lon_name = self.config.data_convention.lon_name
self.lat_name = self.config.data_convention.lat_name
self.alt_coord_names = {'time_counter': 'time', 'time_counter_bnds': 'time_bnds',
'time_counter_bounds': 'time_bnds',
......@@ -465,10 +463,11 @@ class Cmorizer(object):
netcdf_file = NetCDFFile()
netcdf_file.data_manager = self.data_manager
netcdf_file.local_file = temp
netcdf_file.remote_file = self.data_manager.get_file_path(self.startdate, self.member,
var_cmor.domain, var_cmor.short_name, var_cmor,
None, frequency,
grid=alias.grid, year=None, date_str=date_str)
netcdf_file.remote_file = self.config.data_convention.get_file_path(self.startdate, self.member,
var_cmor.domain, var_cmor.short_name,
var_cmor, None, frequency,
grid=alias.grid, year=None,
date_str=date_str)
netcdf_file.data_convention = self.config.data_convention
netcdf_file.region = region
......@@ -494,7 +493,9 @@ class Cmorizer(object):
cube.coord(lev_original).var_name = lev_target
except iris.exceptions.CoordinateNotFoundError:
pass
iris.save(cube, temp, zlib=True)
Utils.rename_variables(temp, {'dim1': 'j', 'dim2': 'i'}, must_exist=False, rename_dimension=True)
def _set_coordinates_attribute(self, file_path, var_cmor, variable):
handler = Utils.open_cdf(file_path)
......
......@@ -229,7 +229,8 @@ class DataConvention(object):
if not grid:
grid = 'original'
if domain is None:
pass
variable_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep,
self.config.experiment.atmos_timestep)
vargrid_folder = domain.get_varfolder(var, self.config.experiment.ocean_timestep,
......@@ -630,7 +631,12 @@ class Cmor3Convention(DataConvention):
frequency = self.config.var_manager.tables[table].frequency
Log.debug('Creating links for table {0}', table)
for var in os.listdir(os.path.join(path, member, table)):
domain = self.config.var_manager.get_variable(var, silent=True).domain
cmor_var = self.config.var_manager.get_variable(var, silent=True)
domain = None
if cmor_var is not None:
domain = cmor_var.domain
if domain is None:
domain = self.config.var_manager.tables[table].domain
for grid in os.listdir(os.path.join(path, member, table, var)):
if member_str is not None and member_str != member:
continue
......
......@@ -90,9 +90,16 @@ class SelectLevels(Diagnostic):
def compute(self):
"""Run the diagnostic"""
temp = TempFile.get()
Utils.nco.ncks(input=self.variable_file, output=temp,
options=('-O -d lev,{0.min_depth},{0.max_depth}'.format(self.box),))
handler = Utils.open_cdf(self.variable_file.local_file)
var_name = ""
for var in ('lev', 'plev'):
if var in handler.variables:
var_name = var
continue
handler.close()
Utils.nco.ncks(input=self.variable_file.local_file, output=temp,
options=('-O -d {1},{0.min_depth},{0.max_depth}'.format(self.box, var_name),))
self.result.set_local_file(temp)
@staticmethod
......
......@@ -120,7 +120,7 @@ class ModelingRealm(object):
"""
table_name = self.get_table_name(frequency, data_convention)
from earthdiagnostics.variable import CMORTable
return CMORTable(table_name, frequency, 'December 2013')
return CMORTable(table_name, frequency, 'December 2013', self)
class ModelingRealms(object):
......
......@@ -33,6 +33,7 @@ class VariableManager(object):
self._dict_variables = {}
self._dict_aliases = {}
self.tables = {}
self.table_name = None
def get_variable(self, original_name, silent=False):
"""
......@@ -89,37 +90,38 @@ class VariableManager(object):
table_name: str
"""
self.table_name = table_name
self._dict_variables = dict()
self._load_variable_list(table_name)
self._load_variable_list()
self._load_missing_defaults()
self._load_known_aliases(table_name)
self._load_known_aliases()
self.create_aliases_dict()
def _load_variable_list(self, table_name):
def _load_variable_list(self):
xlsx_path = self._get_xlsx_path(table_name)
xlsx_path = self._get_xlsx_path()
if xlsx_path:
self._load_xlsx(xlsx_path)
return
json_folder = self._get_json_folder(table_name)
json_folder = self._get_json_folder()
if os.path.isdir(json_folder):
self._load_json(json_folder)
return
csv_path = self._get_csv_path(table_name)
csv_path = self._get_csv_path(self.table_name)
if os.path.isfile(csv_path):
self._load_file(table_name)
self._load_file(self.table_name)
return
raise Exception('Data convention {0} unknown'.format(table_name))
raise Exception('Data convention {0} unknown'.format(self.table_name))
def _get_csv_path(self, table_name):
csv_table_path = os.path.join(self._cmor_tables_folder, '{0}.csv'.format(table_name))
return csv_table_path
def _get_json_folder(self, table_name):
json_folder = os.path.join(self._cmor_tables_folder, '{0}/Tables'.format(table_name))
def _get_json_folder(self):
json_folder = os.path.join(self._cmor_tables_folder, '{0}/Tables'.format(self.table_name))
return json_folder
def _load_file(self, csv_table_path, default=False):
......@@ -169,7 +171,8 @@ class VariableManager(object):
table_id = data['Header']['table_id'][6:]
table = CMORTable(table_id,
Frequency(data['variable_entry'].values()[0]['frequency']),
data['Header']['table_date'])
data['Header']['table_date'],
ModelingRealms.parse(data['Header']['realm']))
self.tables[table_id] = table
self._load_json_variables(data['variable_entry'], table)
......@@ -189,9 +192,9 @@ class VariableManager(object):
except VariableJsonException:
Log.error('Could not read variable {0}'.format(short_name))
def _load_known_aliases(self, table_name):
def _load_known_aliases(self):
self._load_alias_csv('default')
self._load_alias_csv(table_name)
self._load_alias_csv(self.table_name)
def _load_alias_csv(self, filename):
file_path = self._get_aliases_csv_path(filename)
......@@ -262,12 +265,12 @@ class VariableManager(object):
for alias in cmor_var.known_aliases:
self._dict_aliases[alias.alias] = (alias, cmor_var)
def _get_xlsx_path(self, table_name):
xlsx_table_path = os.path.join(self._cmor_tables_folder, '{0}.xlsx'.format(table_name))
def _get_xlsx_path(self):
xlsx_table_path = os.path.join(self._cmor_tables_folder, '{0}.xlsx'.format(self.table_name))
if os.path.isfile(xlsx_table_path):
return xlsx_table_path
xlsx_table_path = os.path.join(self._cmor_tables_folder, table_name, 'etc', '*.xlsx')
xlsx_table_path = os.path.join(self._cmor_tables_folder, self.table_name, 'etc', '*.xlsx')
xlsx_table_path = glob.glob(xlsx_table_path)
if len(xlsx_table_path) == 1:
return xlsx_table_path[0]
......@@ -292,10 +295,11 @@ class VariableManager(object):
def _load_xlsx_table(self, sheet, table_data):
try:
table_frequency, table_date = table_data[sheet.title]
table = CMORTable(sheet.title, table_frequency, table_date)
realm = self._read_realm_from_json(sheet.title)
table = CMORTable(sheet.title, table_frequency, table_date, realm)
self.tables[sheet.title] = table
for row in sheet.rows:
if row[0].value == 'Priority' or not row[5].value:
if row[0].value in ('Priority', 'rm') or not row[5].value:
continue
self._parse_xlsx_var_row(row, table)
except Exception as ex:
......@@ -303,6 +307,18 @@ class VariableManager(object):
import traceback
traceback.print_exc()
def _read_realm_from_json(self, table_name):
for prefix in ('CMIP6', 'PRIMAVERA'):
json_path = os.path.join(self._get_json_folder(), '{0}_{1}.json'.format(prefix, table_name))
if os.path.isfile(json_path):
with open(json_path) as json_file:
json_data = json_file.read()
data = json.loads(json_data)
# Cogemos el primer realm para las tablas que tienen varios
# Solo se usa al generar los links para una startdate concreta
return ModelingRealms.parse(data['Header']['realm'].split(' ')[0])
return None
def _parse_xlsx_var_row(self, row, table):
cmor_name = row[11].value
if not cmor_name:
......@@ -487,7 +503,7 @@ class Variable(object):
return table
if self.domain:
table_name = self.domain.get_table_name(frequency, data_convention)
return CMORTable(table_name, frequency, 'December 2013')
return CMORTable(table_name, frequency, 'December 2013', self.domain)
raise ValueError('Can not get table for {0} and frequency {1}'.format(self, frequency))
@staticmethod
......@@ -554,16 +570,17 @@ class CMORTable(object):
date: str
"""
def __init__(self, name, frequency, date):
def __init__(self, name, frequency, date, domain):
self.name = name
self.frequency = frequency
self.date = date
self.domain = domain
def __str__(self):
return self.name
def __repr__(self):
return '{0.name} ({0.frequency}, {0.date})'.format(self)
return '{0.name} ({0.domain} {0.frequency}, {0.date})'.format(self)
def __lt__(self, other):
return self.name < other.name
......
This diff is collapsed.
"""Tests for PRIMAVERA Convention"""
import os
import shutil
import tempfile
......@@ -12,8 +13,10 @@ from earthdiagnostics.data_convention import PrimaveraConvention
class TestPrimaveraConvention(TestCase):
"""Tests for PRIMAVERA convetion class"""
def setUp(self):
"""Prepare tests"""
self.tmp_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(self.tmp_dir, 'expid'))
self.config = Mock()
......@@ -37,22 +40,27 @@ class TestPrimaveraConvention(TestCase):
self.convention = PrimaveraConvention('name', self.config)
def tearDown(self):
"""Cleanup"""
shutil.rmtree(self.tmp_dir)
def test_get_startdate_path(self):
"""Test get startdate path"""
self.assertEqual(self.convention.get_startdate_path('19900101'),
os.path.join(self.tmp_dir, 'expid/cmorfiles/activity/institute/model/experiment_name'))
def test_experiment_name(self):
"""Test get expriment name"""
self.assertEqual(self.convention.experiment_name('19900101'),
'experiment_name')
def test_experiment_name_append(self):
"""Test get expriment name when appending startdate"""
self.config.cmor.append_startdate = True
self.assertEqual(self.convention.experiment_name('19900101'),
'experiment_nameS19900101')
def test_get_cmor_folder_path(self):
"""Test get cmor foilder path"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -64,6 +72,7 @@ class TestPrimaveraConvention(TestCase):
'r2i1p1f1/Omon/var/ocean_grid/version'))
def test_get_cmor_folder_path_no_cmor_var(self):
"""Test get cmor folder path when not passing cmor_var"""
file_path = self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var',
Frequencies.monthly, None, None)
self.assertEqual(file_path,
......@@ -71,6 +80,7 @@ class TestPrimaveraConvention(TestCase):
'r2i1p1f1/Omon/var/ocean_grid/version'))
def test_get_cmor_folder_path_atmos(self):
"""Test get cmor foilder path for the atmos"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -82,6 +92,7 @@ class TestPrimaveraConvention(TestCase):
'r2i1p1f1/Omon/var/atmos_grid/version'))
def test_get_cmor_folder_path_custom_grid(self):
"""Test get cmor foilder path for a custom grid"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -93,6 +104,7 @@ class TestPrimaveraConvention(TestCase):
'r2i1p1f1/Omon/var/grid/version'))
def test_get_cmor_folder_path_no_cmor(self):
"""Test get cmor folder path with no cmor_var"""
frequency = Mock()
frequency.__str__ = Mock()
frequency.__str__.return_value = 'mon'
......@@ -104,6 +116,7 @@ class TestPrimaveraConvention(TestCase):
'r2i1p1f1/Omon/var/ocean_grid/version'))
def test_get_file_path_no_version_primavera(self):
"""Test get cmor folder path with no version"""
self.config.cmor.version = ''
cmor_var = Mock()
omon = Mock()
......@@ -113,6 +126,7 @@ class TestPrimaveraConvention(TestCase):
self.convention.get_cmor_folder_path('19900101', 1, ModelingRealms.ocean, 'var', 'mon', 'grid', cmor_var)
def test_get_filename(self):
"""Test get_filename"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -123,6 +137,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc')
def test_get_filename_no_cmor_var(self):
"""Test get_filename not passing cmor_var"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -133,6 +148,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001-199001.nc')
def test_get_filename_daily(self):
"""Test get_filename for daily frequency"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -144,6 +160,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_19900101-19900131.nc')
def test_get_filename_6hourly(self):
"""Test get_filename for 6hourly files"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -155,6 +172,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001010000-199001311800.nc')
def test_get_filename_3hourly(self):
"""Test get_filename for 3hourly files"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -166,6 +184,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_199001010000-199001312100.nc')
def test_get_filename_atmos(self):
"""Test get_filename for atmos"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -176,6 +195,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_atmos_grid_199001-199001.nc')
def test_get_filename_grid(self):
"""Test get_filename for a custom grid"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -186,6 +206,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_grid_199001-199001.nc')
def test_get_filename_year(self):
"""Test get_filename for a whole year"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -196,6 +217,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_1990.nc')
def test_get_filename_date_Str(self):
"""Test get_filename passing date_Str"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -206,6 +228,7 @@ class TestPrimaveraConvention(TestCase):
'var_Omon_model_experiment_name_r2i1p1f1_ocean_grid_date_str.nc')
def test_get_filename_no_date_info(self):
"""Test get_filename with no date info raises ValueError"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -217,6 +240,7 @@ class TestPrimaveraConvention(TestCase):
@mock.patch('os.path.isfile')
def test_is_cmorized(self, mock_is_file):
"""Test is cmorized"""
mock_is_file.return_value = True
cmor_var = Mock()
omon = Mock()
......@@ -230,6 +254,7 @@ class TestPrimaveraConvention(TestCase):
@mock.patch('os.path.isfile')
def test_is_not_cmorized(self, mock_is_file):
"""Test is cmorized false"""
mock_is_file.return_value = False
cmor_var = Mock()
omon = Mock()
......@@ -242,6 +267,7 @@ class TestPrimaveraConvention(TestCase):
self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean))
def test_is_cmorized_false_not_member_folder(self):
"""Test is cmorized false bacause ther is no member folder"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -253,6 +279,7 @@ class TestPrimaveraConvention(TestCase):
self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean))
def test_is_cmorized_false_not_table_folder(self):
"""Test is cmorized false bacause ther is no table folder"""
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
......@@ -265,6 +292,7 @@ class TestPrimaveraConvention(TestCase):
@mock.patch('os.path.isfile')
def test_is_cmorized_not_enough_vars(self, mock_is_file):
"""Test is cmorized false because thera are not eniouch variables"""
mock_is_file.return_value = True
cmor_var = Mock()
omon = Mock()
......@@ -276,18 +304,9 @@ class TestPrimaveraConvention(TestCase):
'mon/ocean/var'))
self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean))
def test_is_cmorized_not_domain_folder(self):
cmor_var = Mock()
omon = Mock()
omon.name = 'Omon'
cmor_var.get_table.return_value = omon
self.config.var_manager.get_variable.return_value = cmor_var
self.config.cmor.min_cmorized_vars = 2
os.makedirs(os.path.join(self.tmp_dir, 'expid/cmorfiles/institute/model/experiment_name/S20000101/mon'))
self.assertFalse(self.convention.is_cmorized('20000101', 1, 1, ModelingRealms.ocean))
@mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link')
def test_create_links_primavera(self, mock_create_link):
"""Test create links"""
member_path = os.path.join(self.tmp_dir,
'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/var/gn')
os.makedirs(member_path)
......@@ -298,6 +317,7 @@ class TestPrimaveraConvention(TestCase):
@mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link')
def test_create_links_with_version_primavera(self, mock_create_link):
"""Test create links with version"""
member_path = os.path.join(self.tmp_dir,
'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/'
'var/gn/version')
......@@ -309,6 +329,7 @@ class TestPrimaveraConvention(TestCase):
@mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link')
def test_create_links_with_version_primavera_no_member(self, mock_create_link):
"""Test create links with version full startdate"""
member_path = os.path.join(self.tmp_dir,
'expid/cmorfiles/activity/institute/model/experiment_name/r2i1p1f1/Omon/'
'var/gn/version')
......@@ -320,6 +341,7 @@ class TestPrimaveraConvention(TestCase):
@mock.patch('earthdiagnostics.data_convention.PrimaveraConvention.create_link')
def test_create_links_member_not_found_primavera(self, mock_create_link):
"""Test create links when the member can not be found"""
member_path = os.path.join(self.tmp_dir,
'expid/cmorfiles/activity/institute/model/experiment_name/r1i1p1f1/Omon/var/gn')
os.makedirs(member_path)
......
# coding=utf-8
from unittest import TestCase
import os
from tempfile import mktemp
import dummydata
from earthdiagnostics.diagnostic import DiagnosticVariableOption
from earthdiagnostics.box import Box
......@@ -21,6 +25,11 @@ class TestAttribute(TestCase):
self.box = Box()
self.box.min_depth = 0
self.box.max_depth = 100
self.var_file = mktemp('.nc')
def tearDown(self):
if os.path.exists(self.var_file):
os.remove(self.var_file)
def fake_parse(self, value):
return value
......@@ -53,3 +62,15 @@ class TestAttribute(TestCase):
self.assertEqual(str(mixed),
'Write attributte output Startdate: 20010101 Member: 0 Chunk: 0 Variable: atmos:var '
'Attributte: att:value Grid: grid')
def test_compute(self):
dummydata.model3.Model3(oname=self.var_file, var='ta', start_year=2000, stop_year=2000, method='constant',
constant=1)
diag = Attribute(self.data_manager, '20010101', 0, 0, ModelingRealms.atmos, 'ta', 'grid', 'att', 'value')
diag.variable_file = Mock()
diag.variable_file.local_file = self.var_file
diag.corrected = Mock()
diag.compute()
diag.corrected.set_local_file.assert_called_once()
# coding=utf-8
from unittest import TestCase
import os
from tempfile import mktemp
import dummydata
from earthdiagnostics.diagnostic import DiagnosticVariableOption
from earthdiagnostics.box import Box
......@@ -21,6 +25,14 @@ class TestModule(TestCase):
self.box = Box()
self.box.min_depth = 0
self.box.max_depth = 100