Commit 719757a5 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Adapted code to work with cfunits 1.1.4

parent d1157ac8
......@@ -5,13 +5,12 @@ import shutil
import threading
import uuid
import pygrib
from cf_units import Unit
from cfunits import Units
from datetime import datetime
import netCDF4
import numpy as np
import os
import stat
from autosubmit.config.log import Log
from autosubmit.date.chunk_date_lib import parse_date, chunk_start_date, chunk_end_date, previous_day, add_months, \
date2str
......@@ -186,6 +185,7 @@ class DataManager(object):
mes2 = grib_handler.readline()
nfrp = mes2.analDate - mes1.analDate
nfrp = int(nfrp.total_seconds() / 3600)
self.nfrp = nfrp
grib_handler.close()
prev_gribfile = os.path.join(self.scratch_dir,
......@@ -892,15 +892,31 @@ class DataManager(object):
if cmor_var.units:
if 'units' in var_handler.ncattrs():
if var_handler.units == 'PSU':
var_handler.units = 'psu'
if var_handler.units == 'C' and cmor_var.units == 'K':
var_handler.units = 'deg_C'
if cmor_var.units != var_handler.units:
new_unit = Unit(cmor_var.units)
old_unit = Unit(var_handler.units)
var_handler[:] = new_unit.convert(var_handler[:], old_unit, inplace=True)
if 'valid_min' in var_handler.ncattrs():
var_handler.valid_min = new_unit.convert(float(var_handler.valid_min), old_unit)
if 'valid_max' in var_handler.ncattrs():
var_handler.valid_max = new_unit.convert(float(var_handler.valid_max), old_unit)
try:
new_unit = Units(cmor_var.units)
old_unit = Units(var_handler.units)
var_handler[:] = Units.conform(var_handler[:], old_unit, new_unit, inplace=True)
if 'valid_min' in var_handler.ncattrs():
var_handler.valid_min = Units.conform(float(var_handler.valid_min), old_unit, new_unit,
inplace=True)
if 'valid_max' in var_handler.ncattrs():
var_handler.valid_max = Units.conform(float(var_handler.valid_max), old_unit, new_unit,
inplace=True)
except ValueError:
factor, offset = UnitConversion.get_conversion_factor_offset(var_handler.units,
cmor_var.units)
var_handler[:] = var_handler[:] * factor + offset
if 'valid_min' in var_handler.ncattrs():
var_handler.valid_min = float(var_handler.valid_min) * factor + offset
if 'valid_max' in var_handler.ncattrs():
var_handler.valid_max = float(var_handler.valid_max) * factor + offset
var_handler.units = cmor_var.units
handler.sync()
......@@ -944,8 +960,6 @@ class DataManager(object):
Utils.rename_variables(filetosend, variables, False, True)
Utils.move_file(filetosend, filepath)
st = os.stat(filepath)
os.chmod(filepath, st.st_mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
self._create_link(domain, filepath, frequency, var)
def _create_link(self, domain, filepath, frequency, var):
......@@ -959,7 +973,7 @@ class DataManager(object):
else:
variable_folder = '{0}_f{1}h'.format(var, self.nfrp)
link_path = os.path.join(self.data_dir, self.expid, freq_str, domain.lower(), variable_folder)
link_path = os.path.join(self.data_dir, self.expid, freq_str, variable_folder)
if not os.path.exists(link_path):
# This can be a race condition
......@@ -1206,8 +1220,7 @@ class UnitConversion(object):
conversion = cls._dict_conversions[(unit, new_unit)]
return conversion.factor, conversion.offset
elif (new_unit, unit) in cls._dict_conversions:
conversion = cls._dict_conversions[(unit, new_unit)]
conversion = cls._dict_conversions[(new_unit, unit)]
return 1 / conversion.factor, -conversion.offset
else:
return None, None
......@@ -7,7 +7,7 @@ DATA_DIR = /esnas/exp/ecearth/
CON_FILES = /esnas/autosubmit/con_files/
# Diagnostics to run, space separated. You must provide for each one the name and the parameters (comma separated) or
# an alias defined in the ALIAS section (see more below)
DIAGS = 3dsal
DIAGS = rewrite,tas,atmos
# Frequency of the data you want to use
FREQUENCY = mon
# Path to CDFTOOLS binaries
......@@ -19,7 +19,7 @@ RESTORE_MESHES = False
[CMOR]
# If true, recreates CMOR files regardless of presence. Default = False
FORCE = True
FORCE = False
# If true, CMORizes ocean files. Default = True
OCEAN_FILES = True
# If true, CMORizes atmosphere files. Default = True
......@@ -39,6 +39,7 @@ INSTITUTE = BSC
MODEL = EC-EARTH
# Model version
MODEL_VERSION =Ec3.2_O1L75
NFRP = 6
# For those who use Autosubmit, this no need documentation
# For those who not, EXPID is the unique identifier of the experiment.
......
......@@ -141,7 +141,7 @@ class Utils(object):
while hash_original != hash_destiny:
if retrials == 0:
raise Exception('Can not move {0} to {1}'.format(source, destiny))
shutil.copy(source, destiny)
shutil.copyfile(source, destiny)
hash_destiny = Utils.get_file_hash(destiny)
os.remove(source)
......
# coding=utf-8
from test_data_manager import TestDataManager, TestVariable
from test_data_manager import TestDataManager, TestVariable, TestConversion
from test_constants import TestBasin, TestBasins
from test_experiment_manager import TestExperimentManager
from test_box import TestBox
......
......@@ -2,7 +2,7 @@
from unittest import TestCase
from earthdiagnostics.datamanager import DataManager, Variable
from earthdiagnostics.datamanager import DataManager, Variable, UnitConversion
from experimentmanager import ExperimentManager
from datetime import date
......@@ -49,3 +49,35 @@ class TestVariable(TestCase):
Variable._dict_variables['var'] = variable
self.assertIs(Variable.get_variable('var'), variable)
self.assertIsNone(Variable.get_variable('novar'))
class TestConversion(TestCase):
def test__init(self):
conversion = UnitConversion('km', 'm', 1000, 0)
self.assertEqual(conversion.source, 'km')
self.assertEqual(conversion.destiny, 'm')
self.assertEqual(conversion.factor, 1000)
self.assertEqual(conversion.offset, 0)
def test_add_conversion(self):
UnitConversion._dict_conversions = dict()
conversion = UnitConversion('km', 'm', 1000, 0)
UnitConversion.add_conversion(conversion)
self.assertIs(UnitConversion._dict_conversions[('km', 'm')], conversion)
UnitConversion._dict_conversions = dict()
def test_get_factor_offset(self):
UnitConversion._dict_conversions = dict()
conversion = UnitConversion('km', 'm', 1000, 0)
UnitConversion.add_conversion(conversion)
self.assertEqual(UnitConversion.get_conversion_factor_offset('km', 'm'), (1000, 0))
self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'km'), (1 / 1000.0, 0))
self.assertEqual(UnitConversion.get_conversion_factor_offset('1e3 m', 'km'), (1, 0))
self.assertEqual(UnitConversion.get_conversion_factor_offset('10^3 m', 'km'), (1, 0))
self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '1e3 m'), (1, 0))
self.assertEqual(UnitConversion.get_conversion_factor_offset('km', '10^3 m'), (1, 0))
self.assertEqual(UnitConversion.get_conversion_factor_offset('m', 'm'), (1, 0))
self.assertEqual(UnitConversion.get_conversion_factor_offset('m²', 'km'), (None, None))
UnitConversion._dict_conversions = dict()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment