Commit 19b3cff9 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Fixed atmos cmorization

parent 57cf4053
# coding=utf-8
import glob
import os
import pygrib
import shutil
import uuid
import os
from datetime import datetime
import pygrib
from bscearth.utils.log import Log
from bscearth.utils.date import parse_date, chunk_end_date, previous_day, date2str, add_months
from bscearth.utils.log import Log
from earthdiagnostics.datafile import NetCDFFile
from earthdiagnostics.frequency import Frequency, Frequencies
from earthdiagnostics.modelingrealm import ModelingRealms
from earthdiagnostics.utils import TempFile, Utils
from earthdiagnostics.datafile import NetCDFFile
class Cmorizer(object):
......@@ -449,7 +448,7 @@ class Cmorizer(object):
def get_date_str(self, file_path):
file_parts = os.path.basename(file_path).split('_')
if file_parts[0] in (self.experiment.expid, 'MMA', 'MMASH', 'MMAGG', 'MMO') or file_parts[0].startswith('ORCA'):
if file_parts[0] in (self.experiment.expid, 't00o', 'MMA', 'MMASH', 'MMAGG', 'MMO') or file_parts[0].startswith('ORCA'):
# Model output
if file_parts[-1].endswith('.tar'):
file_parts = file_parts[-1][0:-4].split('-')
......
# coding=utf-8
import glob
import os
from datetime import datetime
import os
from bscearth.utils.log import Log
from bscearth.utils.date import parse_date, chunk_start_date, chunk_end_date, previous_day
from bscearth.utils.log import Log
from datafile import StorageStatus
from diagnostic import Diagnostic
......@@ -391,6 +391,7 @@ class CMORManager(DataManager):
startdate_path = self._get_startdate_path(startdate)
if not os.path.isdir(startdate_path):
return False
count = 0
if self.config.data_convention == 'specs':
for freq in os.listdir(startdate_path):
domain_path = os.path.join(startdate_path, freq,
......@@ -401,21 +402,29 @@ class CMORManager(DataManager):
var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk,
Frequency(freq))
if os.path.isfile(var_path):
return True
count += 1
if count >= self.config.cmor.min_cmorized_vars:
return True
else:
continue
else:
member_path = os.path.join(startdate_path, self._get_member_str(member))
if not os.path.isdir(member_path):
return False
freq = Frequencies.monthly
table = domain.get_table(freq, self.config.data_convention)
table_dir = os.path.join(member_path, table)
table_dir = os.path.join(member_path, table.name)
if not os.path.isdir(table_dir):
return False
for var in os.listdir(table_dir):
cmor_var = self.variable_list.get_variable(var, True)
var_path = self.get_file_path(startdate, member, domain, var, cmor_var, chunk, frequency=freq)
if os.path.isfile(var_path):
return True
count += 1
if count >= self.config.cmor.min_cmorized_vars:
return True
else:
continue
return False
def _cmorize_member(self, startdate, member):
......
......@@ -142,6 +142,7 @@ class CMORConfig(object):
self.default_ocean_grid = parser.get_option('CMOR', 'DEFAULT_OCEAN_GRID', 'gn')
self.default_atmos_grid = parser.get_option('CMOR', 'DEFAULT_ATMOS_GRID', 'gr')
self.activity = parser.get_option('CMOR', 'ACTIVITY', 'CMIP')
self.min_cmorized_vars = parser.get_int_option('CMOR', 'MIN_CMORIZED_VARS', 10)
vars_string = parser.get_option('CMOR', 'VARIABLE_LIST', '')
self.var_manager = var_manager
......
......@@ -306,6 +306,7 @@ class EarthDiags(object):
Utils.give_group_write_permissions(self.config.scratch_masks)
mesh_mask_scratch_path = os.path.join(self.config.scratch_masks, mesh_mask)
if self._copy_file(mesh_mask_path, mesh_mask_scratch_path,
restore_meshes):
Utils.give_group_write_permissions(mesh_mask_scratch_path)
......
......@@ -93,6 +93,7 @@ class TestCMORConfig(TestCase):
self.assertEquals(config.filter_files, '')
self.assertEquals(config.default_atmos_grid, 'gr')
self.assertEquals(config.default_ocean_grid, 'gn')
self.assertEquals(config.min_cmorized_vars, 10)
def test_cmorize(self):
config = CMORConfig(self.mock_parser, self.var_manager)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment