import os from autosubmit.config.log import Log from autosubmit.date.chunk_date_lib import parse_date, chunk_start_date, chunk_end_date, previous_day from earthdiagnostics import Utils class DataManager(object): def __init__(self, institution, model, expid, datafolder, frequency, chunk_size): self.institution = institution self.model = model self.expid = expid self.data_dir = datafolder self.frequency = frequency self.chunk_size = chunk_size def prepare_CMOR_files(self, startdates, members): # Check if cmorized and convert if not if not os.path.exists(os.path.join(self.data_dir, self.expid)): raise Exception('The experiment {0} is not CMORized. Please, CMORize it and launch again.'.format(self.expid)) for startdate in startdates: for member in members: member_path = os.path.join(self.data_dir, self.expid, startdate, 'fc{0}'.format(member), 'outputs') if not os.path.exists(os.path.join(member_path, 'output')): Utils.execute_shell_command('gunzip {0}'.format(os.path.join(member_path, '*.gz'))) Utils.execute_shell_command('tar -xvf {0}'.format(os.path.join(member_path, '*.tar'))) def get_files(self, startdate, member, chunk, domain, variables): file_names = list() if domain == 'seaIce': domain_abreviattion = 'OI' else: domain_abreviattion = domain[0].upper() start = parse_date(startdate) member_plus = str(member + 1) member_path = os.path.join(self.data_dir, self.expid, startdate, 'fc' + str(member), 'outputs', 'output', self.institution, self.model, self.model, 'S' + startdate, self.frequency, domain) chunk_start = chunk_start_date(start, chunk, self.chunk_size, 'month', 'standard') chunk_end = chunk_end_date(chunk_start, self.chunk_size, 'month', 'standard') chunk_end = previous_day(chunk_end, 'standard') var_file = list() for var in variables: var_file.append(os.path.join(member_path, var, 'r{0}i1p1'.format(member_plus), '{0}_{1}{2}_{3}_output_r{4}i1p1_' '{5}-{6}.nc'.format(var, domain_abreviattion, self.frequency, self.model, member_plus, "{0:04}{1:02}".format(chunk_start.year, chunk_start.month), "{0:04}{1:02}".format(chunk_end.year, chunk_end.month)))) file_names.append(var_file) return file_names