Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
import os
from autosubmit.config.log import Log
from autosubmit.date.chunk_date_lib import parse_date, chunk_start_date, chunk_end_date, previous_day
from earthdiagnostics import Utils
class DataManager(object):
def __init__(self, institution, model, expid, datafolder, frequency, chunk_size):
self.institution = institution
self.model = model
self.expid = expid
self.data_dir = datafolder
self.frequency = frequency
self.chunk_size = chunk_size
def prepare_CMOR_files(self, startdates, members):
# Check if cmorized and convert if not
if not os.path.exists(os.path.join(self.data_dir, self.expid)):
raise Exception('The experiment {0} is not CMORized. Please, CMORize it and launch again.'.format(self.expid))
for startdate in startdates:
for member in members:
member_path = os.path.join(self.data_dir, self.expid, startdate, 'fc{0}'.format(member), 'outputs')
if not os.path.exists(os.path.join(member_path, 'output')):
Utils.execute_shell_command('gunzip {0}'.format(os.path.join(member_path, '*.gz')))
Utils.execute_shell_command('tar -xvf {0}'.format(os.path.join(member_path, '*.tar')))
def get_files(self, startdate, member, chunk, domain, variables):
file_names = list()
if domain == 'seaIce':
domain_abreviattion = 'OI'
else:
domain_abreviattion = domain[0].upper()
start = parse_date(startdate)
member_plus = str(member + 1)
member_path = os.path.join(self.data_dir, self.expid, startdate, 'fc' + str(member), 'outputs', 'output',
self.institution, self.model, self.model, 'S' + startdate, self.frequency,
domain)
chunk_start = chunk_start_date(start, chunk, self.chunk_size, 'month', 'standard')
chunk_end = chunk_end_date(chunk_start, self.chunk_size, 'month', 'standard')
chunk_end = previous_day(chunk_end, 'standard')
var_file = list()
for var in variables:
var_file.append(os.path.join(member_path, var, 'r{0}i1p1'.format(member_plus),
'{0}_{1}{2}_{3}_output_r{4}i1p1_'
'{5}-{6}.nc'.format(var, domain_abreviattion,
self.frequency, self.model, member_plus,
"{0:04}{1:02}".format(chunk_start.year,
chunk_start.month),
"{0:04}{1:02}".format(chunk_end.year,
chunk_end.month))))
file_names.append(var_file)
return file_names