Commit 041b0040 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Look for files in original_files

parent f3a09722
......@@ -193,6 +193,7 @@ class DataConvention(object):
return os.path.join(
self.config.data_dir,
self.config.experiment.expid,
"original_files",
"cmorfiles",
self.config.cmor.activity,
self.config.experiment.institute,
......@@ -643,6 +644,7 @@ class SPECSConvention(Cmor2Convention):
return os.path.join(
self.config.data_dir,
self.config.experiment.expid,
"original_files",
"cmorfiles",
self.config.experiment.institute,
self.config.experiment.model,
......@@ -682,6 +684,7 @@ class PrefaceConvention(Cmor2Convention):
return os.path.join(
self.config.data_dir,
self.config.experiment.expid,
"original_files",
"cmorfiles",
self.config.experiment.institute,
self.experiment_name(startdate),
......
......@@ -186,6 +186,12 @@ class DataFile(Publisher):
return True
return self._modifiers[0] is diagnostic
@property
def remote_diags_file(self):
remote_diags_file = self.remote_file.replace("/cmorfiles/", "/diags/")
remote_diags_file = remote_diags_file.replace("/original_files/", "/")
return remote_diags_file
@property
def local_status(self):
"""Get local storage status"""
......@@ -271,9 +277,8 @@ class DataFile(Publisher):
self.storage_status = StorageStatus.UPLOADING
remote_file = self.remote_file
try:
if "/cmorfiles/" in remote_file:
remote_file = remote_file.replace("/cmorfiles/", "/diags/")
Utils.copy_file(self.local_file, remote_file, save_hash=True)
Utils.copy_file(
self.local_file, self.remote_diags_file, save_hash=True)
except (OSError, Exception) as ex:
Log.error("File {0} can not be uploaded: {1}", remote_file, ex)
self.storage_status = StorageStatus.FAILED
......@@ -437,12 +442,10 @@ class DataFile(Publisher):
except iris.exceptions.CoordinateNotFoundError:
return
try:
old_cube = iris.load_cube(self.remote_file)
old_cube = iris.load_cube(self.remote_diags_file)
except Exception:
try:
old_cube = iris.load_cube(
self.remote_file.replace('/cmorfiles/', '/diags/')
)
old_cube = iris.load_cube(self.remote_file)
except Exception:
# Bad data, overwrite
return
......@@ -656,7 +659,7 @@ class NetCDFFile(DataFile):
def download(self):
"""Get data from remote storage to the local one"""
for path in (
self.remote_file.replace("/cmorfiles/", "/diags/"),
self.remote_diags_file,
self.remote_file,
):
if os.path.isfile(path):
......@@ -705,7 +708,7 @@ class NetCDFFile(DataFile):
def check_is_in_storage(self, update_status=True):
for path in (
self.remote_file,
self.remote_file.replace("/cmorfiles/", "/diags/"),
self.remote_diags_file
):
if os.path.isfile(path):
if update_status:
......
......@@ -43,7 +43,8 @@ class TestDataConvention(TestCase):
self.convention.get_startdate_path("19900101"),
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/expname",
"expid/original_files/cmorfiles/activity/institute/model/"
"expname",
),
)
......
......@@ -40,7 +40,8 @@ class TestMeteoFranceConvention(TestCase):
self.convention.get_startdate_path("19900101"),
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name",
),
)
......
......@@ -41,7 +41,8 @@ class TestPrefaceConvention(TestCase):
self.convention.get_startdate_path("19900101"),
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S19900101",
),
)
......@@ -69,8 +70,8 @@ class TestPrefaceConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/"
"ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S19900101/mon/ocean/var/r2i1p1/version",
),
)
......@@ -86,8 +87,8 @@ class TestPrefaceConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/"
"atmos/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S19900101/mon/atmos/var/r2i1p1/version",
),
)
......@@ -103,8 +104,8 @@ class TestPrefaceConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/"
"ocean/var/grid/r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S19900101/mon/ocean/var/grid/r2i1p1/version",
),
)
......@@ -126,8 +127,8 @@ class TestPrefaceConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/"
"ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S19900101/mon/ocean/var/r2i1p1/version",
),
)
......@@ -150,8 +151,8 @@ class TestPrefaceConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/"
"ocean/var/r2i1p1",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S19900101/mon/ocean/var/r2i1p1",
),
)
......@@ -319,8 +320,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/"
"ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20000101/mon/ocean/var/r2i1p1/version",
)
)
self.assertTrue(
......@@ -339,8 +340,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/"
"ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20000101/mon/ocean/var/r2i1p1/version",
)
)
self.assertFalse(
......@@ -357,8 +358,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon"
"/ocean/var/",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20000101/mon/ocean/var/",
)
)
self.assertFalse(
......@@ -377,8 +378,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/"
"ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20000101/mon/ocean/var/r2i1p1/version",
)
)
self.assertFalse(
......@@ -395,8 +396,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/"
"r2i1p1/version",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20000101/mon/r2i1p1/version",
)
)
self.assertFalse(
......@@ -413,7 +414,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20000101",
)
)
self.assertFalse(
......@@ -426,8 +428,8 @@ class TestPrefaceConvention(TestCase):
def test_create_links(self, mock_create_link):
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20010101/mon/ocean/"
"var/r2i1p1",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20010101/mon/ocean/var/r2i1p1",
)
os.makedirs(member_path)
tempfile.mkstemp(dir=member_path)
......@@ -440,8 +442,8 @@ class TestPrefaceConvention(TestCase):
def test_create_links_member_not_found(self, mock_create_link):
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20010101/mon/ocean/"
"var/r1i1p1",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20010101/mon/ocean/var/r1i1p1",
)
os.makedirs(member_path)
tempfile.mkstemp(dir=member_path)
......@@ -454,8 +456,8 @@ class TestPrefaceConvention(TestCase):
def test_create_links_with_grid(self, mock_create_link):
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20010101/mon/"
"ocean/var/r2i1p1/grid",
"expid/original_files/cmorfiles/institute/experiment_name/"
"S20010101/mon/ocean/var/r2i1p1/grid",
)
os.makedirs(member_path)
tempfile.mkstemp(dir=member_path)
......
......@@ -50,7 +50,8 @@ class TestPrimaveraConvention(TestCase):
self.convention.get_startdate_path("19900101"),
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name",
),
)
......@@ -80,8 +81,8 @@ class TestPrimaveraConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/ocean_grid/version",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/ocean_grid/version",
),
)
......@@ -100,8 +101,8 @@ class TestPrimaveraConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/ocean_grid/version",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/ocean_grid/version",
),
)
......@@ -118,8 +119,8 @@ class TestPrimaveraConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/atmos_grid/version",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/atmos_grid/version",
),
)
......@@ -136,8 +137,8 @@ class TestPrimaveraConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/grid/version",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/grid/version",
),
)
......@@ -160,8 +161,8 @@ class TestPrimaveraConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/ocean_grid/version",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/ocean_grid/version",
),
)
......@@ -481,8 +482,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var",
)
)
self.assertTrue(
......@@ -502,8 +503,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var",
)
)
self.assertFalse(
......@@ -521,7 +522,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/",
)
)
self.assertFalse(
......@@ -539,8 +541,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1",
)
)
self.assertFalse(
......@@ -560,8 +562,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101/"
"mon/ocean/var",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S20000101/mon/ocean/var",
)
)
self.assertFalse(
......@@ -575,8 +577,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links"""
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/gn",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/gn",
)
os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()}
......@@ -591,8 +593,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links with version"""
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/gn/version",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/gn/version",
)
os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()}
......@@ -609,8 +611,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links with version full startdate"""
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r2i1p1f1/Omon/var/gn/version",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r2i1p1f1/Omon/var/gn/version",
)
os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()}
......@@ -625,8 +627,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links when the member can not be found"""
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/"
"r1i1p1f1/Omon/var/gn",
"expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/r1i1p1f1/Omon/var/gn",
)
os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()}
......
......@@ -41,7 +41,8 @@ class TestSpecsConvention(TestCase):
self.convention.get_startdate_path("19900101"),
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S19900101",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S19900101",
),
)
......@@ -69,8 +70,8 @@ class TestSpecsConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S19900101/"
"mon/ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S19900101/mon/ocean/var/r2i1p1/version",
),
)
......@@ -86,8 +87,8 @@ class TestSpecsConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S19900101/"
"mon/atmos/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S19900101/mon/atmos/var/r2i1p1/version",
),
)
......@@ -103,8 +104,8 @@ class TestSpecsConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S19900101/"
"mon/ocean/var/grid/r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S19900101/mon/ocean/var/grid/r2i1p1/version",
),
)
......@@ -126,8 +127,8 @@ class TestSpecsConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S19900101/"
"mon/ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S19900101/mon/ocean/var/r2i1p1/version",
),
)
......@@ -150,8 +151,8 @@ class TestSpecsConvention(TestCase):
file_path,
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S19900101/"
"mon/ocean/var/r2i1p1",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S19900101/mon/ocean/var/r2i1p1",
),
)
......@@ -418,8 +419,8 @@ class TestSpecsConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101/"
"mon/ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S20000101/mon/ocean/var/r2i1p1/version",
)
)
self.assertTrue(
......@@ -438,8 +439,8 @@ class TestSpecsConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101/"
"mon/ocean/var/r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S20000101/mon/ocean/var/r2i1p1/version",
)
)
self.assertFalse(
......@@ -457,7 +458,9 @@ class TestSpecsConvention(TestCase):
self.config.cmor.min_cmorized_vars = 1
os.makedirs(
os.path.join(
self.tmp_dir, "expid/cmorfiles/institute/model/experiment_name"
self.tmp_dir,
"expid/original_files/cmorfiles/institute/model/"
"experiment_name"
)
)
self.assertFalse(
......@@ -474,8 +477,8 @@ class TestSpecsConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101/"
"mon/ocean/var/",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S20000101/mon/ocean/var/",
)
)
self.assertFalse(
......@@ -494,9 +497,8 @@ class TestSpecsConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101/"
"mon/ocean/var/"
"r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S20000101/mon/ocean/var/r2i1p1/version",
)
)
self.assertFalse(
......@@ -513,8 +515,8 @@ class TestSpecsConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101/"
"mon/r2i1p1/version",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S20000101/mon/r2i1p1/version",
)
)
self.assertFalse(
......@@ -531,7 +533,8 @@ class TestSpecsConvention(TestCase):
os.makedirs(
os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101",
"expid/original_files/cmorfiles/institute/model/"
"experiment_name/S20000101",
)
)
self.assertFalse(
......@@ -542,8 +545,8 @@ class TestSpecsConvention(TestCase):
def test_create_links(self, mock_create_link):
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20010101/mon/"
"ocean/var/r2i1p1",
"expid/original_files/cmorfiles/institute/model/experiment_name/"
"S20010101/mon/ocean/var/r2i1p1",
)
os.makedirs(member_path)
tempfile.mkstemp(dir=member_path)
......@@ -554,8 +557,8 @@ class TestSpecsConvention(TestCase):
def test_create_links_member_not_found(self, mock_create_link):
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20010101/mon/"
"ocean/var/r1i1p1",
"expid/original_files/cmorfiles/institute/model/experiment_name/"
"S20010101/mon/ocean/var/r1i1p1",
)
os.makedirs(member_path)
tempfile.mkstemp(dir=member_path)
......@@ -566,9 +569,8 @@ class TestSpecsConvention(TestCase):
def test_create_links_with_grid(self, mock_create_link):
member_path = os.path.join(
self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20010101/"
"mon/ocean/var/"
"r2i1p1/grid",
"expid/original_files/cmorfiles/institute/model/experiment_name/"
"S20010101/mon/ocean/var/r2i1p1/grid",
)
os.makedirs(member_path)
tempfile.mkstemp(dir=member_path)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment