Commit 041b0040 authored by Javier Vegas-Regidor's avatar Javier Vegas-Regidor
Browse files

Look for files in original_files

parent f3a09722
...@@ -193,6 +193,7 @@ class DataConvention(object): ...@@ -193,6 +193,7 @@ class DataConvention(object):
return os.path.join( return os.path.join(
self.config.data_dir, self.config.data_dir,
self.config.experiment.expid, self.config.experiment.expid,
"original_files",
"cmorfiles", "cmorfiles",
self.config.cmor.activity, self.config.cmor.activity,
self.config.experiment.institute, self.config.experiment.institute,
...@@ -643,6 +644,7 @@ class SPECSConvention(Cmor2Convention): ...@@ -643,6 +644,7 @@ class SPECSConvention(Cmor2Convention):
return os.path.join( return os.path.join(
self.config.data_dir, self.config.data_dir,
self.config.experiment.expid, self.config.experiment.expid,
"original_files",
"cmorfiles", "cmorfiles",
self.config.experiment.institute, self.config.experiment.institute,
self.config.experiment.model, self.config.experiment.model,
...@@ -682,6 +684,7 @@ class PrefaceConvention(Cmor2Convention): ...@@ -682,6 +684,7 @@ class PrefaceConvention(Cmor2Convention):
return os.path.join( return os.path.join(
self.config.data_dir, self.config.data_dir,
self.config.experiment.expid, self.config.experiment.expid,
"original_files",
"cmorfiles", "cmorfiles",
self.config.experiment.institute, self.config.experiment.institute,
self.experiment_name(startdate), self.experiment_name(startdate),
......
...@@ -186,6 +186,12 @@ class DataFile(Publisher): ...@@ -186,6 +186,12 @@ class DataFile(Publisher):
return True return True
return self._modifiers[0] is diagnostic return self._modifiers[0] is diagnostic
@property
def remote_diags_file(self):
remote_diags_file = self.remote_file.replace("/cmorfiles/", "/diags/")
remote_diags_file = remote_diags_file.replace("/original_files/", "/")
return remote_diags_file
@property @property
def local_status(self): def local_status(self):
"""Get local storage status""" """Get local storage status"""
...@@ -271,9 +277,8 @@ class DataFile(Publisher): ...@@ -271,9 +277,8 @@ class DataFile(Publisher):
self.storage_status = StorageStatus.UPLOADING self.storage_status = StorageStatus.UPLOADING
remote_file = self.remote_file remote_file = self.remote_file
try: try:
if "/cmorfiles/" in remote_file: Utils.copy_file(
remote_file = remote_file.replace("/cmorfiles/", "/diags/") self.local_file, self.remote_diags_file, save_hash=True)
Utils.copy_file(self.local_file, remote_file, save_hash=True)
except (OSError, Exception) as ex: except (OSError, Exception) as ex:
Log.error("File {0} can not be uploaded: {1}", remote_file, ex) Log.error("File {0} can not be uploaded: {1}", remote_file, ex)
self.storage_status = StorageStatus.FAILED self.storage_status = StorageStatus.FAILED
...@@ -437,12 +442,10 @@ class DataFile(Publisher): ...@@ -437,12 +442,10 @@ class DataFile(Publisher):
except iris.exceptions.CoordinateNotFoundError: except iris.exceptions.CoordinateNotFoundError:
return return
try: try:
old_cube = iris.load_cube(self.remote_file) old_cube = iris.load_cube(self.remote_diags_file)
except Exception: except Exception:
try: try:
old_cube = iris.load_cube( old_cube = iris.load_cube(self.remote_file)
self.remote_file.replace('/cmorfiles/', '/diags/')
)
except Exception: except Exception:
# Bad data, overwrite # Bad data, overwrite
return return
...@@ -656,7 +659,7 @@ class NetCDFFile(DataFile): ...@@ -656,7 +659,7 @@ class NetCDFFile(DataFile):
def download(self): def download(self):
"""Get data from remote storage to the local one""" """Get data from remote storage to the local one"""
for path in ( for path in (
self.remote_file.replace("/cmorfiles/", "/diags/"), self.remote_diags_file,
self.remote_file, self.remote_file,
): ):
if os.path.isfile(path): if os.path.isfile(path):
...@@ -705,7 +708,7 @@ class NetCDFFile(DataFile): ...@@ -705,7 +708,7 @@ class NetCDFFile(DataFile):
def check_is_in_storage(self, update_status=True): def check_is_in_storage(self, update_status=True):
for path in ( for path in (
self.remote_file, self.remote_file,
self.remote_file.replace("/cmorfiles/", "/diags/"), self.remote_diags_file
): ):
if os.path.isfile(path): if os.path.isfile(path):
if update_status: if update_status:
......
...@@ -43,7 +43,8 @@ class TestDataConvention(TestCase): ...@@ -43,7 +43,8 @@ class TestDataConvention(TestCase):
self.convention.get_startdate_path("19900101"), self.convention.get_startdate_path("19900101"),
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/expname", "expid/original_files/cmorfiles/activity/institute/model/"
"expname",
), ),
) )
......
...@@ -40,7 +40,8 @@ class TestMeteoFranceConvention(TestCase): ...@@ -40,7 +40,8 @@ class TestMeteoFranceConvention(TestCase):
self.convention.get_startdate_path("19900101"), self.convention.get_startdate_path("19900101"),
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name", "expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name",
), ),
) )
......
...@@ -41,7 +41,8 @@ class TestPrefaceConvention(TestCase): ...@@ -41,7 +41,8 @@ class TestPrefaceConvention(TestCase):
self.convention.get_startdate_path("19900101"), self.convention.get_startdate_path("19900101"),
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101", "expid/original_files/cmorfiles/institute/experiment_name/"
"S19900101",
), ),
) )
...@@ -69,8 +70,8 @@ class TestPrefaceConvention(TestCase): ...@@ -69,8 +70,8 @@ class TestPrefaceConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/r2i1p1/version", "S19900101/mon/ocean/var/r2i1p1/version",
), ),
) )
...@@ -86,8 +87,8 @@ class TestPrefaceConvention(TestCase): ...@@ -86,8 +87,8 @@ class TestPrefaceConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"atmos/var/r2i1p1/version", "S19900101/mon/atmos/var/r2i1p1/version",
), ),
) )
...@@ -103,8 +104,8 @@ class TestPrefaceConvention(TestCase): ...@@ -103,8 +104,8 @@ class TestPrefaceConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/grid/r2i1p1/version", "S19900101/mon/ocean/var/grid/r2i1p1/version",
), ),
) )
...@@ -126,8 +127,8 @@ class TestPrefaceConvention(TestCase): ...@@ -126,8 +127,8 @@ class TestPrefaceConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/r2i1p1/version", "S19900101/mon/ocean/var/r2i1p1/version",
), ),
) )
...@@ -150,8 +151,8 @@ class TestPrefaceConvention(TestCase): ...@@ -150,8 +151,8 @@ class TestPrefaceConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S19900101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/r2i1p1", "S19900101/mon/ocean/var/r2i1p1",
), ),
) )
...@@ -319,8 +320,8 @@ class TestPrefaceConvention(TestCase): ...@@ -319,8 +320,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/r2i1p1/version", "S20000101/mon/ocean/var/r2i1p1/version",
) )
) )
self.assertTrue( self.assertTrue(
...@@ -339,8 +340,8 @@ class TestPrefaceConvention(TestCase): ...@@ -339,8 +340,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/r2i1p1/version", "S20000101/mon/ocean/var/r2i1p1/version",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -357,8 +358,8 @@ class TestPrefaceConvention(TestCase): ...@@ -357,8 +358,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon" "expid/original_files/cmorfiles/institute/experiment_name/"
"/ocean/var/", "S20000101/mon/ocean/var/",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -377,8 +378,8 @@ class TestPrefaceConvention(TestCase): ...@@ -377,8 +378,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/r2i1p1/version", "S20000101/mon/ocean/var/r2i1p1/version",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -395,8 +396,8 @@ class TestPrefaceConvention(TestCase): ...@@ -395,8 +396,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"r2i1p1/version", "S20000101/mon/r2i1p1/version",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -413,7 +414,8 @@ class TestPrefaceConvention(TestCase): ...@@ -413,7 +414,8 @@ class TestPrefaceConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20000101", "expid/original_files/cmorfiles/institute/experiment_name/"
"S20000101",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -426,8 +428,8 @@ class TestPrefaceConvention(TestCase): ...@@ -426,8 +428,8 @@ class TestPrefaceConvention(TestCase):
def test_create_links(self, mock_create_link): def test_create_links(self, mock_create_link):
member_path = os.path.join( member_path = os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20010101/mon/ocean/" "expid/original_files/cmorfiles/institute/experiment_name/"
"var/r2i1p1", "S20010101/mon/ocean/var/r2i1p1",
) )
os.makedirs(member_path) os.makedirs(member_path)
tempfile.mkstemp(dir=member_path) tempfile.mkstemp(dir=member_path)
...@@ -440,8 +442,8 @@ class TestPrefaceConvention(TestCase): ...@@ -440,8 +442,8 @@ class TestPrefaceConvention(TestCase):
def test_create_links_member_not_found(self, mock_create_link): def test_create_links_member_not_found(self, mock_create_link):
member_path = os.path.join( member_path = os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20010101/mon/ocean/" "expid/original_files/cmorfiles/institute/experiment_name/"
"var/r1i1p1", "S20010101/mon/ocean/var/r1i1p1",
) )
os.makedirs(member_path) os.makedirs(member_path)
tempfile.mkstemp(dir=member_path) tempfile.mkstemp(dir=member_path)
...@@ -454,8 +456,8 @@ class TestPrefaceConvention(TestCase): ...@@ -454,8 +456,8 @@ class TestPrefaceConvention(TestCase):
def test_create_links_with_grid(self, mock_create_link): def test_create_links_with_grid(self, mock_create_link):
member_path = os.path.join( member_path = os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/experiment_name/S20010101/mon/" "expid/original_files/cmorfiles/institute/experiment_name/"
"ocean/var/r2i1p1/grid", "S20010101/mon/ocean/var/r2i1p1/grid",
) )
os.makedirs(member_path) os.makedirs(member_path)
tempfile.mkstemp(dir=member_path) tempfile.mkstemp(dir=member_path)
......
...@@ -50,7 +50,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -50,7 +50,8 @@ class TestPrimaveraConvention(TestCase):
self.convention.get_startdate_path("19900101"), self.convention.get_startdate_path("19900101"),
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name", "expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name",
), ),
) )
...@@ -80,8 +81,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -80,8 +81,8 @@ class TestPrimaveraConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/ocean_grid/version", "experiment_name/r2i1p1f1/Omon/var/ocean_grid/version",
), ),
) )
...@@ -100,8 +101,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -100,8 +101,8 @@ class TestPrimaveraConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/ocean_grid/version", "experiment_name/r2i1p1f1/Omon/var/ocean_grid/version",
), ),
) )
...@@ -118,8 +119,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -118,8 +119,8 @@ class TestPrimaveraConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/atmos_grid/version", "experiment_name/r2i1p1f1/Omon/var/atmos_grid/version",
), ),
) )
...@@ -136,8 +137,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -136,8 +137,8 @@ class TestPrimaveraConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/grid/version", "experiment_name/r2i1p1f1/Omon/var/grid/version",
), ),
) )
...@@ -160,8 +161,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -160,8 +161,8 @@ class TestPrimaveraConvention(TestCase):
file_path, file_path,
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/ocean_grid/version", "experiment_name/r2i1p1f1/Omon/var/ocean_grid/version",
), ),
) )
...@@ -481,8 +482,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -481,8 +482,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var", "experiment_name/r2i1p1f1/Omon/var",
) )
) )
self.assertTrue( self.assertTrue(
...@@ -502,8 +503,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -502,8 +503,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var", "experiment_name/r2i1p1f1/Omon/var",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -521,7 +522,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -521,7 +522,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/", "expid/original_files/cmorfiles/activity/institute/model/"
"experiment_name/",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -539,8 +541,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -539,8 +541,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1", "experiment_name/r2i1p1f1",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -560,8 +562,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -560,8 +562,8 @@ class TestPrimaveraConvention(TestCase):
os.makedirs( os.makedirs(
os.path.join( os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/institute/model/experiment_name/S20000101/" "expid/original_files/cmorfiles/institute/model/"
"mon/ocean/var", "experiment_name/S20000101/mon/ocean/var",
) )
) )
self.assertFalse( self.assertFalse(
...@@ -575,8 +577,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -575,8 +577,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links""" """Test create links"""
member_path = os.path.join( member_path = os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/gn", "experiment_name/r2i1p1f1/Omon/var/gn",
) )
os.makedirs(member_path) os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()} self.config.var_manager.tables = {"Omon": Mock()}
...@@ -591,8 +593,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -591,8 +593,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links with version""" """Test create links with version"""
member_path = os.path.join( member_path = os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/gn/version", "experiment_name/r2i1p1f1/Omon/var/gn/version",
) )
os.makedirs(member_path) os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()} self.config.var_manager.tables = {"Omon": Mock()}
...@@ -609,8 +611,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -609,8 +611,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links with version full startdate""" """Test create links with version full startdate"""
member_path = os.path.join( member_path = os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r2i1p1f1/Omon/var/gn/version", "experiment_name/r2i1p1f1/Omon/var/gn/version",
) )
os.makedirs(member_path) os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()} self.config.var_manager.tables = {"Omon": Mock()}
...@@ -625,8 +627,8 @@ class TestPrimaveraConvention(TestCase): ...@@ -625,8 +627,8 @@ class TestPrimaveraConvention(TestCase):
"""Test create links when the member can not be found""" """Test create links when the member can not be found"""
member_path = os.path.join( member_path = os.path.join(
self.tmp_dir, self.tmp_dir,
"expid/cmorfiles/activity/institute/model/experiment_name/" "expid/original_files/cmorfiles/activity/institute/model/"
"r1i1p1f1/Omon/var/gn", "experiment_name/r1i1p1f1/Omon/var/gn",
) )
os.makedirs(member_path) os.makedirs(member_path)
self.config.var_manager.tables = {"Omon": Mock()} self.config.var_manager.tables = {"Omon": Mock()}
......