From 93888d0fffcbf0dcbeccfece996252e31b66c413 Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Mon, 9 Dec 2024 11:30:41 +0100 Subject: [PATCH 1/4] Added all provenance features to test merge --- autosubmit/autosubmit.py | 40 ++++++++++++++++++++++ autosubmit/provenance/rocrate.py | 4 ++- test/unit/test_provenance.py | 59 ++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 test/unit/test_provenance.py diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 740628161..1586a2fc6 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -626,6 +626,12 @@ class Autosubmit: subparser.add_argument('expid', help='experiment identifier') subparser.add_argument('-v', '--update_version', action='store_true', default=False, help='Update experiment version') + # Provenance + subparser = subparsers.add_parser( + 'provenance', description = 'Produce provenance for autosubmit') + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('--rocrate', action='store_true', default=False, + help='Produce an RO-Crate file') # Archive subparser = subparsers.add_parser( 'archive', description='archives an experiment') @@ -767,6 +773,8 @@ class Autosubmit: return Autosubmit.update_version(args.expid) elif args.command == 'upgrade': return Autosubmit.upgrade_scripts(args.expid,files=args.files) + elif args.command == 'provenance': + return Autosubmit.provenance(args.expid, rocrate=args.rocrate) elif args.command == 'archive': return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress, rocrate=args.rocrate) elif args.command == 'unarchive': @@ -2433,6 +2441,12 @@ class Autosubmit: exp_history.finish_current_experiment_run() except Exception: Log.warning("Database is locked") + # Create rocrate object if requested + rocrate_data = as_conf.experiment_data.get("ROCRATE", None) + if rocrate_data: + Autosubmit.provenance(expid, rocrate=True) + else: + print("rocrate.yml not found in CONFIG. Can't create rocrate object. ") except BaseLockException: terminate_child_process(expid) raise @@ -4256,6 +4270,32 @@ class Autosubmit: from autosubmit.provenance.rocrate import create_rocrate_archive return create_rocrate_archive(as_conf, rocrate_json, jobs, start_time, end_time, path) + + @staticmethod + def provenance(expid, rocrate=False): + """" + :param expid: experiment identifier + :type expid: str + :param rocrate: flag to enable RO-Crate + :type rocrate: bool + """"" + aslogs_folder = Path( + BasicConfig.LOCAL_ROOT_DIR, + expid, + BasicConfig.LOCAL_TMP_DIR, + BasicConfig.LOCAL_ASLOG_DIR + ) + + if rocrate: + try: + Autosubmit.rocrate(expid, Path(aslogs_folder)) + Log.info('RO-Crate ZIP file created!') + except Exception as e: + raise AutosubmitCritical( + f"Error creating RO-Crate ZIP file: {str(e)}", 7012) + else: + raise AutosubmitCritical( + "Can not create RO-Crate ZIP file. Argument '--rocrate' required", 7012) @staticmethod def archive(expid, noclean=True, uncompress=True, rocrate=False): diff --git a/autosubmit/provenance/rocrate.py b/autosubmit/provenance/rocrate.py index cbfac2da5..3f7aa9e8c 100644 --- a/autosubmit/provenance/rocrate.py +++ b/autosubmit/provenance/rocrate.py @@ -23,6 +23,7 @@ import datetime import json import mimetypes import os +import time import subprocess from pathlib import Path from textwrap import dedent @@ -557,6 +558,7 @@ def create_rocrate_archive( crate.add_or_update_jsonld(jsonld_node) # Write RO-Crate ZIP. - crate.write_zip(Path(path, f"{expid}.zip")) + date = time.strftime("%Y%m%d%H%M%S", time.localtime(os.path.getmtime(path))) + crate.write_zip(Path(path, f"{expid}-{date}.zip")) Log.info(f'RO-Crate archive written to {experiment_path}') return crate diff --git a/test/unit/test_provenance.py b/test/unit/test_provenance.py new file mode 100644 index 000000000..43d29272d --- /dev/null +++ b/test/unit/test_provenance.py @@ -0,0 +1,59 @@ +import pytest +from pathlib import Path +from autosubmit.autosubmit import Autosubmit +from log.log import AutosubmitCritical +import os +from unittest.mock import patch + +@pytest.fixture +def mock_paths(tmp_path): + """ + Fixture to set temporary paths for BasicConfig values. + """ + with patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ROOT_DIR', str(tmp_path)), \ + patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_TMP_DIR', 'tmp'), \ + patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ASLOG_DIR', 'ASLOGS'): + yield tmp_path + +def test_provenance_rocrate_success(mock_paths): + """ + Test the provenance function when rocrate=True and the process is successful. + """ + with patch('autosubmit.autosubmit.Autosubmit.rocrate') as mock_rocrate, \ + patch('log.log.Log.info') as mock_log_info: + + expid = "expid123" + exp_folder = os.path.join(str(mock_paths), expid) + tmp_folder = os.path.join(exp_folder, 'tmp') + aslogs_folder = os.path.join(tmp_folder, 'ASLOGS') + expected_aslogs_path = aslogs_folder + + Autosubmit.provenance(expid, rocrate=True) + + mock_rocrate.assert_called_once_with(expid, Path(expected_aslogs_path)) + mock_log_info.assert_called_once_with('RO-Crate ZIP file created!') + +def test_provenance_rocrate_failure(): + """ + Test the provenance function when Autosubmit.rocrate fails + """ + with patch('autosubmit.autosubmit.Autosubmit.rocrate', side_effect=Exception("Mocked exception")) as mock_rocrate: + + with pytest.raises(AutosubmitCritical) as excinfo: + Autosubmit.provenance("expid123", rocrate=True) + + assert "Error creating RO-Crate ZIP file: Mocked exception" in str(excinfo) + + mock_rocrate.assert_called_once() + + +def test_provenance_no_rocrate(): + """ + Test the provenance function when rocrate=False + """ + with patch('autosubmit.autosubmit.Autosubmit.rocrate') as mock_rocrate: + with pytest.raises(AutosubmitCritical) as excinfo: + Autosubmit.provenance("expid123", rocrate=False) + + assert "Can not create RO-Crate ZIP file. Argument '--rocrate' required" in str(excinfo) + mock_rocrate.assert_not_called() -- GitLab From c550e107d1a1e714f04bbf22b0feca276398c8af Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Mon, 9 Dec 2024 11:50:21 +0100 Subject: [PATCH 2/4] Minor changes --- autosubmit/autosubmit.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1586a2fc6..a801238f9 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2444,9 +2444,7 @@ class Autosubmit: # Create rocrate object if requested rocrate_data = as_conf.experiment_data.get("ROCRATE", None) if rocrate_data: - Autosubmit.provenance(expid, rocrate=True) - else: - print("rocrate.yml not found in CONFIG. Can't create rocrate object. ") + Autosubmit.provenance(expid, rocrate=True) except BaseLockException: terminate_child_process(expid) raise -- GitLab From f5f78980388ec36f34800e6a7f5e259daf61af9c Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Mon, 9 Dec 2024 11:51:47 +0100 Subject: [PATCH 3/4] Indentation fixed --- autosubmit/autosubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index a801238f9..81fc7182f 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2444,7 +2444,7 @@ class Autosubmit: # Create rocrate object if requested rocrate_data = as_conf.experiment_data.get("ROCRATE", None) if rocrate_data: - Autosubmit.provenance(expid, rocrate=True) + Autosubmit.provenance(expid, rocrate=True) except BaseLockException: terminate_child_process(expid) raise -- GitLab From 5ba2cfedf1d188d0a24b770419123f266288c313 Mon Sep 17 00:00:00 2001 From: Albert Puiggros Date: Mon, 9 Dec 2024 12:11:37 +0100 Subject: [PATCH 4/4] Try to fix pipeline --- autosubmit/autosubmit.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 81fc7182f..41a808713 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2445,6 +2445,8 @@ class Autosubmit: rocrate_data = as_conf.experiment_data.get("ROCRATE", None) if rocrate_data: Autosubmit.provenance(expid, rocrate=True) + else: + Log.warning("Can't find ROCRATE in YAML configuration") except BaseLockException: terminate_child_process(expid) raise -- GitLab