diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 6bc39f8627db2dba04db63e5d975ad57b756e3e4..a7fb1573bd27bba5605e23a0f69dd77251308080 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -626,6 +626,12 @@ class Autosubmit: subparser.add_argument('expid', help='experiment identifier') subparser.add_argument('-v', '--update_version', action='store_true', default=False, help='Update experiment version') + # Provenance + subparser = subparsers.add_parser( + 'provenance', description = 'Produce provenance for autosubmit') + subparser.add_argument('expid', help='experiment identifier') + subparser.add_argument('--rocrate', action='store_true', default=False, + help='Produce an RO-Crate file') # Archive subparser = subparsers.add_parser( 'archive', description='archives an experiment') @@ -649,7 +655,7 @@ class Autosubmit: subparser.add_argument('-v', '--update_version', action='store_true', default=False, help='Update experiment version') subparser.add_argument('--rocrate', action='store_true', default=False, - help='Unarchive an RO-Crate file') + help='Unarchive an RO-Crate file') # update proj files subparser = subparsers.add_parser('upgrade', description='Updates autosubmit 3 proj files to autosubmit 4') subparser.add_argument('expid', help='experiment identifier') @@ -767,8 +773,10 @@ class Autosubmit: return Autosubmit.update_version(args.expid) elif args.command == 'upgrade': return Autosubmit.upgrade_scripts(args.expid,files=args.files) + elif args.command == 'provenance': + return Autosubmit.provenance(args.expid, rocrate=args.rocrate) elif args.command == 'archive': - return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress, rocrate=args.rocrate) + return Autosubmit.archive(args.expid, noclean=args.noclean, uncompress=args.uncompress, rocrate=args.rocrate) elif args.command == 'unarchive': return Autosubmit.unarchive(args.expid, uncompressed=args.uncompressed, rocrate=args.rocrate) @@ -2381,6 +2389,12 @@ class Autosubmit: exp_history.finish_current_experiment_run() except: Log.warning("Database is locked") + ### Create rocrate object if requested + rocrate_data = as_conf.experiment_data.get("ROCRATE") + if rocrate_data: + Autosubmit.provenance(expid, rocrate=True) + else: + print("rocrate.yml not found in CONFIG. Can't create rocrate object.") except (portalocker.AlreadyLocked, portalocker.LockException) as e: message = "We have detected that there is another Autosubmit instance using the experiment\n. Stop other Autosubmit instances that are using the experiment or delete autosubmit.lock file located on tmp folder" terminate_child_process(expid) @@ -4219,7 +4233,33 @@ class Autosubmit: from autosubmit.provenance.rocrate import create_rocrate_archive return create_rocrate_archive(as_conf, rocrate_json, jobs, start_time, end_time, path) + + @staticmethod + def provenance(expid, rocrate=False): + """" + :param expid: experiment identifier + :type expid: str + :param rocrate: flag to enable RO-Crate + :type rocrate: bool + """"" + aslogs_folder = Path( + BasicConfig.LOCAL_ROOT_DIR, + expid, + BasicConfig.LOCAL_TMP_DIR, + BasicConfig.LOCAL_ASLOG_DIR + ) + if rocrate: + try: + Autosubmit.rocrate(expid, Path(aslogs_folder)) + Log.info('RO-Crate ZIP file created!') + except Exception as e: + raise AutosubmitCritical( + f"Error creating RO-Crate ZIP file: {str(e)}", 7012) + else: + raise AutosubmitCritical( + "Can not create RO-Crate ZIP file. Argument '--rocrate' required", 7012) + @staticmethod def archive(expid, noclean=True, uncompress=True, rocrate=False): """ @@ -4314,6 +4354,7 @@ class Autosubmit: Log.result("Experiment archived successfully") return True + @staticmethod def unarchive(experiment_id, uncompressed=True, rocrate=False): """ @@ -4383,6 +4424,7 @@ class Autosubmit: Log.result("Experiment {0} unarchived successfully", experiment_id) return True + @staticmethod def _create_project_associated_conf(as_conf, force_model_conf, force_jobs_conf): project_destiny = as_conf.get_file_project_conf() diff --git a/autosubmit/provenance/rocrate.py b/autosubmit/provenance/rocrate.py index de77b3e5ba48009c0d51473af37e50bfd0e9f2f1..027fc2b9fb8cfb38a928d941e682788c2cfb8c36 100644 --- a/autosubmit/provenance/rocrate.py +++ b/autosubmit/provenance/rocrate.py @@ -23,6 +23,7 @@ import datetime import json import mimetypes import os +import time import subprocess from pathlib import Path from textwrap import dedent @@ -545,7 +546,7 @@ def create_rocrate_archive( encoding_format=None, exampleOfWork={'@id': formal_parameter['@id']}) create_action.append_to('result', {'@id': file_entity['@id']}) - + # Merge with user provided values. # NOTE: It is important that this call happens after the JSON-LD has # been constructed by ro-crate-py, as methods like ``add`` will @@ -555,8 +556,13 @@ def create_rocrate_archive( patch = json.loads(rocrate_json['PATCH']) for jsonld_node in patch['@graph']: crate.add_or_update_jsonld(jsonld_node) - + # Write RO-Crate ZIP. - crate.write_zip(Path(path, f"{expid}.zip")) + #What date/time we want to use to define the zip file? I guess it should be the LAST modification time, right? + #Should I re-use the code in archive() using the full date instead of only the year? The problem is that the + #zip is defined in this function so all the (modified) code should be moved here. Still, I'm exploring other posibilites + #to query the las modified time within this function (Not very used to the code still...) + date = time.strftime("%Y%m%d%H%M%S", time.localtime(os.path.getmtime(path))) + crate.write_zip(Path(path, f"{expid}-{date}.zip")) Log.info(f'RO-Crate archive written to {experiment_path}') return crate diff --git a/metadata/data/job_data_a000.sql b/metadata/data/job_data_a000.sql new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/unit/test_provenance.py b/test/unit/test_provenance.py new file mode 100644 index 0000000000000000000000000000000000000000..43d29272d858afad0171e9549ea749d77abe943f --- /dev/null +++ b/test/unit/test_provenance.py @@ -0,0 +1,59 @@ +import pytest +from pathlib import Path +from autosubmit.autosubmit import Autosubmit +from log.log import AutosubmitCritical +import os +from unittest.mock import patch + +@pytest.fixture +def mock_paths(tmp_path): + """ + Fixture to set temporary paths for BasicConfig values. + """ + with patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ROOT_DIR', str(tmp_path)), \ + patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_TMP_DIR', 'tmp'), \ + patch('autosubmitconfigparser.config.basicconfig.BasicConfig.LOCAL_ASLOG_DIR', 'ASLOGS'): + yield tmp_path + +def test_provenance_rocrate_success(mock_paths): + """ + Test the provenance function when rocrate=True and the process is successful. + """ + with patch('autosubmit.autosubmit.Autosubmit.rocrate') as mock_rocrate, \ + patch('log.log.Log.info') as mock_log_info: + + expid = "expid123" + exp_folder = os.path.join(str(mock_paths), expid) + tmp_folder = os.path.join(exp_folder, 'tmp') + aslogs_folder = os.path.join(tmp_folder, 'ASLOGS') + expected_aslogs_path = aslogs_folder + + Autosubmit.provenance(expid, rocrate=True) + + mock_rocrate.assert_called_once_with(expid, Path(expected_aslogs_path)) + mock_log_info.assert_called_once_with('RO-Crate ZIP file created!') + +def test_provenance_rocrate_failure(): + """ + Test the provenance function when Autosubmit.rocrate fails + """ + with patch('autosubmit.autosubmit.Autosubmit.rocrate', side_effect=Exception("Mocked exception")) as mock_rocrate: + + with pytest.raises(AutosubmitCritical) as excinfo: + Autosubmit.provenance("expid123", rocrate=True) + + assert "Error creating RO-Crate ZIP file: Mocked exception" in str(excinfo) + + mock_rocrate.assert_called_once() + + +def test_provenance_no_rocrate(): + """ + Test the provenance function when rocrate=False + """ + with patch('autosubmit.autosubmit.Autosubmit.rocrate') as mock_rocrate: + with pytest.raises(AutosubmitCritical) as excinfo: + Autosubmit.provenance("expid123", rocrate=False) + + assert "Can not create RO-Crate ZIP file. Argument '--rocrate' required" in str(excinfo) + mock_rocrate.assert_not_called()