From 5f260906e751dc692783a65228b27d6c32b0f2a0 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 14 Mar 2024 12:09:05 +0100 Subject: [PATCH 1/3] Fix "previous" when there is only a single job --- autosubmit/job/job_dict.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index ec22a2a25..6baee369c 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -497,13 +497,19 @@ class DicJobs: end = len(final_jobs_list) final_jobs_list_special = final_jobs_list[split_index:end] if "previous" in filters_to_of_parent.get("SPLITS_TO", ""): - final_jobs_list_special = [final_jobs_list_special[-1]] + if type(final_jobs_list_special) is list: + final_jobs_list_special = [final_jobs_list_special[-1]] + else: + final_jobs_list_special = final_jobs_list_special else: # get 1-N (part 2) my_complete_slice = matches[0].strip(",").split("*") split_index = int(my_complete_slice[0]) - 1 final_jobs_list_special = final_jobs_list[split_index] if "previous" in filters_to_of_parent.get("SPLITS_TO", ""): - final_jobs_list_special = [final_jobs_list_special[-1]] + if type(final_jobs_list_special) is list: + final_jobs_list_special = [final_jobs_list_special[-1]] + else: + final_jobs_list_special = final_jobs_list_special else: final_jobs_list_special = [] if type(final_jobs_list_special) is not list: @@ -626,6 +632,7 @@ class DicJobs: section_data.append(job) self.changes["NEWJOBS"] = True else: + self._job_list[name].update_dict_parameters(self.as_conf) self._job_list[name].status = Status.WAITING if self._job_list[name].status in [Status.DELAYED, Status.PREPARED, Status.READY] else \ -- GitLab From 7417aacf7b3f8bfa6a34b9747f64c55b9225db3c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 19 Mar 2024 13:25:09 +0100 Subject: [PATCH 2/3] fixed test, added test --- autosubmit/autosubmit.py | 2 ++ test/unit/test_catlog.py | 3 +-- test/unit/test_dependencies.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 53e7f528a..53f7dd037 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -657,6 +657,8 @@ class Autosubmit: if unknown or args.command is None: parser.print_help() return 1 + except SystemExit as e: + return 1 except BaseException as e: raise AutosubmitCritical( "Incorrect arguments for this command", 7011) diff --git a/test/unit/test_catlog.py b/test/unit/test_catlog.py index 86dc6ae83..f7b2be15e 100644 --- a/test/unit/test_catlog.py +++ b/test/unit/test_catlog.py @@ -120,7 +120,6 @@ class TestJob(TestCase): def test_command_line_help(self): args = ['autosubmit', 'cat-log', '--help'] with patch.object(sys, 'argv', args) as _, io.StringIO() as buf, redirect_stdout(buf): - with suppress(SystemExit): - assert Autosubmit.parse_args() + assert Autosubmit.parse_args() assert buf assert 'View workflow and job logs.' in buf.getvalue() diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 21938bec0..daf6e6877 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -7,6 +7,7 @@ import tempfile import unittest from copy import deepcopy from datetime import datetime +from mock import patch from autosubmit.job.job_dict import DicJobs from autosubmit.job.job import Job @@ -435,5 +436,38 @@ class TestJobList(unittest.TestCase): job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent2) self.assertEqual(len(job.edge_info.get("RUNNING", "")), 2) + @patch('autosubmit.job.job_dict.date2str') + def test_jobdict_get_jobs_filtered(self, mock_date2str): + # Test the get_jobs_filtered function + # DicJobs.get_jobs_filtered(self, section, job, filters_to, natural_date, natural_member, natural_chunk,filters_to_of_parent) + as_conf = mock.Mock() + as_conf.experiment_data = dict() + as_conf.experiment_data = {'CONFIG': {'AUTOSUBMIT_VERSION': '4.1.2', 'MAXWAITINGJOBS': 20, 'TOTALJOBS': 20, 'SAFETYSLEEPTIME': 10, 'RETRIALS': 0}, 'MAIL': {'NOTIFICATIONS': False, 'TO': None}, 'STORAGE': {'TYPE': 'pkl', 'COPY_REMOTE_LOGS': True}, 'DEFAULT': {'EXPID': 'a03b', 'HPCARCH': 'marenostrum4'}, 'EXPERIMENT': {'DATELIST': '20000101', 'MEMBERS': 'fc0', 'CHUNKSIZEUNIT': 'month', 'CHUNKSIZE': 4, 'NUMCHUNKS': 5, 'CHUNKINI': '', 'CALENDAR': 'standard'}, 'PROJECT': {'PROJECT_TYPE': 'none', 'PROJECT_DESTINATION': ''}, 'GIT': {'PROJECT_ORIGIN': '', 'PROJECT_BRANCH': '', 'PROJECT_COMMIT': '', 'PROJECT_SUBMODULES': '', 'FETCH_SINGLE_BRANCH': True}, 'SVN': {'PROJECT_URL': '', 'PROJECT_REVISION': ''}, 'LOCAL': {'PROJECT_PATH': ''}, 'PROJECT_FILES': {'FILE_PROJECT_CONF': '', 'FILE_JOBS_CONF': '', 'JOB_SCRIPTS_TYPE': ''}, 'RERUN': {'RERUN': False, 'RERUN_JOBLIST': ''}, 'JOBS': {'SIM': {'FILE': 'SIM.sh', 'RUNNING': 'once', 'DEPENDENCIES': {'SIM-1': {'CHUNKS_FROM': {'ALL': {'SPLITS_TO': 'previous'}}}}, 'WALLCLOCK': '00:05', 'SPLITS': 10, 'ADDITIONAL_FILES': []}, 'TEST': {'FILE': 'Test.sh', 'DEPENDENCIES': {'TEST-1': {'CHUNKS_FROM': {'ALL': {'SPLITS_TO': 'previous'}}}, 'SIM': None}, 'RUNNING': 'once', 'WALLCLOCK': '00:05', 'SPLITS': 10, 'ADDITIONAL_FILES': []}}, 'PLATFORMS': {'MARENOSTRUM4': {'TYPE': 'slurm', 'HOST': 'mn2.bsc.es', 'PROJECT': 'bsc32', 'USER': 'bsc32070', 'QUEUE': 'debug', 'SCRATCH_DIR': '/gpfs/scratch', 'ADD_PROJECT_TO_HOST': False, 'MAX_WALLCLOCK': '48:00', 'TEMP_DIR': ''}, 'MARENOSTRUM_ARCHIVE': {'TYPE': 'ps', 'HOST': 'dt02.bsc.es', 'PROJECT': 'bsc32', 'USER': None, 'SCRATCH_DIR': '/gpfs/scratch', 'ADD_PROJECT_TO_HOST': False, 'TEST_SUITE': False}, 'TRANSFER_NODE': {'TYPE': 'ps', 'HOST': 'dt01.bsc.es', 'PROJECT': 'bsc32', 'USER': None, 'ADD_PROJECT_TO_HOST': False, 'SCRATCH_DIR': '/gpfs/scratch'}, 'TRANSFER_NODE_BSCEARTH000': {'TYPE': 'ps', 'HOST': 'bscearth000', 'USER': None, 'PROJECT': 'Earth', 'ADD_PROJECT_TO_HOST': False, 'QUEUE': 'serial', 'SCRATCH_DIR': '/esarchive/scratch'}, 'BSCEARTH000': {'TYPE': 'ps', 'HOST': 'bscearth000', 'USER': None, 'PROJECT': 'Earth', 'ADD_PROJECT_TO_HOST': False, 'QUEUE': 'serial', 'SCRATCH_DIR': '/esarchive/scratch'}, 'NORD3': {'TYPE': 'SLURM', 'HOST': 'nord1.bsc.es', 'PROJECT': 'bsc32', 'USER': None, 'QUEUE': 'debug', 'SCRATCH_DIR': '/gpfs/scratch', 'MAX_WALLCLOCK': '48:00'}, 'ECMWF-XC40': {'TYPE': 'ecaccess', 'VERSION': 'pbs', 'HOST': 'cca', 'USER': None, 'PROJECT': 'spesiccf', 'ADD_PROJECT_TO_HOST': False, 'SCRATCH_DIR': '/scratch/ms', 'QUEUE': 'np', 'SERIAL_QUEUE': 'ns', 'MAX_WALLCLOCK': '48:00'}}, 'ROOTDIR': '/home/dbeltran/new_autosubmit/a03b', 'PROJDIR': '/home/dbeltran/new_autosubmit/a03b/proj/'} + as_conf.jobs_data = as_conf.experiment_data["JOBS"] + as_conf.last_experiment_data = as_conf.experiment_data + as_conf.detailed_deep_diff = Mock() + as_conf.detailed_deep_diff.return_value = {} + self.dictionary = DicJobs(self.date_list, self.member_list, self.chunk_list, "", default_retrials=0,as_conf=as_conf) + self.dictionary.read_section("SIM", 1, "bash") + job = Job("SIM", 1, Status.READY, 1) + job.date = None + job.member = None + job.chunk = None + job.running = "once" + job.split = 2 + job.splits = 2 + job.max_checkpoint_step = 0 + job_list = Mock(wraps=self.JobList) + job_list._job_list = [job] + filters_to = {'SPLITS_TO': 'previous'} + filters_to_of_parent = {} + natural_chunk = 1 + natural_member = 'fc0' + section = 'SIM' + result = self.dictionary.get_jobs_filtered(section, job, filters_to, None, natural_member, natural_chunk, filters_to_of_parent) + expected_output = [self.dictionary._dic["SIM"][0]] + self.assertEqual(expected_output, result) + + if __name__ == '__main__': unittest.main() -- GitLab From e597eb4d8a52a743beeb00593e77787b4fbebe89 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 20 Mar 2024 14:36:37 +0100 Subject: [PATCH 3/3] Changed test --- test/unit/test_dependencies.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index daf6e6877..2bb912111 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -454,13 +454,13 @@ class TestJobList(unittest.TestCase): job.member = None job.chunk = None job.running = "once" - job.split = 2 + job.split = 1 job.splits = 2 job.max_checkpoint_step = 0 job_list = Mock(wraps=self.JobList) job_list._job_list = [job] - filters_to = {'SPLITS_TO': 'previous'} - filters_to_of_parent = {} + filters_to = {'SPLITS_TO': "1*\\1"} + filters_to_of_parent = {'SPLITS_TO': 'previous'} natural_chunk = 1 natural_member = 'fc0' section = 'SIM' @@ -469,5 +469,6 @@ class TestJobList(unittest.TestCase): self.assertEqual(expected_output, result) + if __name__ == '__main__': unittest.main() -- GitLab