From 36991eb73e6c2b241dabe236c2520ba8a55155d7 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 28 Oct 2020 10:16:41 +0100 Subject: [PATCH 1/3] Added RUN_ONLY_MEMBERS setting and filter logic. --- autosubmit/autosubmit.py | 8 +++++--- autosubmit/config/config_common.py | 5 +++-- autosubmit/config/files/expdef.conf | 2 ++ autosubmit/job/job_list.py | 16 +++++++++++++++- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 8f22a65c4..b6efc6ce1 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3188,6 +3188,8 @@ class Autosubmit: num_chunks = as_conf.get_num_chunks() chunk_ini = as_conf.get_chunk_ini() member_list = as_conf.get_member_list() + run_only_members = as_conf.get_member_list(run_only=True) + print("Run only members {0}".format(run_only_members)) if len(member_list) != len(set(member_list)): raise AutosubmitCritical( "There are repeated member names!") @@ -3208,7 +3210,7 @@ class Autosubmit: job_list.generate(date_list, member_list, num_chunks, chunk_ini, parameters, date_format, as_conf.get_retrials(), as_conf.get_default_job_type(), - as_conf.get_wrapper_type(), as_conf.get_wrapper_jobs(), notransitive=notransitive, update_structure=True) + as_conf.get_wrapper_type(), as_conf.get_wrapper_jobs(), notransitive=notransitive, update_structure=True, run_only_members=run_only_members) if rerun == "true": chunk_list = Autosubmit._create_json( @@ -4359,7 +4361,7 @@ class Autosubmit: job_list = JobList(expid, BasicConfig, ConfigParserFactory(), Autosubmit._get_job_list_persistence(expid, as_conf)) - + run_only_members = as_conf.get_member_list(run_only=True) date_list = as_conf.get_date_list() date_format = '' if as_conf.get_chunk_size_unit() is 'hour': @@ -4372,7 +4374,7 @@ class Autosubmit: job_list.generate(date_list, as_conf.get_member_list(), as_conf.get_num_chunks(), as_conf.get_chunk_ini(), as_conf.load_parameters(), date_format, as_conf.get_retrials(), as_conf.get_default_job_type(), as_conf.get_wrapper_type(), as_conf.get_wrapper_jobs(), - new=False, notransitive=notransitive) + new=False, notransitive=notransitive, run_only_members=run_only_members) if rerun == "true": chunk_list = Autosubmit._create_json(as_conf.get_chunk_list()) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index d3bd39090..3149a5e09 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -1067,7 +1067,7 @@ class AutosubmitConfig(object): return default return int(chunk_size) - def get_member_list(self): + def get_member_list(self, run_only=False): """ Returns members list from experiment's config file @@ -1075,7 +1075,8 @@ class AutosubmitConfig(object): :rtype: list """ member_list = list() - string = self._exp_parser.get('experiment', 'MEMBERS') + string = self._exp_parser.get('experiment', 'MEMBERS') if run_only == False else self._exp_parser.get_option( + 'experiment', 'RUN_ONLY_MEMBERS', '') if not string.startswith("["): string = '[{0}]'.format(string) split_string = nestedExpr('[', ']').parseString(string).asList() diff --git a/autosubmit/config/files/expdef.conf b/autosubmit/config/files/expdef.conf index a5512812c..986910def 100644 --- a/autosubmit/config/files/expdef.conf +++ b/autosubmit/config/files/expdef.conf @@ -30,6 +30,8 @@ NUMCHUNKS = CHUNKINI = # Calendar used. LIST: standard, noleap CALENDAR = standard +# List of members that can be included in this run. Optional. +RUN_ONLY_MEMBERS = [project] # Select project type. STRING = git, svn, local, none diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 4e030b88b..30ecb2db0 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -121,7 +121,7 @@ class JobList(object): # print(job.parents) def generate(self, date_list, member_list, num_chunks, chunk_ini, parameters, date_format, default_retrials, - default_job_type, wrapper_type=None, wrapper_jobs=None, new=True, notransitive=False, update_structure=False): + default_job_type, wrapper_type=None, wrapper_jobs=None, new=True, notransitive=False, update_structure=False, run_only_members=[]): """ Creates all jobs needed for the current workflow @@ -183,6 +183,20 @@ class JobList(object): for job in self._job_list: job.parameters = parameters + # Checking for member constraints + if len(run_only_members) > 0: + # Found + Log.info("Considering only members {0}".format( + str(run_only_members))) + old_job_list = [job for job in self._job_list] + self._job_list = [ + job for job in old_job_list if job.member is None or job.member in run_only_members or job.status not in [Status.WAITING, Status.READY]] + for job in self._job_list: + job.parents = [ + jobp for jobp in job.parents if jobp in self._job_list] + job.children = [ + jobc for jobc in job._children if jobc in self._job_list] + # Perhaps this should be done by default independent of the wrapper_type supplied if wrapper_type == 'vertical-mixed': self._ordered_jobs_by_date_member = self._create_sorted_dict_jobs( -- GitLab From 648f789153e05dc3e989f44e8cf499138c6833a1 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 28 Oct 2020 15:06:18 +0100 Subject: [PATCH 2/3] Removed print of run only members --- autosubmit/autosubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index b6efc6ce1..b5b112240 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3189,7 +3189,7 @@ class Autosubmit: chunk_ini = as_conf.get_chunk_ini() member_list = as_conf.get_member_list() run_only_members = as_conf.get_member_list(run_only=True) - print("Run only members {0}".format(run_only_members)) + # print("Run only members {0}".format(run_only_members)) if len(member_list) != len(set(member_list)): raise AutosubmitCritical( "There are repeated member names!") -- GitLab From c2d9b653038347d9fd2296e64a70c2266049a2c9 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Mon, 2 Nov 2020 15:52:01 +0100 Subject: [PATCH 3/3] Significant job status change message is now on debug log --- autosubmit/database/db_jobdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/database/db_jobdata.py b/autosubmit/database/db_jobdata.py index 63c4cf289..da40e5be7 100644 --- a/autosubmit/database/db_jobdata.py +++ b/autosubmit/database/db_jobdata.py @@ -795,7 +795,7 @@ class JobDataStructure(MainDataBase): if len(tracking_dictionary.keys()) >= int(current_date_member_completed_count * 0.9): # If setstatus changes more than 90% of date-member completed jobs, it's a new run # Must create a new experiment run - Log.result( + Log.debug( "Since a significant amount of jobs have changed status. Autosubmit will consider a new run of the same experiment.") self.validate_current_run( job_list, chunk_unit, chunk_size, True) -- GitLab