From ee74b65b75abf4e6d2b3427211860057c1083953 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Mon, 22 Jul 2019 12:29:31 +0200 Subject: [PATCH 1/3] Fixed issue #374: Adding new funcionality to set status + a string representation of Job List --- autosubmit/autosubmit.py | 151 ++++++++++++++++++++++++++++++++++- autosubmit/job/job_common.py | 9 +++ autosubmit/job/job_list.py | 87 +++++++++++++++++++- 3 files changed, 244 insertions(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1983dbce5..065f74954 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -355,6 +355,11 @@ class Autosubmit: "Valid values = ['Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN']") group.add_argument('-ft', '--filter_type', type=str, help='Select the job type to filter the list of jobs') + group.add_argument('-ftc', '--filter_type_chunk', type=str, + help='Supply the list of chunks to change the status. Default = "Any". When the member name "all" is set, all the chunks \ + selected from for that member will be updated for all the members. Example: all [1], will have as a result that the \ + chunks 1 for all the members will be updated. Follow the format: ' + '"[ 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16-30] ] ],SIM,SIM2,SIM3"') subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') @@ -456,7 +461,7 @@ class Autosubmit: return Autosubmit.install() elif args.command == 'setstatus': return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, - args.filter_chunks, args.filter_status, args.filter_type, args.hide, + args.filter_chunks, args.filter_status, args.filter_type, args.filter_type_chunk, args.hide, args.group_by, args.expand, args.expand_status, args.notransitive,args.check_wrapper) elif args.command == 'testcase': return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate, @@ -2619,8 +2624,10 @@ class Autosubmit: job.status = final_status Log.info("CHANGED: job: " + job.name + " status to: " + final) + + @staticmethod - def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, hide, group_by=None, + def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, filter_type_chunk, hide, group_by=None, expand=list(), expand_status=list(), notransitive=False,check_wrapper=False): """ Set status @@ -2688,6 +2695,144 @@ class Autosubmit: jobs_filtered.append(job) else: Autosubmit.change_status(final, final_status, job) + + # New feature : Change status by section, member, and chunk; freely. + # Including inner validation. Trying to make it independent. + if filter_type_chunk: + validation_message = "## -ftc Validation Message ##" + filter_is_correct = True + selected_sections = filter_type_chunk.split(",")[1:] + selected_formula = filter_type_chunk.split(",")[0] + deserializedJson = object() + performed_changes = dict() + + # Starting Validation + if len(selected_sections) == 0: + filter_is_correct = False + validation_message += "\n\tMust include a section (job type). If you want to apply the changes to all sections, include 'Any'." + else: + for section in selected_sections: + # Validating empty sections + if len(section) == 0: + filter_is_correct = False + validation_message += "\n\tEmpty sections are not accepted." + break + # Validating existing sections + # Retrieve experiment data + current_sections = as_conf.get_jobs_sections() + if section not in current_sections and section != "Any": + filter_is_correct = False + validation_message += "\n\tSection " + section + " does not exist in experiment." + + # Validating chunk formula + if len(selected_formula) == 0: + filter_is_correct = False + validation_message += "\n\tA formula for chunk filtering has not been provided. If you want to change all chunks, include 'Any'." + + # If everything is fine until this point + if filter_is_correct == True: + # Retrieve experiment data + current_dates = as_conf._exp_parser.get_option('experiment','DATELIST','').split() + current_members = as_conf.get_member_list() + # Parse json + deserializedJson = json.loads(Autosubmit._create_json(selected_formula)) + for startingDate in deserializedJson['sds']: + if startingDate['sd'] not in current_dates: + filter_is_correct = False + validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." + for member in startingDate['ms']: + if member['m'] not in current_members and member['m'] != "Any": + filter_is_correct_ = False + validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." + + + # Ending validation + if filter_is_correct == False: + print(validation_message) + return False + # If input is valid, continue. + record = dict() + final_list = [] + # Get current list + working_list = job_list.get_job_list() + for section in selected_sections: + if section == "Any": + # Any section + section_selection = working_list + # Go through start dates + for starting_date in deserializedJson['sds']: + date = starting_date['sd'] + date_selection = filter(lambda j: date2str(j.date) == date, section_selection) + # Members for given start date + for member_group in starting_date['ms']: + member = member_group['m'] + if member == "Any": + # Any member + member_selection = date_selection + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + else: + # Selected members + member_selection = filter(lambda j: j.member == member, date_selection) + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + else: + # Only given section + section_selection = filter(lambda j: j.section == section, working_list) + # Go through start dates + for starting_date in deserializedJson['sds']: + date = starting_date['sd'] + date_selection = filter(lambda j: date2str(j.date) == date, section_selection) + # Members for given start date + for member_group in starting_date['ms']: + member = member_group['m'] + if member == "Any": + # Any member + member_selection = date_selection + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + else: + # Selected members + member_selection = filter(lambda j: j.member == member, date_selection) + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + status = Status() + for job in final_list: + if job.status != final_status: + # Only real changes + performed_changes[job.name] = str(Status.VALUE_TO_KEY[job.status]) + " -> " + str(final) + Autosubmit.change_status(final, final_status, job) + # If changes have been performed + if len(performed_changes.keys()) > 0: + if Autosubmit._user_yes_no_query("Would you like to see an extended representation of the changes?") == True: + Log.info(job_list.print_with_status(statusChange = performed_changes)) + else: + Log.warning("No changes were performed.") + # End of New Feature if filter_chunks: if len(jobs_filtered) == 0: @@ -2754,6 +2899,8 @@ class Autosubmit: if job.name in jobs: Autosubmit.change_status(final, final_status, job) + + job_list.update_list(as_conf,False,True) if save and wrongExpid == 0: diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index b2a1fdd96..51205c98d 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -40,6 +40,15 @@ class Status: def retval(self, value): return getattr(self, value) +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' class Type: """ diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index e933cc96f..e99f37143 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -34,7 +34,7 @@ from autosubmit.job.job import Job from bscearth.utils.log import Log from autosubmit.job.job_dict import DicJobs from autosubmit.job.job_utils import Dependency -from autosubmit.job.job_common import Status, Type +from autosubmit.job.job_common import Status, Type, bcolors from bscearth.utils.date import date2str, parse_date, sum_str_hours from autosubmit.job.job_packages import JobPackageSimple, JobPackageArray, JobPackageThread @@ -1050,3 +1050,88 @@ class JobList: if flag: self.update_genealogy(notransitive=notransitive) del self._dic_jobs + + def print_with_status(self, statusChange = None): + """ + Returns the string representation of the dependency tree of + the Job List + + :return: String representation + :rtype: String + """ + allJobs = self.get_all() + # Header + result = bcolors.BOLD + "## String representation of Job List [" + str(len(allJobs)) + "] with " + \ + bcolors.OKGREEN + str(len(statusChange.keys())) + " Changes ##" + bcolors.ENDC + bcolors.ENDC + + # Find root + root = None + for job in allJobs: + if job.has_parents() == False: + root = job + + # root exists + if root is not None: + result += self._recursion_print(root, 0, statusChange = statusChange) + else: + result += "\nCannot find root." + + return result + + def __str__(self): + """ + Returns the string representation of the class. + Usage print(class) + + :return: String representation. + :rtype: String + """ + allJobs = self.get_all() + result = bcolors.BOLD + "## String representation of Job List [" + str(len(allJobs)) + "] ##" + bcolors.ENDC + + # Find root + root = None + for job in allJobs: + if job.has_parents() == False: + root = job + + # root exists + if root is not None: + result += self._recursion_print(root, 0) + else: + result += "\nCannot find root." + + return result + + def _recursion_print(self, job, level, statusChange = None): + """ + Returns the list of children in a recursive way + Traverses the dependency tree + + :return: parent + list of children + :rtype: String + """ + result = "" + prefix = "" + for i in range(level): + prefix += "| " + # Prefix + Job Name + result = "\n"+ prefix + bcolors.BOLD + job.name + bcolors.ENDC + if len(job._children) > 0: + level += 1 + children = job._children + total_children = len(job._children) + # Writes children number + result += " ~ [" + str(total_children) + (" children] " if total_children > 1 else " child] ") + if statusChange is not None: + # Writes change if performed + result += bcolors.BOLD + bcolors.OKGREEN + statusChange[job.name] if job.name in statusChange else "" + result += bcolors.ENDC + bcolors.ENDC + + for child in children: + # Continues recursion + result += self._recursion_print(child, level, statusChange=statusChange) + else: + pass + + return result \ No newline at end of file -- GitLab From 9a93d889e84c803ead51a89f6d89f025beda2cf2 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 25 Jul 2019 09:37:41 +0200 Subject: [PATCH 2/3] Added flag to ftc --- autosubmit/autosubmit.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 065f74954..f39c6397d 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -71,6 +71,7 @@ from job.job_package_persistence import JobPackagePersistence from job.job_list_persistence import JobListPersistenceDb from job.job_list_persistence import JobListPersistencePkl from job.job_grouping import JobGrouping +# from API.testAPI import Monitor # noinspection PyPackageRequirements from bscearth.utils.log import Log from database.db_common import create_db @@ -371,6 +372,7 @@ class Autosubmit: subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded') subparser.add_argument('-nt', '--notransitive', action='store_true', default=False, help='Disable transitive reduction') subparser.add_argument('-cw', '--check_wrapper', action='store_true', default=False, help='Generate possible wrapper in the current workflow') + subparser.add_argument('-d', '--detail', action='store_true', default=False, help='Generate detailed view of changes') # Test Case @@ -462,7 +464,7 @@ class Autosubmit: elif args.command == 'setstatus': return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, args.filter_chunks, args.filter_status, args.filter_type, args.filter_type_chunk, args.hide, - args.group_by, args.expand, args.expand_status, args.notransitive,args.check_wrapper) + args.group_by, args.expand, args.expand_status, args.notransitive,args.check_wrapper, args.detail) elif args.command == 'testcase': return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate, args.HPC, args.branch) @@ -2628,7 +2630,7 @@ class Autosubmit: @staticmethod def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, filter_type_chunk, hide, group_by=None, - expand=list(), expand_status=list(), notransitive=False,check_wrapper=False): + expand=list(), expand_status=list(), notransitive=False,check_wrapper=False, detail=False): """ Set status @@ -2828,7 +2830,7 @@ class Autosubmit: Autosubmit.change_status(final, final_status, job) # If changes have been performed if len(performed_changes.keys()) > 0: - if Autosubmit._user_yes_no_query("Would you like to see an extended representation of the changes?") == True: + if detail == True: Log.info(job_list.print_with_status(statusChange = performed_changes)) else: Log.warning("No changes were performed.") @@ -3274,6 +3276,7 @@ class Autosubmit: @staticmethod def load_job_list(expid, as_conf, notransitive=False,monitor=False): + BasicConfig.read() rerun = as_conf.get_rerun() job_list = JobList(expid, BasicConfig, ConfigParserFactory(), Autosubmit._get_job_list_persistence(expid, as_conf)) -- GitLab From 11df4499ef668ab35e156eca5ddfd4afd681a6cd Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 25 Jul 2019 14:46:42 +0200 Subject: [PATCH 3/3] Added small changes in validation process --- autosubmit/autosubmit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index f39c6397d..5957adbce 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2709,13 +2709,13 @@ class Autosubmit: performed_changes = dict() # Starting Validation - if len(selected_sections) == 0: + if len(str(selected_sections).strip()) == 0: filter_is_correct = False validation_message += "\n\tMust include a section (job type). If you want to apply the changes to all sections, include 'Any'." else: for section in selected_sections: # Validating empty sections - if len(section) == 0: + if len(str(section).strip()) == 0: filter_is_correct = False validation_message += "\n\tEmpty sections are not accepted." break -- GitLab