From ac0e09c4f4a1bee17333699b7e117cbb4ab1084e Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Tue, 16 Jul 2019 17:04:59 +0200 Subject: [PATCH 01/10] Fixed issue #372. Now autosubmit stops when setstatus parameters are not found, it also returns some details to the user. --- autosubmit/autosubmit.py | 54 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1983dbce5..ac29623f2 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2669,7 +2669,61 @@ class Autosubmit: Log.critical('Can not run with invalid configuration') return False + + #Verifying job sections, if filter_section has been set: + section_no_found = False + section_no_foundList = list() + if filter_section is not None: + if len(filter_section.split()) > 0: + jobSections = as_conf.get_jobs_sections() + for section in filter_section.split(): + if section not in jobSections: + section_no_found = True + section_no_foundList.append(section) + if section_no_found == True: + Log.warning("Specified section(s) : [%s] not found in the experiment %s. \nProcess stopped. Comparison is case sensitive." % (map(str,section_no_foundList), expid)) + return False + + job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) + + #Verifying list of jobs, if filter_list has been set: + #Seems that load_job_list call is necessary before verification is executed + #Not case sensitive + job_no_found = False + job_no_foundList = list() + #Building a simple list of job names + jobs = list() + if job_list is not None: + for job in job_list.get_job_list(): + jobs.append(job.name) + + if lst is not None: + #print(jobs) + if len(lst.split()) > 0: + for sentJob in lst.split(): + if sentJob not in jobs: + job_no_found = True + job_no_foundList.append(sentJob) + + if job_no_found == True: + Log.warning("Specified job(s) : [%s] not found in the experiment %s. \nProcess stopped. Comparison is case sensitive." % (map(str,job_no_foundList), expid)) + return False + + #Verifying format of fc input filter_chunks. Simple format verification. + incorrect_fc_format = False + if filter_chunks is not None: + count_open = filter_chunks.count("[") + count_close = filter_chunks.count("]") + start_sentence = filter_chunks[0] + end_sentence = filter_chunks[-1] + #print(start_sentence, end_sentence) + if count_open == 0 or count_close == 0 or count_open != count_close or start_sentence != "[" or end_sentence != "]": + Log.warning('Format for -fc chunk list input is not correct. \nProcess stopped. Review format: "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + return False + + + #job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) jobs_filtered =[] final_status = Autosubmit._get_status(final) if filter_section or filter_chunks: -- GitLab From ff5b4c5ead9bfbbd7155de3e2a04eec8fb15555f Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 17 Jul 2019 15:12:30 +0200 Subject: [PATCH 02/10] Some var names have been modified. warning replaced by critical and input format improved. --- autosubmit/autosubmit.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index ac29623f2..1170e3a47 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2671,17 +2671,17 @@ class Autosubmit: #Verifying job sections, if filter_section has been set: - section_no_found = False - section_no_foundList = list() + section_not_found = False + section_not_foundList = list() if filter_section is not None: if len(filter_section.split()) > 0: jobSections = as_conf.get_jobs_sections() for section in filter_section.split(): if section not in jobSections: - section_no_found = True - section_no_foundList.append(section) - if section_no_found == True: - Log.warning("Specified section(s) : [%s] not found in the experiment %s. \nProcess stopped. Comparison is case sensitive." % (map(str,section_no_foundList), expid)) + section_not_found = True + section_not_foundList.append(section) + if section_not_found == True: + Log.critical("Specified section(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive.", map(str,section_not_foundList), expid) return False @@ -2689,9 +2689,8 @@ class Autosubmit: #Verifying list of jobs, if filter_list has been set: #Seems that load_job_list call is necessary before verification is executed - #Not case sensitive - job_no_found = False - job_no_foundList = list() + job_not_found = False + job_not_foundList = list() #Building a simple list of job names jobs = list() if job_list is not None: @@ -2703,15 +2702,14 @@ class Autosubmit: if len(lst.split()) > 0: for sentJob in lst.split(): if sentJob not in jobs: - job_no_found = True - job_no_foundList.append(sentJob) + job_not_found = True + job_not_foundList.append(sentJob) - if job_no_found == True: - Log.warning("Specified job(s) : [%s] not found in the experiment %s. \nProcess stopped. Comparison is case sensitive." % (map(str,job_no_foundList), expid)) + if job_not_found == True: + Log.critical("Specified job(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive." % map(str,job_not_foundList), expid) return False - #Verifying format of fc input filter_chunks. Simple format verification. - incorrect_fc_format = False + #Verifying format of fc input filter_chunks. Simple format verification. if filter_chunks is not None: count_open = filter_chunks.count("[") count_close = filter_chunks.count("]") @@ -2719,11 +2717,11 @@ class Autosubmit: end_sentence = filter_chunks[-1] #print(start_sentence, end_sentence) if count_open == 0 or count_close == 0 or count_open != count_close or start_sentence != "[" or end_sentence != "]": - Log.warning('Format for -fc chunk list input is not correct. \nProcess stopped. Review format: "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + Log.critical('Format for -fc chunk list input is not correct. \nProcess stopped. Review format: "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') return False - #job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) + jobs_filtered =[] final_status = Autosubmit._get_status(final) if filter_section or filter_chunks: -- GitLab From d0a87e780b168a65a2fcd7cac14106364b41b0ba Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Fri, 19 Jul 2019 12:34:46 +0200 Subject: [PATCH 03/10] Removing fc verification, it was too restrictive. Including 'Any' keyword in filter_section and filter_job(lst) verification. --- autosubmit/autosubmit.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1170e3a47..1a46b896e 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2674,14 +2674,15 @@ class Autosubmit: section_not_found = False section_not_foundList = list() if filter_section is not None: - if len(filter_section.split()) > 0: + if len(filter_section.split()) > 0: jobSections = as_conf.get_jobs_sections() for section in filter_section.split(): - if section not in jobSections: + # Provided section is not an existing section or it is not the keyword 'Any' + if section not in jobSections and (section != "Any"): section_not_found = True section_not_foundList.append(section) if section_not_found == True: - Log.critical("Specified section(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive.", map(str,section_not_foundList), expid) + Log.critical("Specified section(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive.", section_not_foundList, expid) return False @@ -2701,25 +2702,14 @@ class Autosubmit: #print(jobs) if len(lst.split()) > 0: for sentJob in lst.split(): - if sentJob not in jobs: + # Provided job does not exist or it is not the keyword 'Any' + if sentJob not in jobs and (sentJob != "Any"): job_not_found = True job_not_foundList.append(sentJob) if job_not_found == True: - Log.critical("Specified job(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive." % map(str,job_not_foundList), expid) - return False - - #Verifying format of fc input filter_chunks. Simple format verification. - if filter_chunks is not None: - count_open = filter_chunks.count("[") - count_close = filter_chunks.count("]") - start_sentence = filter_chunks[0] - end_sentence = filter_chunks[-1] - #print(start_sentence, end_sentence) - if count_open == 0 or count_close == 0 or count_open != count_close or start_sentence != "[" or end_sentence != "]": - Log.critical('Format for -fc chunk list input is not correct. \nProcess stopped. Review format: "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') - return False - + Log.critical("Specified job(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive.", job_not_foundList, expid) + return False jobs_filtered =[] @@ -2797,7 +2787,6 @@ class Autosubmit: if wrongExpid > 0: Log.warning("There are {0} job.name with an invalid Expid",wrongExpid) - if jobs == 'Any': for job in job_list.get_job_list(): Autosubmit.change_status(final, final_status, job) -- GitLab From ee74b65b75abf4e6d2b3427211860057c1083953 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Mon, 22 Jul 2019 12:29:31 +0200 Subject: [PATCH 04/10] Fixed issue #374: Adding new funcionality to set status + a string representation of Job List --- autosubmit/autosubmit.py | 151 ++++++++++++++++++++++++++++++++++- autosubmit/job/job_common.py | 9 +++ autosubmit/job/job_list.py | 87 +++++++++++++++++++- 3 files changed, 244 insertions(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1983dbce5..065f74954 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -355,6 +355,11 @@ class Autosubmit: "Valid values = ['Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN']") group.add_argument('-ft', '--filter_type', type=str, help='Select the job type to filter the list of jobs') + group.add_argument('-ftc', '--filter_type_chunk', type=str, + help='Supply the list of chunks to change the status. Default = "Any". When the member name "all" is set, all the chunks \ + selected from for that member will be updated for all the members. Example: all [1], will have as a result that the \ + chunks 1 for all the members will be updated. Follow the format: ' + '"[ 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16-30] ] ],SIM,SIM2,SIM3"') subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') @@ -456,7 +461,7 @@ class Autosubmit: return Autosubmit.install() elif args.command == 'setstatus': return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, - args.filter_chunks, args.filter_status, args.filter_type, args.hide, + args.filter_chunks, args.filter_status, args.filter_type, args.filter_type_chunk, args.hide, args.group_by, args.expand, args.expand_status, args.notransitive,args.check_wrapper) elif args.command == 'testcase': return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate, @@ -2619,8 +2624,10 @@ class Autosubmit: job.status = final_status Log.info("CHANGED: job: " + job.name + " status to: " + final) + + @staticmethod - def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, hide, group_by=None, + def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, filter_type_chunk, hide, group_by=None, expand=list(), expand_status=list(), notransitive=False,check_wrapper=False): """ Set status @@ -2688,6 +2695,144 @@ class Autosubmit: jobs_filtered.append(job) else: Autosubmit.change_status(final, final_status, job) + + # New feature : Change status by section, member, and chunk; freely. + # Including inner validation. Trying to make it independent. + if filter_type_chunk: + validation_message = "## -ftc Validation Message ##" + filter_is_correct = True + selected_sections = filter_type_chunk.split(",")[1:] + selected_formula = filter_type_chunk.split(",")[0] + deserializedJson = object() + performed_changes = dict() + + # Starting Validation + if len(selected_sections) == 0: + filter_is_correct = False + validation_message += "\n\tMust include a section (job type). If you want to apply the changes to all sections, include 'Any'." + else: + for section in selected_sections: + # Validating empty sections + if len(section) == 0: + filter_is_correct = False + validation_message += "\n\tEmpty sections are not accepted." + break + # Validating existing sections + # Retrieve experiment data + current_sections = as_conf.get_jobs_sections() + if section not in current_sections and section != "Any": + filter_is_correct = False + validation_message += "\n\tSection " + section + " does not exist in experiment." + + # Validating chunk formula + if len(selected_formula) == 0: + filter_is_correct = False + validation_message += "\n\tA formula for chunk filtering has not been provided. If you want to change all chunks, include 'Any'." + + # If everything is fine until this point + if filter_is_correct == True: + # Retrieve experiment data + current_dates = as_conf._exp_parser.get_option('experiment','DATELIST','').split() + current_members = as_conf.get_member_list() + # Parse json + deserializedJson = json.loads(Autosubmit._create_json(selected_formula)) + for startingDate in deserializedJson['sds']: + if startingDate['sd'] not in current_dates: + filter_is_correct = False + validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." + for member in startingDate['ms']: + if member['m'] not in current_members and member['m'] != "Any": + filter_is_correct_ = False + validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." + + + # Ending validation + if filter_is_correct == False: + print(validation_message) + return False + # If input is valid, continue. + record = dict() + final_list = [] + # Get current list + working_list = job_list.get_job_list() + for section in selected_sections: + if section == "Any": + # Any section + section_selection = working_list + # Go through start dates + for starting_date in deserializedJson['sds']: + date = starting_date['sd'] + date_selection = filter(lambda j: date2str(j.date) == date, section_selection) + # Members for given start date + for member_group in starting_date['ms']: + member = member_group['m'] + if member == "Any": + # Any member + member_selection = date_selection + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + else: + # Selected members + member_selection = filter(lambda j: j.member == member, date_selection) + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + else: + # Only given section + section_selection = filter(lambda j: j.section == section, working_list) + # Go through start dates + for starting_date in deserializedJson['sds']: + date = starting_date['sd'] + date_selection = filter(lambda j: date2str(j.date) == date, section_selection) + # Members for given start date + for member_group in starting_date['ms']: + member = member_group['m'] + if member == "Any": + # Any member + member_selection = date_selection + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + else: + # Selected members + member_selection = filter(lambda j: j.member == member, date_selection) + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = filter(lambda j: j.chunk == int(chunk), member_selection) + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in filter(lambda j: j.chunk == int(chunk) and j.synchronize is not None, date_selection): + final_list.append(job) + status = Status() + for job in final_list: + if job.status != final_status: + # Only real changes + performed_changes[job.name] = str(Status.VALUE_TO_KEY[job.status]) + " -> " + str(final) + Autosubmit.change_status(final, final_status, job) + # If changes have been performed + if len(performed_changes.keys()) > 0: + if Autosubmit._user_yes_no_query("Would you like to see an extended representation of the changes?") == True: + Log.info(job_list.print_with_status(statusChange = performed_changes)) + else: + Log.warning("No changes were performed.") + # End of New Feature if filter_chunks: if len(jobs_filtered) == 0: @@ -2754,6 +2899,8 @@ class Autosubmit: if job.name in jobs: Autosubmit.change_status(final, final_status, job) + + job_list.update_list(as_conf,False,True) if save and wrongExpid == 0: diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index b2a1fdd96..51205c98d 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -40,6 +40,15 @@ class Status: def retval(self, value): return getattr(self, value) +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' class Type: """ diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index e933cc96f..e99f37143 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -34,7 +34,7 @@ from autosubmit.job.job import Job from bscearth.utils.log import Log from autosubmit.job.job_dict import DicJobs from autosubmit.job.job_utils import Dependency -from autosubmit.job.job_common import Status, Type +from autosubmit.job.job_common import Status, Type, bcolors from bscearth.utils.date import date2str, parse_date, sum_str_hours from autosubmit.job.job_packages import JobPackageSimple, JobPackageArray, JobPackageThread @@ -1050,3 +1050,88 @@ class JobList: if flag: self.update_genealogy(notransitive=notransitive) del self._dic_jobs + + def print_with_status(self, statusChange = None): + """ + Returns the string representation of the dependency tree of + the Job List + + :return: String representation + :rtype: String + """ + allJobs = self.get_all() + # Header + result = bcolors.BOLD + "## String representation of Job List [" + str(len(allJobs)) + "] with " + \ + bcolors.OKGREEN + str(len(statusChange.keys())) + " Changes ##" + bcolors.ENDC + bcolors.ENDC + + # Find root + root = None + for job in allJobs: + if job.has_parents() == False: + root = job + + # root exists + if root is not None: + result += self._recursion_print(root, 0, statusChange = statusChange) + else: + result += "\nCannot find root." + + return result + + def __str__(self): + """ + Returns the string representation of the class. + Usage print(class) + + :return: String representation. + :rtype: String + """ + allJobs = self.get_all() + result = bcolors.BOLD + "## String representation of Job List [" + str(len(allJobs)) + "] ##" + bcolors.ENDC + + # Find root + root = None + for job in allJobs: + if job.has_parents() == False: + root = job + + # root exists + if root is not None: + result += self._recursion_print(root, 0) + else: + result += "\nCannot find root." + + return result + + def _recursion_print(self, job, level, statusChange = None): + """ + Returns the list of children in a recursive way + Traverses the dependency tree + + :return: parent + list of children + :rtype: String + """ + result = "" + prefix = "" + for i in range(level): + prefix += "| " + # Prefix + Job Name + result = "\n"+ prefix + bcolors.BOLD + job.name + bcolors.ENDC + if len(job._children) > 0: + level += 1 + children = job._children + total_children = len(job._children) + # Writes children number + result += " ~ [" + str(total_children) + (" children] " if total_children > 1 else " child] ") + if statusChange is not None: + # Writes change if performed + result += bcolors.BOLD + bcolors.OKGREEN + statusChange[job.name] if job.name in statusChange else "" + result += bcolors.ENDC + bcolors.ENDC + + for child in children: + # Continues recursion + result += self._recursion_print(child, level, statusChange=statusChange) + else: + pass + + return result \ No newline at end of file -- GitLab From 9a93d889e84c803ead51a89f6d89f025beda2cf2 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 25 Jul 2019 09:37:41 +0200 Subject: [PATCH 05/10] Added flag to ftc --- autosubmit/autosubmit.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 065f74954..f39c6397d 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -71,6 +71,7 @@ from job.job_package_persistence import JobPackagePersistence from job.job_list_persistence import JobListPersistenceDb from job.job_list_persistence import JobListPersistencePkl from job.job_grouping import JobGrouping +# from API.testAPI import Monitor # noinspection PyPackageRequirements from bscearth.utils.log import Log from database.db_common import create_db @@ -371,6 +372,7 @@ class Autosubmit: subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded') subparser.add_argument('-nt', '--notransitive', action='store_true', default=False, help='Disable transitive reduction') subparser.add_argument('-cw', '--check_wrapper', action='store_true', default=False, help='Generate possible wrapper in the current workflow') + subparser.add_argument('-d', '--detail', action='store_true', default=False, help='Generate detailed view of changes') # Test Case @@ -462,7 +464,7 @@ class Autosubmit: elif args.command == 'setstatus': return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, args.filter_chunks, args.filter_status, args.filter_type, args.filter_type_chunk, args.hide, - args.group_by, args.expand, args.expand_status, args.notransitive,args.check_wrapper) + args.group_by, args.expand, args.expand_status, args.notransitive,args.check_wrapper, args.detail) elif args.command == 'testcase': return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate, args.HPC, args.branch) @@ -2628,7 +2630,7 @@ class Autosubmit: @staticmethod def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, filter_type_chunk, hide, group_by=None, - expand=list(), expand_status=list(), notransitive=False,check_wrapper=False): + expand=list(), expand_status=list(), notransitive=False,check_wrapper=False, detail=False): """ Set status @@ -2828,7 +2830,7 @@ class Autosubmit: Autosubmit.change_status(final, final_status, job) # If changes have been performed if len(performed_changes.keys()) > 0: - if Autosubmit._user_yes_no_query("Would you like to see an extended representation of the changes?") == True: + if detail == True: Log.info(job_list.print_with_status(statusChange = performed_changes)) else: Log.warning("No changes were performed.") @@ -3274,6 +3276,7 @@ class Autosubmit: @staticmethod def load_job_list(expid, as_conf, notransitive=False,monitor=False): + BasicConfig.read() rerun = as_conf.get_rerun() job_list = JobList(expid, BasicConfig, ConfigParserFactory(), Autosubmit._get_job_list_persistence(expid, as_conf)) -- GitLab From ab31d2763184ff35fe1149d4954713ec2115ee30 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 25 Jul 2019 13:01:51 +0200 Subject: [PATCH 06/10] Added extended validation to -fl and -ft --- autosubmit/autosubmit.py | 92 +++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 30 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1a46b896e..2786ac119 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2670,46 +2670,78 @@ class Autosubmit: return False - #Verifying job sections, if filter_section has been set: - section_not_found = False + #Verifying job sections, if filter_section has been set: + section_validation_error = False + section_error = False section_not_foundList = list() - if filter_section is not None: - if len(filter_section.split()) > 0: - jobSections = as_conf.get_jobs_sections() - for section in filter_section.split(): - # Provided section is not an existing section or it is not the keyword 'Any' - if section not in jobSections and (section != "Any"): - section_not_found = True - section_not_foundList.append(section) - if section_not_found == True: - Log.critical("Specified section(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive.", section_not_foundList, expid) - return False + section_validation_message = "\n## Section Validation Message ##" + if filter_section is not None: + countStart = filter_section.count('[') + countEnd = filter_section.count(']') + if countStart > 1 or countEnd > 1: + section_validation_error = True + section_validation_message += "\n\tList of sections has a format error. Perhaps you were trying to use -fc instead." + if section_validation_error == False: + if len(str(filter_section).strip()) > 0: + if len(filter_section.split()) > 0: + jobSections = as_conf.get_jobs_sections() + for section in filter_section.split(): + # print(section) + # Provided section is not an existing section or it is not the keyword 'Any' + if section not in jobSections and (section != "Any"): + section_error = True + section_not_foundList.append(section) + else: + section_validation_error = True + section_validation_message += "\n\tEmpty input. No changes performed." + if section_validation_error == True or section_error == True: + if section_error == True: + section_validation_message += "\n\tSpecified section(s) : [" + str(section_not_foundList) + \ + "] not found in the experiment " + str(expid) + ".\n\tProcess stopped. Comparison is case sensitive." + Log.info(section_validation_message) + Log.critical("Error in the supplied input for -ft.") + return False job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) #Verifying list of jobs, if filter_list has been set: #Seems that load_job_list call is necessary before verification is executed - job_not_found = False + job_validation_error = False + job_error = False job_not_foundList = list() #Building a simple list of job names + job_validation_message = "\n## Job Validation Message ##" jobs = list() - if job_list is not None: - for job in job_list.get_job_list(): - jobs.append(job.name) - - if lst is not None: - #print(jobs) - if len(lst.split()) > 0: - for sentJob in lst.split(): - # Provided job does not exist or it is not the keyword 'Any' - if sentJob not in jobs and (sentJob != "Any"): - job_not_found = True - job_not_foundList.append(sentJob) - - if job_not_found == True: - Log.critical("Specified job(s) : [{0}] not found in the experiment {1}. \nProcess stopped. Comparison is case sensitive.", job_not_foundList, expid) - return False + if job_list is not None and lst is not None: + countStart = lst.count('[') + countEnd = lst.count(']') + if countStart > 1 or countEnd > 1: + job_validation_error = True + job_validation_message += "\n\tList of jobs has a format error. Perhaps you were trying to use -fc instead." + + if job_validation_error == False: + for job in job_list.get_job_list(): + jobs.append(job.name) + if len(str(lst).strip()) > 0: + if len(lst.split()) > 0: + for sentJob in lst.split(): + #print(sentJob) + # Provided job does not exist or it is not the keyword 'Any' + if sentJob not in jobs and (sentJob != "Any"): + job_error = True + job_not_foundList.append(sentJob) + else: + job_validation_error = True + job_validation_message += "\n\tEmpty input. No changes performed." + + if job_validation_error == True or job_error == True: + if job_error == True: + job_validation_message += "\n\tSpecified job(s) : [" + str(job_not_foundList) + "] not found in the experiment " + \ + str(expid) + ". \n\tProcess stopped. Comparison is case sensitive." + Log.info(job_validation_message) + Log.critical("Error in the supplied input for -fl") + return False jobs_filtered =[] -- GitLab From a568fc85808811d1aec64f581e3a6dc7d866e91f Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 25 Jul 2019 13:28:06 +0200 Subject: [PATCH 07/10] Added -fc validation. --- autosubmit/autosubmit.py | 86 ++++++++++++++++++++++++++++++++-------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 2786ac119..84a2ab88c 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2670,12 +2670,12 @@ class Autosubmit: return False - #Verifying job sections, if filter_section has been set: - section_validation_error = False - section_error = False - section_not_foundList = list() - section_validation_message = "\n## Section Validation Message ##" + # Validating job sections, if filter_section -ft has been set: if filter_section is not None: + section_validation_error = False + section_error = False + section_not_foundList = list() + section_validation_message = "\n## Section Validation Message ##" countStart = filter_section.count('[') countEnd = filter_section.count(']') if countStart > 1 or countEnd > 1: @@ -2687,7 +2687,7 @@ class Autosubmit: jobSections = as_conf.get_jobs_sections() for section in filter_section.split(): # print(section) - # Provided section is not an existing section or it is not the keyword 'Any' + # Provided section is not an existing section or it is not the keyword 'Any' if section not in jobSections and (section != "Any"): section_error = True section_not_foundList.append(section) @@ -2705,15 +2705,14 @@ class Autosubmit: job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) - #Verifying list of jobs, if filter_list has been set: - #Seems that load_job_list call is necessary before verification is executed - job_validation_error = False - job_error = False - job_not_foundList = list() - #Building a simple list of job names - job_validation_message = "\n## Job Validation Message ##" - jobs = list() - if job_list is not None and lst is not None: + # Validating list of jobs, if filter_list -fl has been set: + # Seems that Autosubmit.load_job_list call is necessary before verification is executed + if job_list is not None and lst is not None: + job_validation_error = False + job_error = False + job_not_foundList = list() + job_validation_message = "\n## Job Validation Message ##" + jobs = list() countStart = lst.count('[') countEnd = lst.count(']') if countStart > 1 or countEnd > 1: @@ -2740,8 +2739,63 @@ class Autosubmit: job_validation_message += "\n\tSpecified job(s) : [" + str(job_not_foundList) + "] not found in the experiment " + \ str(expid) + ". \n\tProcess stopped. Comparison is case sensitive." Log.info(job_validation_message) - Log.critical("Error in the supplied input for -fl") + Log.critical("Error in the supplied input for -fl.") return False + + # Validating fc if filter_chunks -fc has been set: + if filter_chunks is not None: + fc_validation_message = "## -fc Validation Message ##" + fc_filter_is_correct = True + selected_sections = filter_chunks.split(",")[1:] + selected_formula = filter_chunks.split(",")[0] + current_sections = as_conf.get_jobs_sections() + fc_deserializedJson = object() + # Starting Validation + if len(str(selected_sections).strip()) == 0: + fc_filter_is_correct = False + fc_validation_message += "\n\tMust include a section (job type)." + else: + for section in selected_sections: + # section = section.strip() + # Validating empty sections + if len(str(section).strip()) == 0: + fc_filter_is_correct = False + fc_validation_message += "\n\tEmpty sections are not accepted." + break + # Validating existing sections + # Retrieve experiment data + + if section not in current_sections and section != "Any": + fc_filter_is_correct = False + fc_validation_message += "\n\tSection " + section + " does not exist in experiment. Remember not to include blank spaces." + + # Validating chunk formula + if len(selected_formula) == 0: + fc_filter_is_correct = False + fc_validation_message += "\n\tA formula for chunk filtering has not been provided." + + # If everything is fine until this point + if fc_filter_is_correct == True: + # Retrieve experiment data + current_dates = as_conf._exp_parser.get_option('experiment','DATELIST','').split() + current_members = as_conf.get_member_list() + # Parse json + fc_deserializedJson = json.loads(Autosubmit._create_json(selected_formula)) + for startingDate in fc_deserializedJson['sds']: + if startingDate['sd'] not in current_dates: + fc_filter_is_correct = False + fc_validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." + for member in startingDate['ms']: + if member['m'] not in current_members and member['m'] != "Any": + fc_filter_is_correct = False + fc_validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." + + # Ending validation + if fc_filter_is_correct == False: + Log.info(fc_validation_message) + Log.critical("Error in the supplied input for -fc.") + return False + jobs_filtered =[] -- GitLab From 11df4499ef668ab35e156eca5ddfd4afd681a6cd Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 25 Jul 2019 14:46:42 +0200 Subject: [PATCH 08/10] Added small changes in validation process --- autosubmit/autosubmit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index f39c6397d..5957adbce 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2709,13 +2709,13 @@ class Autosubmit: performed_changes = dict() # Starting Validation - if len(selected_sections) == 0: + if len(str(selected_sections).strip()) == 0: filter_is_correct = False validation_message += "\n\tMust include a section (job type). If you want to apply the changes to all sections, include 'Any'." else: for section in selected_sections: # Validating empty sections - if len(section) == 0: + if len(str(section).strip()) == 0: filter_is_correct = False validation_message += "\n\tEmpty sections are not accepted." break -- GitLab From 6dd0ed25a166312999b3d624d340bc1438179061 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Mon, 29 Jul 2019 11:01:20 +0200 Subject: [PATCH 09/10] Added validation for new scenarios. --- autosubmit/autosubmit.py | 68 +++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 84a2ab88c..2370585fe 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2681,6 +2681,10 @@ class Autosubmit: if countStart > 1 or countEnd > 1: section_validation_error = True section_validation_message += "\n\tList of sections has a format error. Perhaps you were trying to use -fc instead." + countUnderscore = filter_section.count('_') + if countUnderscore > 1: + section_validation_error = True + section_validation_message += "\n\tList of sections provided has a format error. Perhaps you were trying to use -fl instead." if section_validation_error == False: if len(str(filter_section).strip()) > 0: if len(filter_section.split()) > 0: @@ -2697,7 +2701,10 @@ class Autosubmit: if section_validation_error == True or section_error == True: if section_error == True: section_validation_message += "\n\tSpecified section(s) : [" + str(section_not_foundList) + \ - "] not found in the experiment " + str(expid) + ".\n\tProcess stopped. Comparison is case sensitive." + "] not found in the experiment " + str(expid) + \ + ".\n\tProcess stopped. Review the format of the provided input. Comparison is case sensitive." + \ + "\n\tRemember that this option expects section names separated by a blank space as input." + Log.info(section_validation_message) Log.critical("Error in the supplied input for -ft.") return False @@ -2737,14 +2744,15 @@ class Autosubmit: if job_validation_error == True or job_error == True: if job_error == True: job_validation_message += "\n\tSpecified job(s) : [" + str(job_not_foundList) + "] not found in the experiment " + \ - str(expid) + ". \n\tProcess stopped. Comparison is case sensitive." + str(expid) + ". \n\tProcess stopped. Review the format of the provided input. Comparison is case sensitive." + \ + "\n\tRemember that this option expects job names separated by a blank space as input." Log.info(job_validation_message) Log.critical("Error in the supplied input for -fl.") return False # Validating fc if filter_chunks -fc has been set: if filter_chunks is not None: - fc_validation_message = "## -fc Validation Message ##" + fc_validation_message = "## -fc Validation Message ##" fc_filter_is_correct = True selected_sections = filter_chunks.split(",")[1:] selected_formula = filter_chunks.split(",")[0] @@ -2780,15 +2788,20 @@ class Autosubmit: current_dates = as_conf._exp_parser.get_option('experiment','DATELIST','').split() current_members = as_conf.get_member_list() # Parse json - fc_deserializedJson = json.loads(Autosubmit._create_json(selected_formula)) - for startingDate in fc_deserializedJson['sds']: - if startingDate['sd'] not in current_dates: - fc_filter_is_correct = False - fc_validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." - for member in startingDate['ms']: - if member['m'] not in current_members and member['m'] != "Any": + try: + fc_deserializedJson = json.loads(Autosubmit._create_json(selected_formula)) + except: + fc_filter_is_correct = False + fc_validation_message += "\n\tProvided chunk formula does not have the right format. Were you trying to use another option?" + if fc_filter_is_correct == True: + for startingDate in fc_deserializedJson['sds']: + if startingDate['sd'] not in current_dates: fc_filter_is_correct = False - fc_validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." + fc_validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." + for member in startingDate['ms']: + if member['m'] not in current_members and member['m'] != "Any": + fc_filter_is_correct = False + fc_validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." # Ending validation if fc_filter_is_correct == False: @@ -2796,6 +2809,39 @@ class Autosubmit: Log.critical("Error in the supplied input for -fc.") return False + # Validating status, if filter_status -fs has been set: + # At this point we already have job_list from where we are getting the allows STATUS + if filter_status is not None: + status_validation_error = False + status_validation_message = "\n## Status Validation Message ##" + # Trying to identify chunk formula + countStart = filter_status.count('[') + countEnd = filter_status.count(']') + if countStart > 1 or countEnd > 1: + status_validation_error = True + status_validation_message += "\n\tList of status provided has a format error. Perhaps you were trying to use -fc instead." + # Trying to identify job names, implying status names won't use more than 1 underscore _ + countUnderscore = filter_status.count('_') + if countUnderscore > 1: + status_validation_error = True + status_validation_message += "\n\tList of status provided has a format error. Perhaps you were trying to use -fl instead." + # If everything is fine until this point + if status_validation_error == False: + status_filter = filter_status.split() + status_reference = Status() + status_list = list() + for job in job_list.get_job_list(): + reference = status_reference.VALUE_TO_KEY[job.status] + if reference not in status_list: + status_list.append(reference) + for status in status_filter: + if status not in status_list: + status_validation_error = True + status_validation_message += "\n\t There are no jobs with status " + status + " in this experiment." + if status_validation_error == True: + Log.info(status_validation_message) + Log.critical("Error in the supplied input for -fs.") + return False jobs_filtered =[] -- GitLab From d97b73cba54efdc3aaad1d1cb2810ea57519ed16 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 31 Jul 2019 11:14:15 +0200 Subject: [PATCH 10/10] After merging issue #374 into issue #372, and doing the corresponding testing, validation for -ftc has been improved and documentation for set status has been expanded to include the new feature. --- autosubmit/autosubmit.py | 31 +++++++++++++++++++------------ autosubmit/job/job_list.py | 2 +- docs/source/usage/setstatus.rst | 9 +++++++++ 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 2a9327876..493d3ea03 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2782,7 +2782,7 @@ class Autosubmit: # Validating existing sections # Retrieve experiment data - if section not in current_sections and section != "Any": + if section not in current_sections: fc_filter_is_correct = False fc_validation_message += "\n\tSection " + section + " does not exist in experiment. Remember not to include blank spaces." @@ -2808,7 +2808,7 @@ class Autosubmit: fc_filter_is_correct = False fc_validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." for member in startingDate['ms']: - if member['m'] not in current_members and member['m'] != "Any": + if member['m'] not in current_members: fc_filter_is_correct = False fc_validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." @@ -2911,21 +2911,28 @@ class Autosubmit: current_dates = as_conf._exp_parser.get_option('experiment','DATELIST','').split() current_members = as_conf.get_member_list() # Parse json - deserializedJson = json.loads(Autosubmit._create_json(selected_formula)) - for startingDate in deserializedJson['sds']: - if startingDate['sd'] not in current_dates: - filter_is_correct = False - validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." - for member in startingDate['ms']: - if member['m'] not in current_members and member['m'] != "Any": - filter_is_correct_ = False - validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." + try: + deserializedJson = json.loads(Autosubmit._create_json(selected_formula)) + except: + filter_is_correct = False + validation_message += "\n\tProvided chunk formula does not have the right format. Were you trying to use another option?" + if filter_is_correct == True: + for startingDate in deserializedJson['sds']: + if startingDate['sd'] not in current_dates: + filter_is_correct = False + validation_message += "\n\tStarting date " + startingDate['sd'] + " does not exist in experiment." + for member in startingDate['ms']: + if member['m'] not in current_members and member['m'] != "Any": + filter_is_correct_ = False + validation_message += "\n\tMember " + member['m'] + " does not exist in experiment." # Ending validation if filter_is_correct == False: - print(validation_message) + Log.info(validation_message) + Log.critical("Error in the supplied input for -ftc.") return False + # If input is valid, continue. record = dict() final_list = [] diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index e99f37143..f7d75d408 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1062,7 +1062,7 @@ class JobList: allJobs = self.get_all() # Header result = bcolors.BOLD + "## String representation of Job List [" + str(len(allJobs)) + "] with " + \ - bcolors.OKGREEN + str(len(statusChange.keys())) + " Changes ##" + bcolors.ENDC + bcolors.ENDC + bcolors.OKGREEN + str(len(statusChange.keys())) + " Change(s) ##" + bcolors.ENDC + bcolors.ENDC # Find root root = None diff --git a/docs/source/usage/setstatus.rst b/docs/source/usage/setstatus.rst index ed8b4fecb..6d8975f9f 100644 --- a/docs/source/usage/setstatus.rst +++ b/docs/source/usage/setstatus.rst @@ -28,6 +28,15 @@ Options: List of status to be changed -ft FILTER_TYPE, --filter_type List of types to be changed + -ftc FILTER_TYPE_CHUNK --filter_type_chunk + Accepts a string with the formula: "[ 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16 30] ] ],SIM,SIM2" + Where SIM, SIM2 are section (or job type) names that also accept the keyword "Any" so the changes apply to all sections. + Starting Date (19601101) does not accept the keyword "Any". + Member names (fc0) accept the keyword "Any", so the chunks ([1 2 3 4]) given will be updated in all members. + Chunks must be in the format "[1 2 3 4 n]" where "n" is an integer representing the number of the chunk in the member, + no range format is allowed. + -d When using the option -ftc and sending this flag, a tree view of the experiment with markers indicating which jobs + have been changed will be generated. --hide, hide the plot -group_by {date,member,chunk,split,automatic} criteria to use for grouping jobs -- GitLab