From 6c301d98c584199f8a7945726f88dff5c4812699 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 1 Jul 2021 12:39:01 +0200 Subject: [PATCH 1/2] Fixing a couple of bugs I noticed while running the dummy experiment. 1) RUN_ONLY_MEMBERS was being set with the default member list because it was caught by the regular expression that sets the default value for MEMBERs. 2) A Critical warning was being shown the first time the experiment is created, I changed it to a regular warning (not visible in the terminal by default) so it does not mislead the user. --- autosubmit/autosubmit.py | 49 ++++++++++++++++---------- autosubmit/job/job_list_persistence.py | 6 ++-- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d5af3da5c..46b1a70d8 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -629,7 +629,7 @@ class Autosubmit: return Autosubmit.pkl_fix(args.expid) @staticmethod - def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): + def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): Log.set_console_level(console_level) expid_less = ["expid", "testcase", "install", "-v", "readme", "changelog", "configure", "unarchive"] @@ -1222,7 +1222,8 @@ class Autosubmit: wrapper_jobs = dict() if as_conf.get_wrapper_type() == "multi": for wrapper_section in as_conf.get_wrapper_multi(): - wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs(wrapper_section) + wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs( + wrapper_section) wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs("wrapper") Log.warning("Aux Job_list was generated successfully") submitter = Autosubmit._get_submitter(as_conf) @@ -1245,11 +1246,13 @@ class Autosubmit: unparsed_two_step_start = as_conf.get_parse_two_step_start() if unparsed_two_step_start != "": job_list.parse_two_step_start(unparsed_two_step_start) - job_list.create_dictionary(date_list, member_list, num_chunks, chunk_ini, date_format, as_conf.get_retrials(), wrapper_jobs ) + job_list.create_dictionary(date_list, member_list, num_chunks, + chunk_ini, date_format, as_conf.get_retrials(), wrapper_jobs) while job_list.get_active(): - Autosubmit.submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers, hold=False) - #for job in job_list.get_uncompleted_and_not_waiting(): + Autosubmit.submit_ready_jobs( + as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers, hold=False) + # for job in job_list.get_uncompleted_and_not_waiting(): # job.status = Status.COMPLETED job_list.update_list(as_conf, False) @@ -1713,11 +1716,13 @@ class Autosubmit: for job in failed_jobs: if job.fail_count > 0: failed_names[job.name] = job.fail_count - job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) + job_list = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive) if len(job_list._job_list) == 0: sleep(5) raise IOError - Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) + Autosubmit._load_parameters( + as_conf, job_list, submitter.platforms) for job in job_list.get_job_list(): if job in failed_names: job.fail_count = failed_names[job.name] @@ -1735,7 +1740,8 @@ class Autosubmit: raise IOError for (exp_id, package_name, job_name) in packages: if package_name not in job_list.packages_dict: - job_list.packages_dict[package_name] = [] + job_list.packages_dict[package_name] = [ + ] job_list.packages_dict[package_name].append( job_list.get_job_by_name(job_name)) for package_name, jobs in job_list.packages_dict.items(): @@ -1753,7 +1759,8 @@ class Autosubmit: except IOError as e: IO_issues = True except BaseException as e: - AutosubmitCritical("Unknown error during the recovery of the job_list",7056,e) + AutosubmitCritical( + "Unknown error during the recovery of the job_list", 7056, e) # Restore platforms and try again, to avoid endless loop with failed configuration, a hard limit is set. reconnected = False @@ -1824,7 +1831,7 @@ class Autosubmit: except AutosubmitCritical as e: raise AutosubmitCritical(e.message, e.code, e.trace) except IOError as e: - raise AutosubmitError(e.message,e.code,e.trace) + raise AutosubmitError(e.message, e.code, e.trace) except BaseException as e: raise @@ -1945,10 +1952,11 @@ class Autosubmit: raise except Exception as e: raise - if platform.type == "slurm" and not inspect and not only_wrappers: # return to == + if platform.type == "slurm" and not inspect and not only_wrappers: # return to == try: save = True - valid_packages_to_submit = [ package for package in valid_packages_to_submit if package.x11 != True] + valid_packages_to_submit = [ + package for package in valid_packages_to_submit if package.x11 != True] if len(valid_packages_to_submit) > 0: try: jobs_id = platform.submit_Script(hold=hold) @@ -2171,7 +2179,7 @@ class Autosubmit: job_list_wrappers = copy.deepcopy(job_list) jobs_wr_aux = copy.deepcopy(jobs) jobs_wr = [] - [jobs_wr.append(job) for job in jobs_wr_aux ] + [jobs_wr.append(job) for job in jobs_wr_aux] for job in jobs_wr: for child in job.children: if child not in jobs_wr: @@ -2184,7 +2192,6 @@ class Autosubmit: job.children = job.children - referenced_jobs_to_remove job.parents = job.parents - referenced_jobs_to_remove - Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wrappers, jobs_wr, packages_persistence, True) @@ -3809,8 +3816,10 @@ class Autosubmit: wrapper_jobs = dict() if as_conf.get_wrapper_type() == "multi": for wrapper_section in as_conf.get_wrapper_multi(): - wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs(wrapper_section) - wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs("wrapper") + wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs( + wrapper_section) + wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs( + "wrapper") job_list.generate(date_list, member_list, num_chunks, chunk_ini, parameters, date_format, as_conf.get_retrials(), @@ -4598,7 +4607,7 @@ class Autosubmit: # job_data_structure.update_jobs_from_change_status(job_tracked_changes) job_data_structure.process_status_changes( job_tracked_changes, job_list.get_job_list(), as_conf.get_chunk_size_unit(), as_conf.get_chunk_size(), check_run=True, current_config=as_conf.get_full_config_as_json(), is_setstatus=True) - + else: Log.printlog( "Changes NOT saved to the JobList!!!!: use -s option to save", 3000) @@ -4719,6 +4728,8 @@ class Autosubmit: "NUMCHUNKS = 1") content = content.replace(re.search('^PROJECT_TYPE =.*', content, re.MULTILINE).group(0), "PROJECT_TYPE = none") + content = content.replace(re.search('^RUN_ONLY_MEMBERS = .*', content, re.MULTILINE).group(0), + "RUN_ONLY_MEMBERS = ") open(as_conf.experiment_file, 'w').write(content) @@ -5035,8 +5046,8 @@ class Autosubmit: wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs() if as_conf.get_wrapper_type() == "multi": for wrapper_section in as_conf.get_wrapper_multi(): - wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs(wrapper_section) - + wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs( + wrapper_section) job_list.generate(date_list, as_conf.get_member_list(), as_conf.get_num_chunks(), as_conf.get_chunk_ini(), as_conf.load_parameters(), date_format, as_conf.get_retrials(), diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 3065905a9..064589e0d 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -72,7 +72,8 @@ class JobListPersistencePkl(JobListPersistence): fd = open(path, 'r') return pickle.load(fd) else: - Log.printlog('File {0} does not exist'.format(path),7040) + # Changing it to a regular warning. + Log.warning('File {0} does not exist'.format(path), 7040) return list() def save(self, persistence_path, persistence_file, job_list): @@ -110,7 +111,8 @@ class JobListPersistenceDb(JobListPersistence): 'remote_out', 'remote_err'] def __init__(self, persistence_path, persistence_file): - self.db_manager = DbManager(persistence_path, persistence_file, self.VERSION) + self.db_manager = DbManager( + persistence_path, persistence_file, self.VERSION) def load(self, persistence_path, persistence_file): """ -- GitLab From b58a355e939486791396c1378ba7fb00967d084b Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Thu, 1 Jul 2021 12:46:15 +0200 Subject: [PATCH 2/2] Adding Xlib to the pipeline environment --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index fd563f85f..cc544f2b9 100644 --- a/environment.yml +++ b/environment.yml @@ -15,6 +15,7 @@ dependencies: - mock - portalocker - networkx +- Xlib - pip: - bscearth.utils -- GitLab