diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d5af3da5c7f4edf53bb1c60ec02abd0bd2260731..46b1a70d83a745075b734273b47f12e706baee65 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -629,7 +629,7 @@ class Autosubmit: return Autosubmit.pkl_fix(args.expid) @staticmethod - def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): + def _init_logs(args, console_level='INFO', log_level='DEBUG', expid='None'): Log.set_console_level(console_level) expid_less = ["expid", "testcase", "install", "-v", "readme", "changelog", "configure", "unarchive"] @@ -1222,7 +1222,8 @@ class Autosubmit: wrapper_jobs = dict() if as_conf.get_wrapper_type() == "multi": for wrapper_section in as_conf.get_wrapper_multi(): - wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs(wrapper_section) + wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs( + wrapper_section) wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs("wrapper") Log.warning("Aux Job_list was generated successfully") submitter = Autosubmit._get_submitter(as_conf) @@ -1245,11 +1246,13 @@ class Autosubmit: unparsed_two_step_start = as_conf.get_parse_two_step_start() if unparsed_two_step_start != "": job_list.parse_two_step_start(unparsed_two_step_start) - job_list.create_dictionary(date_list, member_list, num_chunks, chunk_ini, date_format, as_conf.get_retrials(), wrapper_jobs ) + job_list.create_dictionary(date_list, member_list, num_chunks, + chunk_ini, date_format, as_conf.get_retrials(), wrapper_jobs) while job_list.get_active(): - Autosubmit.submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers, hold=False) - #for job in job_list.get_uncompleted_and_not_waiting(): + Autosubmit.submit_ready_jobs( + as_conf, job_list, platforms_to_test, packages_persistence, True, only_wrappers, hold=False) + # for job in job_list.get_uncompleted_and_not_waiting(): # job.status = Status.COMPLETED job_list.update_list(as_conf, False) @@ -1713,11 +1716,13 @@ class Autosubmit: for job in failed_jobs: if job.fail_count > 0: failed_names[job.name] = job.fail_count - job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) + job_list = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive) if len(job_list._job_list) == 0: sleep(5) raise IOError - Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) + Autosubmit._load_parameters( + as_conf, job_list, submitter.platforms) for job in job_list.get_job_list(): if job in failed_names: job.fail_count = failed_names[job.name] @@ -1735,7 +1740,8 @@ class Autosubmit: raise IOError for (exp_id, package_name, job_name) in packages: if package_name not in job_list.packages_dict: - job_list.packages_dict[package_name] = [] + job_list.packages_dict[package_name] = [ + ] job_list.packages_dict[package_name].append( job_list.get_job_by_name(job_name)) for package_name, jobs in job_list.packages_dict.items(): @@ -1753,7 +1759,8 @@ class Autosubmit: except IOError as e: IO_issues = True except BaseException as e: - AutosubmitCritical("Unknown error during the recovery of the job_list",7056,e) + AutosubmitCritical( + "Unknown error during the recovery of the job_list", 7056, e) # Restore platforms and try again, to avoid endless loop with failed configuration, a hard limit is set. reconnected = False @@ -1824,7 +1831,7 @@ class Autosubmit: except AutosubmitCritical as e: raise AutosubmitCritical(e.message, e.code, e.trace) except IOError as e: - raise AutosubmitError(e.message,e.code,e.trace) + raise AutosubmitError(e.message, e.code, e.trace) except BaseException as e: raise @@ -1945,10 +1952,11 @@ class Autosubmit: raise except Exception as e: raise - if platform.type == "slurm" and not inspect and not only_wrappers: # return to == + if platform.type == "slurm" and not inspect and not only_wrappers: # return to == try: save = True - valid_packages_to_submit = [ package for package in valid_packages_to_submit if package.x11 != True] + valid_packages_to_submit = [ + package for package in valid_packages_to_submit if package.x11 != True] if len(valid_packages_to_submit) > 0: try: jobs_id = platform.submit_Script(hold=hold) @@ -2171,7 +2179,7 @@ class Autosubmit: job_list_wrappers = copy.deepcopy(job_list) jobs_wr_aux = copy.deepcopy(jobs) jobs_wr = [] - [jobs_wr.append(job) for job in jobs_wr_aux ] + [jobs_wr.append(job) for job in jobs_wr_aux] for job in jobs_wr: for child in job.children: if child not in jobs_wr: @@ -2184,7 +2192,6 @@ class Autosubmit: job.children = job.children - referenced_jobs_to_remove job.parents = job.parents - referenced_jobs_to_remove - Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wrappers, jobs_wr, packages_persistence, True) @@ -3809,8 +3816,10 @@ class Autosubmit: wrapper_jobs = dict() if as_conf.get_wrapper_type() == "multi": for wrapper_section in as_conf.get_wrapper_multi(): - wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs(wrapper_section) - wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs("wrapper") + wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs( + wrapper_section) + wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs( + "wrapper") job_list.generate(date_list, member_list, num_chunks, chunk_ini, parameters, date_format, as_conf.get_retrials(), @@ -4598,7 +4607,7 @@ class Autosubmit: # job_data_structure.update_jobs_from_change_status(job_tracked_changes) job_data_structure.process_status_changes( job_tracked_changes, job_list.get_job_list(), as_conf.get_chunk_size_unit(), as_conf.get_chunk_size(), check_run=True, current_config=as_conf.get_full_config_as_json(), is_setstatus=True) - + else: Log.printlog( "Changes NOT saved to the JobList!!!!: use -s option to save", 3000) @@ -4719,6 +4728,8 @@ class Autosubmit: "NUMCHUNKS = 1") content = content.replace(re.search('^PROJECT_TYPE =.*', content, re.MULTILINE).group(0), "PROJECT_TYPE = none") + content = content.replace(re.search('^RUN_ONLY_MEMBERS = .*', content, re.MULTILINE).group(0), + "RUN_ONLY_MEMBERS = ") open(as_conf.experiment_file, 'w').write(content) @@ -5035,8 +5046,8 @@ class Autosubmit: wrapper_jobs["wrapper"] = as_conf.get_wrapper_jobs() if as_conf.get_wrapper_type() == "multi": for wrapper_section in as_conf.get_wrapper_multi(): - wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs(wrapper_section) - + wrapper_jobs[wrapper_section] = as_conf.get_wrapper_jobs( + wrapper_section) job_list.generate(date_list, as_conf.get_member_list(), as_conf.get_num_chunks(), as_conf.get_chunk_ini(), as_conf.load_parameters(), date_format, as_conf.get_retrials(), diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 3065905a93f2e5c9ddba1d725aadd4555f08ccc1..064589e0dfd5d7a2206cb29a83ea384dd7aa6996 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -72,7 +72,8 @@ class JobListPersistencePkl(JobListPersistence): fd = open(path, 'r') return pickle.load(fd) else: - Log.printlog('File {0} does not exist'.format(path),7040) + # Changing it to a regular warning. + Log.warning('File {0} does not exist'.format(path), 7040) return list() def save(self, persistence_path, persistence_file, job_list): @@ -110,7 +111,8 @@ class JobListPersistenceDb(JobListPersistence): 'remote_out', 'remote_err'] def __init__(self, persistence_path, persistence_file): - self.db_manager = DbManager(persistence_path, persistence_file, self.VERSION) + self.db_manager = DbManager( + persistence_path, persistence_file, self.VERSION) def load(self, persistence_path, persistence_file): """ diff --git a/environment.yml b/environment.yml index fd563f85fa3b28146ff81f207dd571f9956f2551..cc544f2b91dd3368d775ce61b056ac5d9892f7b6 100644 --- a/environment.yml +++ b/environment.yml @@ -15,6 +15,7 @@ dependencies: - mock - portalocker - networkx +- Xlib - pip: - bscearth.utils