diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000000000000000000000000000000000000..a73ca4c7c17aef9019ad929ae9892facfcbedcad --- /dev/null +++ b/.mailmap @@ -0,0 +1,36 @@ +# FORMAT: +# Omit commit-name or commit-email if same as replace-with. +# git log --pretty="%aN <%aE>%n%cN <%cE>" | sort | uniq + +Bruno P. Kinoshita Bruno de Paula Kinoshita <777-bdepaula@users.noreply.earth.bsc.es> +Bruno P. Kinoshita Bruno P. Kinoshita +Daniel Beltran Mora dbeltran +Daniel Beltran Mora dbeltran +Daniel Beltran Mora Daniel Beltran Mora +Danila Volpi Danila Volpi +Domingo Manubens-Gil dmanubens +Domingo Manubens-Gil Domingo Manubens +Domingo Manubens-Gil Domingo Manubens +Domingo Manubens-Gil Domingo Manubens-Gil +Francesc Roura froura +Isabel Andreu-Burillo Isabel Andreu-Burillo +Javier Vegas-Regidor jvegas +Javier Vegas-Regidor Javier Vegas-Regidor +Javier Vegas-Regidor Javier Vegas-Regidor +Joan Lopez jlope2 +Julian Berlin jberlin +Larissa Batista Leite lbatista +Manuel G. Marciani Manuel Giménez de Castro +Miguel Andrés Martínez Miguel Andres-Martinez +Muhammad Asif Muhammad Asif +Muhammad Asif Muhammad Asif +Oriol Mula-Valls Oriol Mula-Valls +Oriol Mula-Valls Oriol Mula Valls +Oriol Mula-Valls Oriol Mula Valls +Oriol Mula-Valls Oriol Mula-Valls +Oriol Mula-Valls Administrator +Pierre-Antoine Bretonniere Pierre-Antoine Bretonniere +Virginie Guemas vguemas +Virginie Guemas Virginie Guemas +Virginie Guemas Virginie +Wilmer Uruchi Ticona wuruchi diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000000000000000000000000000000000..21093d904a354f202fdcb4f02ade2b8e1d388488 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,38 @@ +# Security Policies and Procedures + +This document outlines security procedures and general policies for the +Autosubmit project. + + * [Reporting Security Issues](#reporting-security-issues) + * [Preferred Languages](#preferred-languages) + * [Policy](#policy) + +## Reporting Security Issues + +**Please do NOT report security vulnerabilities through public issues.** + +The Autosubumit maintainers take security bugs seriously. Thank you for +improving the security of Autosubumit. We appreciate your efforts and responsible +disclosure and will make every effort to acknowledge your contributions. + +If you believe you have found a security vulneratibility in Autosubmit, +please report it by sending an email to +support-autosubmit@bsc.es. + +## Preferred Languages + +All communications are preffered to be in English, Spanish, or Catalan. + +## Policy + +When the Autosubmit maintainers receive a security bug report, they will +assign it to a primary handler. This person will coordinate the fix and +release process as follows: + +* Confirm the problem and determine the affected versions. +* Audit code to find any potential similar problems. +* Prepare fixes for all releases still under maintenance. +* Cut new releases as soon as possible. + +CVE's may also be issued depending on the risk level, with credit to +the reporter. diff --git a/VERSION b/VERSION index 511f5bac1eaf6d222d50f1fe3595050627ad5d00..ee74734aa2258df77aa09402d55798a1e2e55212 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.0.99 +4.1.0 diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d429e109c397b8b5a8158210531a93a9b8022885..e622b7723b142a7766061720866043b0224ef0cd 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -442,6 +442,8 @@ class Autosubmit: default=False, help='Update experiment version') subparser.add_argument('-p', '--profile', action='store_true', default=False, required=False, help='Prints performance parameters of the execution of this command.') + subparser.add_argument( + '-f', '--force', action='store_true', default=False, help='force regenerate job_list') # Configure subparser = subparsers.add_parser('configure', description="configure database and path for autosubmit. It " "can be done at machine, user or local level." @@ -505,6 +507,11 @@ class Autosubmit: selected from for that member will be updated for all the members. Example: all [1], will have as a result that the \ chunks 1 for all the members will be updated. Follow the format: ' '"[ 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16-30] ] ],SIM,SIM2,SIM3"') + group.add_argument('-ftcs', '--filter_type_chunk_split', type=str, + help='Supply the list of chunks & splits to change the status. Default = "Any". When the member name "all" is set, all the chunks \ + selected from for that member will be updated for all the members. Example: all [1], will have as a result that the \ + chunks 1 for all the members will be updated. Follow the format: ' + '"[ 19601101 [ fc0 [1 [1 2] 2 3 4] Any [1] ] 19651101 [ fc0 [16-30] ] ],SIM,SIM2,SIM3"') subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') @@ -692,7 +699,7 @@ class Autosubmit: return Autosubmit.migrate(args.expid, args.offer, args.pickup, args.onlyremote) elif args.command == 'create': return Autosubmit.create(args.expid, args.noplot, args.hide, args.output, args.group_by, args.expand, - args.expand_status, args.notransitive, args.check_wrapper, args.detail, args.profile) + args.expand_status, args.notransitive, args.check_wrapper, args.detail, args.profile, args.force) elif args.command == 'configure': if not args.advanced or (args.advanced and dialog is None): return Autosubmit.configure(args.advanced, args.databasepath, args.databasefilename, @@ -705,7 +712,7 @@ class Autosubmit: elif args.command == 'setstatus': return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, args.filter_chunks, args.filter_status, args.filter_type, - args.filter_type_chunk, args.hide, + args.filter_type_chunk, args.filter_type_chunk_split, args.hide, args.group_by, args.expand, args.expand_status, args.notransitive, args.check_wrapper, args.detail) elif args.command == 'testcase': @@ -1417,7 +1424,8 @@ class Autosubmit: packages_persistence.reset_table(True) job_list_original = Autosubmit.load_job_list( expid, as_conf, notransitive=notransitive) - job_list = copy.deepcopy(job_list_original) + job_list = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive) job_list.packages_dict = {} Log.debug("Length of the jobs list: {0}", len(job_list)) @@ -1498,30 +1506,12 @@ class Autosubmit: else: jobs = job_list.get_job_list() if isinstance(jobs, type([])): - referenced_jobs_to_remove = set() - for job in jobs: - for child in job.children: - if child not in jobs: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs: - referenced_jobs_to_remove.add(parent) - for job in jobs: job.status = Status.WAITING Autosubmit.generate_scripts_andor_wrappers( as_conf, job_list, jobs, packages_persistence, False) if len(jobs_cw) > 0: - referenced_jobs_to_remove = set() - for job in jobs_cw: - for child in job.children: - if child not in jobs_cw: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs_cw: - referenced_jobs_to_remove.add(parent) - for job in jobs_cw: job.status = Status.WAITING Autosubmit.generate_scripts_andor_wrappers( @@ -1592,9 +1582,11 @@ class Autosubmit: job.platform = submitter.platforms[job.platform_name] if job.platform is not None and job.platform != "": platforms_to_test.add(job.platform) - - job_list.check_scripts(as_conf) - + if not only_wrappers: + job_list.check_scripts(as_conf) # added only in inspect + else: # no longer check_Scripts if -cw is added to monitor or create, just update the parameters + for job in ( job for job in job_list.get_job_list() ): + job.update_parameters(as_conf,parameters) job_list.update_list(as_conf, False) # Loading parameters again Autosubmit._load_parameters(as_conf, job_list, submitter.platforms) @@ -1603,7 +1595,7 @@ class Autosubmit: if unparsed_two_step_start != "": job_list.parse_jobs_by_filter(unparsed_two_step_start) job_list.create_dictionary(date_list, member_list, num_chunks, chunk_ini, date_format, as_conf.get_retrials(), - wrapper_jobs) + wrapper_jobs, as_conf) for job in job_list.get_active(): if job.status != Status.WAITING: job.status = Status.READY @@ -1614,6 +1606,7 @@ class Autosubmit: # job.status = Status.COMPLETED job_list.update_list(as_conf, False) + @staticmethod def terminate(all_threads): # Closing threads on Ctrl+C @@ -1886,7 +1879,7 @@ class Autosubmit: Log.info("Recovering job_list") try: job_list = Autosubmit.load_job_list( - expid, as_conf, notransitive=notransitive) + expid, as_conf, notransitive=notransitive, new=False) except IOError as e: raise AutosubmitError( "Job_list not found", 6016, str(e)) @@ -1913,7 +1906,7 @@ class Autosubmit: job.platform_name = hpcarch # noinspection PyTypeChecker try: - job.platform = submitter.platforms[job.platform_name.upper()] + job.platform = submitter.platforms[job.platform_name.upper()] except Exception as e: raise AutosubmitCritical( "hpcarch={0} not found in the platforms configuration file".format(job.platform_name), @@ -1963,6 +1956,7 @@ class Autosubmit: Log.debug("Checking job_list current status") job_list.update_list(as_conf, first_time=True) job_list.save() + as_conf.save() if not recover: Log.info("Autosubmit is running with v{0}", Autosubmit.autosubmit_version) # Before starting main loop, setup historical database tables and main information @@ -2075,6 +2069,8 @@ class Autosubmit: try: if Autosubmit.exit: Autosubmit.terminate(threading.enumerate()) + if job_list.get_failed(): + return 1 return 0 # reload parameters changes Log.debug("Reloading parameters...") @@ -2116,6 +2112,8 @@ class Autosubmit: Autosubmit.submit_ready_jobs(as_conf, job_list, platforms_to_test, packages_persistence, hold=False) job_list.update_list(as_conf, submitter=submitter) job_list.save() + as_conf.save() + # Submit jobs that are prepared to hold (if remote dependencies parameter are enabled) # This currently is not used as SLURM no longer allows to jobs to adquire priority while in hold state. # This only works for SLURM. ( Prepare status can not be achieved in other platforms ) @@ -2124,6 +2122,7 @@ class Autosubmit: as_conf, job_list, platforms_to_test, packages_persistence, hold=True) job_list.update_list(as_conf, submitter=submitter) job_list.save() + as_conf.save() # Safe spot to store changes try: exp_history = Autosubmit.process_historical_data_iteration(job_list, job_changes_tracker, expid) @@ -2140,6 +2139,7 @@ class Autosubmit: job_changes_tracker = {} if Autosubmit.exit: job_list.save() + as_conf.save() time.sleep(safetysleeptime) #Log.debug(f"FD endsubmit: {fd_show.fd_table_status_str()}") @@ -2264,7 +2264,11 @@ class Autosubmit: else: Log.result("Run successful") # Updating finish time for job data header - exp_history.finish_current_experiment_run() + # Database is locked, may be related to my local db todo 4.1.1 + try: + exp_history.finish_current_experiment_run() + except: + Log.warning("Database is locked") except (portalocker.AlreadyLocked, portalocker.LockException) as e: message = "We have detected that there is another Autosubmit instance using the experiment\n. Stop other Autosubmit instances that are using the experiment or delete autosubmit.lock file located on tmp folder" raise AutosubmitCritical(message, 7000) @@ -2276,6 +2280,12 @@ class Autosubmit: if profile: profiler.stop() + # Suppress in case ``job_list`` was not defined yet... + with suppress(NameError): + if job_list.get_failed(): + return 1 + return 0 + @staticmethod def restore_platforms(platform_to_test, mail_notify=False, as_conf=None, expid=None): Log.info("Checking the connection to all platforms in use") @@ -2376,6 +2386,9 @@ class Autosubmit: hold=hold) # Jobs that are being retrieved in batch. Right now, only available for slurm platforms. if not inspect and len(valid_packages_to_submit) > 0: + for package in (package for package in valid_packages_to_submit): + for job in (job for job in package.jobs): + job._clean_runtime_parameters() job_list.save() save_2 = False if platform.type.lower() in [ "slurm" , "pjm" ] and not inspect and not only_wrappers: @@ -2384,6 +2397,9 @@ class Autosubmit: failed_packages, error_message="", hold=hold) if not inspect and len(valid_packages_to_submit) > 0: + for package in (package for package in valid_packages_to_submit): + for job in (job for job in package.jobs): + job._clean_runtime_parameters() job_list.save() # Save wrappers(jobs that has the same id) to be visualized and checked in other parts of the code job_list.save_wrappers(valid_packages_to_submit, failed_packages, as_conf, packages_persistence, @@ -2459,7 +2475,7 @@ class Autosubmit: output_type = as_conf.get_output_type() pkl_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, 'pkl') job_list = Autosubmit.load_job_list( - expid, as_conf, notransitive=notransitive, monitor=True) + expid, as_conf, notransitive=notransitive, monitor=True, new=False) Log.debug("Job list restored from {0} files", pkl_dir) except AutosubmitError as e: raise AutosubmitCritical(e.message, e.code, e.trace) @@ -2534,21 +2550,9 @@ class Autosubmit: if profile: profiler.stop() - referenced_jobs_to_remove = set() - for job in jobs: - for child in job.children: - if child not in jobs: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs: - referenced_jobs_to_remove.add(parent) - if len(referenced_jobs_to_remove) > 0: - for job in jobs: - job.children = job.children - referenced_jobs_to_remove - job.parents = job.parents - referenced_jobs_to_remove # WRAPPERS try: - if as_conf.get_wrapper_type() != 'none' and check_wrapper: + if len(as_conf.experiment_data.get("WRAPPERS", {})) > 0 and check_wrapper: # Class constructor creates table if it does not exist packages_persistence = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), "job_packages_" + expid) @@ -2556,24 +2560,10 @@ class Autosubmit: os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl", "job_packages_" + expid + ".db"), 0o644) # Database modification packages_persistence.reset_table(True) - referenced_jobs_to_remove = set() - job_list_wrappers = copy.deepcopy(job_list) - jobs_wr_aux = copy.deepcopy(jobs) - jobs_wr = [] - [jobs_wr.append(job) for job in jobs_wr_aux] - for job in jobs_wr: - for child in job.children: - if child not in jobs_wr: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs_wr: - referenced_jobs_to_remove.add(parent) - - for job in jobs_wr: - job.children = job.children - referenced_jobs_to_remove - job.parents = job.parents - referenced_jobs_to_remove - - Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wrappers, jobs_wr, + # Load another job_list to go through that goes through the jobs, but we want to monitor the other one + job_list_wr = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive, monitor=True, new=False) + Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wr, job_list_wr.get_job_list(), packages_persistence, True) packages = packages_persistence.load(True) @@ -2668,6 +2658,8 @@ class Autosubmit: pkl_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, 'pkl') job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive) + for job in job_list.get_job_list(): + job._init_runtime_parameters() Log.debug("Job list restored from {0} files", pkl_dir) jobs = StatisticsUtils.filter_by_section(job_list.get_job_list(), filter_type) jobs, period_ini, period_fi = StatisticsUtils.filter_by_time_period(jobs, filter_period) @@ -2679,7 +2671,7 @@ class Autosubmit: current_table_structure = get_structure(expid, BasicConfig.STRUCTURES_DIR) subjobs = [] for job in job_list.get_job_list(): - job_info = JobList.retrieve_times(job.status, job.name, job._tmp_path, make_exception=False, + job_info = JobList.retrieve_times(job.status, job.name, job._tmp_path, make_exception=True, job_times=None, seconds=True, job_data_collection=None) time_total = (job_info.queue_time + job_info.run_time) if job_info else 0 subjobs.append( @@ -2793,7 +2785,7 @@ class Autosubmit: Log.info('Recovering experiment {0}'.format(expid)) pkl_dir = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, 'pkl') job_list = Autosubmit.load_job_list( - expid, as_conf, notransitive=notransitive, monitor=True) + expid, as_conf, notransitive=notransitive, new=False, monitor=True) current_active_jobs = job_list.get_in_queue() @@ -2859,7 +2851,6 @@ class Autosubmit: job.platform_name = hpcarch # noinspection PyTypeChecker job.platform = platforms[job.platform_name] - if job.platform.get_completed_files(job.name, 0, recovery=True): job.status = Status.COMPLETED Log.info( @@ -3328,7 +3319,7 @@ class Autosubmit: if job.platform_name is None: job.platform_name = hpc_architecture job.platform = submitter.platforms[job.platform_name] - job.update_parameters(as_conf, job_list.parameters) + except AutosubmitError: raise except BaseException as e: @@ -3423,6 +3414,7 @@ class Autosubmit: try: for job in job_list.get_job_list(): job_parameters = job.update_parameters(as_conf, {}) + job._clean_runtime_parameters() for key, value in job_parameters.items(): jobs_parameters["JOBS"+"."+job.section+"."+key] = value except: @@ -4044,7 +4036,7 @@ class Autosubmit: shutil.copyfile(template_path, backup_path) template_content = open(template_path, 'r', encoding=locale.getlocale()[1]).read() # Look for %_% - variables = re.findall('%(? -1: - template_content = re.sub('(?m)^( )*(EXPID:)( )*[a-zA-Z0-9]*(\n)*', "", template_content, flags=re.I) + template_content = re.sub('(?m)^( )*(EXPID:)( )*[a-zA-Z0-9._-]*(\n)*', "", template_content, flags=re.I) # Write final result open(template_path, "w").write(template_content) @@ -4591,7 +4583,7 @@ class Autosubmit: @staticmethod def create(expid, noplot, hide, output='pdf', group_by=None, expand=list(), expand_status=list(), - notransitive=False, check_wrappers=False, detail=False, profile=False): + notransitive=False, check_wrappers=False, detail=False, profile=False, force=False): """ Creates job list for given experiment. Configuration files must be valid before executing this process. @@ -4681,11 +4673,11 @@ class Autosubmit: rerun = as_conf.get_rerun() Log.info("\nCreating the jobs list...") - job_list = JobList(expid, BasicConfig, YAMLParserFactory(), - Autosubmit._get_job_list_persistence(expid, as_conf), as_conf) - prev_job_list = Autosubmit.load_job_list( - expid, as_conf, notransitive=notransitive) - + job_list = JobList(expid, BasicConfig, YAMLParserFactory(),Autosubmit._get_job_list_persistence(expid, as_conf), as_conf) + try: + prev_job_list_logs = Autosubmit.load_logs_from_previous_run(expid, as_conf) + except: + prev_job_list_logs = None date_format = '' if as_conf.get_chunk_size_unit() == 'hour': date_format = 'H' @@ -4702,20 +4694,20 @@ class Autosubmit: continue wrapper_jobs[wrapper_name] = as_conf.get_wrapper_jobs(wrapper_parameters) - job_list.generate(date_list, member_list, num_chunks, chunk_ini, parameters, date_format, + job_list.generate(as_conf,date_list, member_list, num_chunks, chunk_ini, parameters, date_format, as_conf.get_retrials(), as_conf.get_default_job_type(), - as_conf.get_wrapper_type(), wrapper_jobs, notransitive=notransitive, - update_structure=True, run_only_members=run_only_members, - jobs_data=as_conf.experiment_data, as_conf=as_conf) + wrapper_jobs, run_only_members=run_only_members, force=force) if str(rerun).lower() == "true": job_list.rerun(as_conf.get_rerun_jobs(),as_conf) else: job_list.remove_rerun_only_jobs(notransitive) Log.info("\nSaving the jobs list...") - job_list.add_logs(prev_job_list.get_logs()) + if prev_job_list_logs: + job_list.add_logs(prev_job_list_logs) job_list.save() + as_conf.save() JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), "job_packages_" + expid).reset_table() groups_dict = dict() @@ -4760,30 +4752,14 @@ class Autosubmit: packages_persistence = JobPackagePersistence( os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), "job_packages_" + expid) packages_persistence.reset_table(True) - referenced_jobs_to_remove = set() - job_list_wrappers = copy.deepcopy(job_list) - jobs_wr = job_list_wrappers.get_job_list() - for job in jobs_wr: - for child in job.children: - if child not in jobs_wr: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs_wr: - referenced_jobs_to_remove.add(parent) - - for job in jobs_wr: - job.children = job.children - referenced_jobs_to_remove - job.parents = job.parents - referenced_jobs_to_remove + job_list_wr = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive, monitor=True, new=False) Autosubmit.generate_scripts_andor_wrappers( - as_conf, job_list_wrappers, jobs_wr, packages_persistence, True) + as_conf, job_list_wr, job_list_wr.get_job_list(), packages_persistence, True) packages = packages_persistence.load(True) else: packages = None - Log.info("\nSaving unified data..") - as_conf.save() - Log.info("") - Log.info("\nPlotting the jobs list...") monitor_exp = Monitor() # if output is set, use output @@ -4868,14 +4844,13 @@ class Autosubmit: submitter = Autosubmit._get_submitter(as_conf) submitter.load_platforms(as_conf) try: - hpcarch = submitter.platforms[as_conf.get_platform()] + hpcarch = submitter.platforms.get(as_conf.get_platform(), "local") except BaseException as e: error = str(e) try: hpcarch = submitter.platforms[as_conf.get_platform()] except Exception as e: hpcarch = "local" - #Log.warning("Remote clone may be disabled due to: " + error) return AutosubmitGit.clone_repository(as_conf, force, hpcarch) elif project_type == "svn": svn_project_url = as_conf.get_svn_project_url() @@ -4992,36 +4967,362 @@ class Autosubmit: Log.status("CHANGED: job: " + job.name + " status to: " + final) @staticmethod - def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, filter_type_chunk, + def _validate_section(as_conf,filter_section): + section_validation_error = False + section_error = False + section_not_foundList = list() + section_validation_message = "\n## Section Validation Message ##" + countStart = filter_section.count('[') + countEnd = filter_section.count(']') + if countStart > 1 or countEnd > 1: + section_validation_error = True + section_validation_message += "\n\tList of sections has a format error. Perhaps you were trying to use -fc instead." + if section_validation_error is False: + if len(str(filter_section).strip()) > 0: + if len(filter_section.split()) > 0: + jobSections = as_conf.jobs_data + for section in filter_section.split(): + # print(section) + # Provided section is not an existing section, or it is not the keyword 'Any' + if section not in jobSections and (section != "Any"): + section_error = True + section_not_foundList.append(section) + else: + section_validation_error = True + section_validation_message += "\n\tEmpty input. No changes performed." + if section_validation_error is True or section_error is True: + if section_error is True: + section_validation_message += "\n\tSpecified section(s) : [" + str(section_not_foundList) + " not found"\ + ".\n\tProcess stopped. Review the format of the provided input. Comparison is case sensitive." + \ + "\n\tRemember that this option expects section names separated by a blank space as input." + + raise AutosubmitCritical("Error in the supplied input for -ft.", 7011, section_validation_message) + @staticmethod + def _validate_list(as_conf,job_list,filter_list): + job_validation_error = False + job_error = False + job_not_foundList = list() + job_validation_message = "\n## Job Validation Message ##" + jobs = list() + countStart = filter_list.count('[') + countEnd = filter_list.count(']') + if countStart > 1 or countEnd > 1: + job_validation_error = True + job_validation_message += "\n\tList of jobs has a format error. Perhaps you were trying to use -fc instead." + + if job_validation_error is False: + for job in job_list.get_job_list(): + jobs.append(job.name) + if len(str(filter_list).strip()) > 0: + if len(filter_list.split()) > 0: + for sentJob in filter_list.split(): + # Provided job does not exist, or it is not the keyword 'Any' + if sentJob not in jobs and (sentJob != "Any"): + job_error = True + job_not_foundList.append(sentJob) + else: + job_validation_error = True + job_validation_message += "\n\tEmpty input. No changes performed." + + if job_validation_error is True or job_error is True: + if job_error is True: + job_validation_message += "\n\tSpecified job(s) : [" + str( + job_not_foundList) + "] not found in the experiment " + \ + str(as_conf.expid) + ". \n\tProcess stopped. Review the format of the provided input. Comparison is case sensitive." + \ + "\n\tRemember that this option expects job names separated by a blank space as input." + raise AutosubmitCritical( + "Error in the supplied input for -ft.", 7011, job_validation_message) + @staticmethod + def _validate_chunks(as_conf,filter_chunks): + fc_validation_message = "## -fc Validation Message ##" + fc_filter_is_correct = True + selected_sections = filter_chunks.split(",")[1:] + selected_formula = filter_chunks.split(",")[0] + current_sections = as_conf.jobs_data + fc_deserialized_json = object() + # Starting Validation + if len(str(selected_sections).strip()) == 0: + fc_filter_is_correct = False + fc_validation_message += "\n\tMust include a section (job type)." + else: + for section in selected_sections: + # section = section.strip() + # Validating empty sections + if len(str(section).strip()) == 0: + fc_filter_is_correct = False + fc_validation_message += "\n\tEmpty sections are not accepted." + break + # Validating existing sections + # Retrieve experiment data + + if section not in current_sections: + fc_filter_is_correct = False + fc_validation_message += "\n\tSection " + section + \ + " does not exist in experiment. Remember not to include blank spaces." + + # Validating chunk formula + if len(selected_formula) == 0: + fc_filter_is_correct = False + fc_validation_message += "\n\tA formula for chunk filtering has not been provided." + + # If everything is fine until this point + if fc_filter_is_correct is True: + # Retrieve experiment data + current_dates = as_conf.experiment_data["EXPERIMENT"]["DATELIST"].split() + current_members = as_conf.get_member_list() + # Parse json + try: + fc_deserialized_json = json.loads( + Autosubmit._create_json(selected_formula)) + except Exception as e: + fc_filter_is_correct = False + fc_validation_message += "\n\tProvided chunk formula does not have the right format. Were you trying to use another option?" + if fc_filter_is_correct is True: + for startingDate in fc_deserialized_json['sds']: + if startingDate['sd'] not in current_dates: + fc_filter_is_correct = False + fc_validation_message += "\n\tStarting date " + \ + startingDate['sd'] + \ + " does not exist in experiment." + for member in startingDate['ms']: + if member['m'] not in current_members and member['m'].lower() != "any": + fc_filter_is_correct = False + fc_validation_message += "\n\tMember " + \ + member['m'] + \ + " does not exist in experiment." + + # Ending validation + if fc_filter_is_correct is False: + raise AutosubmitCritical( + "Error in the supplied input for -fc.", 7011, fc_validation_message) + @staticmethod + def _validate_status(job_list,filter_status): + status_validation_error = False + status_validation_message = "\n## Status Validation Message ##" + # Trying to identify chunk formula + countStart = filter_status.count('[') + countEnd = filter_status.count(']') + if countStart > 1 or countEnd > 1: + status_validation_error = True + status_validation_message += "\n\tList of status provided has a format error. Perhaps you were trying to use -fc instead." + # If everything is fine until this point + if status_validation_error is False: + status_filter = filter_status.split() + status_reference = Status() + status_list = list() + for job in job_list.get_job_list(): + reference = status_reference.VALUE_TO_KEY[job.status] + if reference not in status_list: + status_list.append(reference) + for status in status_filter: + if status not in status_list: + status_validation_error = True + status_validation_message += "\n\t There are no jobs with status " + \ + status + " in this experiment." + if status_validation_error is True: + raise AutosubmitCritical("Error in the supplied input for -fs.", 7011, status_validation_message) + + @staticmethod + def _validate_type_chunk(as_conf,filter_type_chunk): + #Change status by section, member, and chunk; freely. + # Including inner validation. Trying to make it independent. + # 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16-30] ] ],SIM,SIM2,SIM3 + validation_message = "## -ftc Validation Message ##" + filter_is_correct = True + selected_sections = filter_type_chunk.split(",")[1:] + selected_formula = filter_type_chunk.split(",")[0] + deserialized_json = object() + # Starting Validation + if len(str(selected_sections).strip()) == 0: + filter_is_correct = False + validation_message += "\n\tMust include a section (job type). If you want to apply the changes to all sections, include 'Any'." + else: + for section in selected_sections: + # Validating empty sections + if len(str(section).strip()) == 0: + filter_is_correct = False + validation_message += "\n\tEmpty sections are not accepted." + break + # Validating existing sections + # Retrieve experiment data + current_sections = as_conf.jobs_data + if section not in current_sections and section != "Any": + filter_is_correct = False + validation_message += "\n\tSection " + \ + section + " does not exist in experiment." + + # Validating chunk formula + if len(selected_formula) == 0: + filter_is_correct = False + validation_message += "\n\tA formula for chunk filtering has not been provided. If you want to change all chunks, include 'Any'." + + if filter_is_correct is False: + raise AutosubmitCritical( + "Error in the supplied input for -ftc.", 7011, validation_message) + + @staticmethod + def _validate_chunk_split(as_conf,filter_chunk_split): + # new filter + pass + @staticmethod + def _validate_set_status_filters(as_conf,job_list,filter_list,filter_chunks,filter_status,filter_section,filter_type_chunk, filter_chunk_split): + if filter_section is not None: + Autosubmit._validate_section(as_conf,filter_section) + if filter_list is not None: + Autosubmit._validate_list(as_conf,job_list,filter_list) + if filter_chunks is not None: + Autosubmit._validate_chunks(as_conf,filter_chunks) + if filter_status is not None: + Autosubmit._validate_status(job_list,filter_status) + if filter_type_chunk is not None: + Autosubmit._validate_type_chunk(as_conf,filter_type_chunk) + if filter_chunk_split is not None: + Autosubmit._validate_chunk_split(as_conf,filter_chunk_split) + + @staticmethod + def _apply_ftc(job_list,filter_type_chunk_split): + """ + Accepts a string with the formula: "[ 19601101 [ fc0 [1 [1] 2 [2 3] 3 4] Any [1] ] 19651101 [ fc0 [16 30] ] ],SIM [ Any ] ,SIM2 [ 1 2]" + Where SIM, SIM2 are section (job types) names that also accept the keyword "Any" so the changes apply to all sections. + Starting Date (19601101) does not accept the keyword "Any", so you must specify the starting dates to be changed. + You can also specify date ranges to apply the change to a range on dates. + Member names (fc0) accept the keyword "Any", so the chunks ([1 2 3 4]) given will be updated for all members. + Chunks must be in the format "[1 2 3 4]" where "1 2 3 4" represent the numbers of the chunks in the member, + Splits must be in the format "[ 1 2 3 4]" where "1 2 3 4" represent the numbers of the splits in the sections. + no range format is allowed. + :param filter_type_chunk_split: string with the formula + :return: final_list + """ + # Get selected sections and formula + final_list = [] + selected_sections = filter_type_chunk_split.split(",")[1:] + selected_formula = filter_type_chunk_split.split(",")[0] + # Retrieve experiment data + # Parse json + deserialized_json = json.loads(Autosubmit._create_json(selected_formula)) + # Get current list + working_list = job_list.get_job_list() + for section in selected_sections: + if str(section).upper() == "ANY": + # Any section + section_selection = working_list + # Go through start dates + for starting_date in deserialized_json['sds']: + date = starting_date['sd'] + date_selection = [j for j in section_selection if date2str( + j.date) == date] + # Members for given start date + for member_group in starting_date['ms']: + member = member_group['m'] + if str(member).upper() == "ANY": + # Any member + member_selection = date_selection + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = [j for j in member_selection if j.chunk == int(chunk)] + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in [j for j in date_selection if + j.chunk == int(chunk) and j.synchronize is not None]: + final_list.append(job) + else: + # Selected members + member_selection = [j for j in date_selection if j.member == member] + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = [j for j in member_selection if j.chunk == int(chunk)] + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in [j for j in date_selection if + j.chunk == int(chunk) and j.synchronize is not None]: + final_list.append(job) + else: + # Only given section + section_splits = section.split("[") + section = section_splits[0].strip(" [") + if len(section_splits) > 1: + if "," in section_splits[1]: + splits = section_splits[1].strip(" ]").split(",") + else: + splits = section_splits[1].strip(" ]").split(" ") + else: + splits = ["ANY"] + final_splits = [] + for split in splits: + start = None + end = None + if split.find("-") != -1: + start = split.split("-")[0] + end = split.split("-")[1] + if split.find(":") != -1: + start = split.split(":")[0] + end = split.split(":")[1] + if start and end: + final_splits += [ str(i) for i in range(int(start),int(end)+1)] + else: + final_splits.append(str(split)) + splits = final_splits + jobs_filtered = [j for j in working_list if j.section == section and ( j.split is None or splits[0] == "ANY" or str(j.split) in splits ) ] + # Go through start dates + for starting_date in deserialized_json['sds']: + date = starting_date['sd'] + date_selection = [j for j in jobs_filtered if date2str( + j.date) == date] + # Members for given start date + for member_group in starting_date['ms']: + member = member_group['m'] + if str(member).upper() == "ANY": + # Any member + member_selection = date_selection + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = [j for j in member_selection if + j.chunk is None or j.chunk == int(chunk)] + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in [j for j in date_selection if + j.chunk == int(chunk) and j.synchronize is not None]: + final_list.append(job) + else: + # Selected members + member_selection = [j for j in date_selection if j.member == member] + chunk_group = member_group['cs'] + for chunk in chunk_group: + filtered_job = [j for j in member_selection if j.chunk == int(chunk)] + for job in filtered_job: + final_list.append(job) + # From date filter and sync is not None + for job in [j for j in date_selection if + j.chunk == int(chunk) and j.synchronize is not None]: + final_list.append(job) + return final_list + @staticmethod + def set_status(expid, noplot, save, final, filter_list, filter_chunks, filter_status, filter_section, filter_type_chunk, filter_type_chunk_split, hide, group_by=None, expand=list(), expand_status=list(), notransitive=False, check_wrapper=False, detail=False): """ - Set status - - :param detail: - :param check_wrapper: - :param notransitive: - :param expand_status: - :param expand: - :param group_by: - :param filter_type_chunk: - :param noplot: - :param expid: experiment identifier - :type expid: str - :param save: if true, saves the new jobs list - :type save: bool - :param final: status to set on jobs - :type final: str - :param lst: list of jobs to change status - :type lst: str - :param filter_chunks: chunks to change status - :type filter_chunks: str - :param filter_status: current status of the jobs to change status - :type filter_status: str - :param filter_section: sections to change status - :type filter_section: str - :param hide: hides plot window - :type hide: bool + Set status of jobs + :param expid: experiment id + :param noplot: do not plot + :param save: save + :param final: final status + :param filter_list: list of jobs + :param filter_chunks: filter chunks + :param filter_status: filter status + :param filter_section: filter section + :param filter_type_chunk: filter type chunk + :param filter_chunk_split: filter chunk split + :param hide: hide + :param group_by: group by + :param expand: expand + :param expand_status: expand status + :param notransitive: notransitive + :param check_wrapper: check wrapper + :param detail: detail + :return: """ Autosubmit._check_ownership(expid, raise_error=True) exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) @@ -5037,10 +5338,11 @@ class Autosubmit: Log.debug('Exp ID: {0}', expid) Log.debug('Save: {0}', save) Log.debug('Final status: {0}', final) - Log.debug('List of jobs to change: {0}', lst) + Log.debug('List of jobs to change: {0}', filter_list) Log.debug('Chunks to change: {0}', filter_chunks) Log.debug('Status of jobs to change: {0}', filter_status) Log.debug('Sections to change: {0}', filter_section) + wrongExpid = 0 as_conf = AutosubmitConfig( expid, BasicConfig, YAMLParserFactory()) @@ -5049,46 +5351,8 @@ class Autosubmit: # Getting output type from configuration output_type = as_conf.get_output_type() # Getting db connections - - # Validating job sections, if filter_section -ft has been set: - if filter_section is not None: - section_validation_error = False - section_error = False - section_not_foundList = list() - section_validation_message = "\n## Section Validation Message ##" - countStart = filter_section.count('[') - countEnd = filter_section.count(']') - if countStart > 1 or countEnd > 1: - section_validation_error = True - section_validation_message += "\n\tList of sections has a format error. Perhaps you were trying to use -fc instead." - # countUnderscore = filter_section.count('_') - # if countUnderscore > 1: - # section_validation_error = True - # section_validation_message += "\n\tList of sections provided has a format error. Perhaps you were trying to use -fl instead." - if section_validation_error is False: - if len(str(filter_section).strip()) > 0: - if len(filter_section.split()) > 0: - jobSections = as_conf.jobs_data - for section in filter_section.split(): - # print(section) - # Provided section is not an existing section, or it is not the keyword 'Any' - if section not in jobSections and (section != "Any"): - section_error = True - section_not_foundList.append(section) - else: - section_validation_error = True - section_validation_message += "\n\tEmpty input. No changes performed." - if section_validation_error is True or section_error is True: - if section_error is True: - section_validation_message += "\n\tSpecified section(s) : [" + str(section_not_foundList) + \ - "] not found in the experiment " + str(expid) + \ - ".\n\tProcess stopped. Review the format of the provided input. Comparison is case sensitive." + \ - "\n\tRemember that this option expects section names separated by a blank space as input." - - raise AutosubmitCritical( - "Error in the supplied input for -ft.", 7011, section_validation_message+job_validation_message) - job_list = Autosubmit.load_job_list( - expid, as_conf, notransitive=notransitive) + # To be added in a function that checks which platforms must be connected to + job_list = Autosubmit.load_job_list(expid, as_conf, notransitive=notransitive, monitor=True, new=False) submitter = Autosubmit._get_submitter(as_conf) submitter.load_platforms(as_conf) hpcarch = as_conf.get_platform() @@ -5107,8 +5371,7 @@ class Autosubmit: job.platform = platforms[job.platform_name] # noinspection PyTypeChecker if job.status in [Status.QUEUING, Status.SUBMITTED, Status.RUNNING]: - platforms_to_test.add( - platforms[job.platform_name]) + platforms_to_test.add(platforms[job.platform_name]) # establish the connection to all platforms definitive_platforms = list() for platform in platforms_to_test: @@ -5117,340 +5380,44 @@ class Autosubmit: definitive_platforms.append(platform.name) except Exception as e: pass - - # Validating list of jobs, if filter_list -fl has been set: - # Seems that Autosubmit.load_job_list call is necessary before verification is executed - if job_list is not None and lst is not None: - job_validation_error = False - job_error = False - job_not_foundList = list() - job_validation_message = "\n## Job Validation Message ##" - jobs = list() - countStart = lst.count('[') - countEnd = lst.count(']') - if countStart > 1 or countEnd > 1: - job_validation_error = True - job_validation_message += "\n\tList of jobs has a format error. Perhaps you were trying to use -fc instead." - - if job_validation_error is False: - for job in job_list.get_job_list(): - jobs.append(job.name) - if len(str(lst).strip()) > 0: - if len(lst.split()) > 0: - for sentJob in lst.split(): - # Provided job does not exist, or it is not the keyword 'Any' - if sentJob not in jobs and (sentJob != "Any"): - job_error = True - job_not_foundList.append(sentJob) - else: - job_validation_error = True - job_validation_message += "\n\tEmpty input. No changes performed." - - if job_validation_error is True or job_error is True: - if job_error is True: - job_validation_message += "\n\tSpecified job(s) : [" + str( - job_not_foundList) + "] not found in the experiment " + \ - str(expid) + ". \n\tProcess stopped. Review the format of the provided input. Comparison is case sensitive." + \ - "\n\tRemember that this option expects job names separated by a blank space as input." - raise AutosubmitCritical( - "Error in the supplied input for -ft.", 7011, section_validation_message+job_validation_message) - - # Validating fc if filter_chunks -fc has been set: - if filter_chunks is not None: - fc_validation_message = "## -fc Validation Message ##" - fc_filter_is_correct = True - selected_sections = filter_chunks.split(",")[1:] - selected_formula = filter_chunks.split(",")[0] - current_sections = as_conf.jobs_data - fc_deserializedJson = object() - # Starting Validation - if len(str(selected_sections).strip()) == 0: - fc_filter_is_correct = False - fc_validation_message += "\n\tMust include a section (job type)." - else: - for section in selected_sections: - # section = section.strip() - # Validating empty sections - if len(str(section).strip()) == 0: - fc_filter_is_correct = False - fc_validation_message += "\n\tEmpty sections are not accepted." - break - # Validating existing sections - # Retrieve experiment data - - if section not in current_sections: - fc_filter_is_correct = False - fc_validation_message += "\n\tSection " + section + \ - " does not exist in experiment. Remember not to include blank spaces." - - # Validating chunk formula - if len(selected_formula) == 0: - fc_filter_is_correct = False - fc_validation_message += "\n\tA formula for chunk filtering has not been provided." - - # If everything is fine until this point - if fc_filter_is_correct is True: - # Retrieve experiment data - current_dates = as_conf.experiment_data["EXPERIMENT"]["DATELIST"].split() - current_members = as_conf.get_member_list() - # Parse json - try: - fc_deserializedJson = json.loads( - Autosubmit._create_json(selected_formula)) - except Exception as e: - fc_filter_is_correct = False - fc_validation_message += "\n\tProvided chunk formula does not have the right format. Were you trying to use another option?" - if fc_filter_is_correct is True: - for startingDate in fc_deserializedJson['sds']: - if startingDate['sd'] not in current_dates: - fc_filter_is_correct = False - fc_validation_message += "\n\tStarting date " + \ - startingDate['sd'] + \ - " does not exist in experiment." - for member in startingDate['ms']: - if member['m'] not in current_members and member['m'].lower() != "any": - fc_filter_is_correct = False - fc_validation_message += "\n\tMember " + \ - member['m'] + \ - " does not exist in experiment." - - # Ending validation - if fc_filter_is_correct is False: - section_validation_message = fc_validation_message - raise AutosubmitCritical( - "Error in the supplied input for -fc.", 7011, section_validation_message+job_validation_message) - # Validating status, if filter_status -fs has been set: - # At this point we already have job_list from where we are getting the allows STATUS - if filter_status is not None: - status_validation_error = False - status_validation_message = "\n## Status Validation Message ##" - # Trying to identify chunk formula - countStart = filter_status.count('[') - countEnd = filter_status.count(']') - if countStart > 1 or countEnd > 1: - status_validation_error = True - status_validation_message += "\n\tList of status provided has a format error. Perhaps you were trying to use -fc instead." - # Trying to identify job names, implying status names won't use more than 1 underscore _ - # countUnderscore = filter_status.count('_') - # if countUnderscore > 1: - # status_validation_error = True - # status_validation_message += "\n\tList of status provided has a format error. Perhaps you were trying to use -fl instead." - # If everything is fine until this point - if status_validation_error is False: - status_filter = filter_status.split() - status_reference = Status() - status_list = list() - for job in job_list.get_job_list(): - reference = status_reference.VALUE_TO_KEY[job.status] - if reference not in status_list: - status_list.append(reference) - for status in status_filter: - if status not in status_list: - status_validation_error = True - status_validation_message += "\n\t There are no jobs with status " + \ - status + " in this experiment." - if status_validation_error is True: - raise AutosubmitCritical("Error in the supplied input for -fs.{0}".format( - status_validation_message), 7011, section_validation_message+job_validation_message) - + ##### End of the ""function"" + # This will raise an autosubmit critical if any of the filters has issues in the format specified by the user + Autosubmit._validate_set_status_filters(as_conf,job_list,filter_list,filter_chunks,filter_status,filter_section,filter_type_chunk, filter_type_chunk_split) + #### Starts the filtering process #### + final_list = [] jobs_filtered = [] + jobs_left_to_be_filtered = True final_status = Autosubmit._get_status(final) - if filter_section or filter_chunks: - if filter_section: - ft = filter_section.split() - else: - ft = filter_chunks.split(",")[1:] - if ft == 'Any': + # I have the impression that whoever did this function thought about the possibility of having multiple filters at the same time + # But, as it was, it is not possible to have multiple filters at the same time due to the way the code is written + if filter_section: + ft = filter_section.split() + if str(ft).upper() == 'ANY': for job in job_list.get_job_list(): - Autosubmit.change_status( - final, final_status, job, save) + final_list.append(job) else: for section in ft: for job in job_list.get_job_list(): if job.section == section: - if filter_chunks: - jobs_filtered.append(job) - else: - Autosubmit.change_status( - final, final_status, job, save) - - # New feature : Change status by section, member, and chunk; freely. - # Including inner validation. Trying to make it independent. - # 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16-30] ] ],SIM,SIM2,SIM3 - if filter_type_chunk: - validation_message = "## -ftc Validation Message ##" - filter_is_correct = True - selected_sections = filter_type_chunk.split(",")[1:] - selected_formula = filter_type_chunk.split(",")[0] - deserializedJson = object() - performed_changes = dict() - - # Starting Validation - if len(str(selected_sections).strip()) == 0: - filter_is_correct = False - validation_message += "\n\tMust include a section (job type). If you want to apply the changes to all sections, include 'Any'." - else: - for section in selected_sections: - # Validating empty sections - if len(str(section).strip()) == 0: - filter_is_correct = False - validation_message += "\n\tEmpty sections are not accepted." - break - # Validating existing sections - # Retrieve experiment data - current_sections = as_conf.jobs_data - if section not in current_sections and section != "Any": - filter_is_correct = False - validation_message += "\n\tSection " + \ - section + " does not exist in experiment." - - # Validating chunk formula - if len(selected_formula) == 0: - filter_is_correct = False - validation_message += "\n\tA formula for chunk filtering has not been provided. If you want to change all chunks, include 'Any'." - - # If everything is fine until this point - if filter_is_correct is True: - # Retrieve experiment data - current_dates = as_conf.experiment_data["EXPERIMENT"]["DATELIST"].split() - current_members = as_conf.get_member_list() - # Parse json - try: - deserializedJson = json.loads( - Autosubmit._create_json(selected_formula)) - except Exception as e: - filter_is_correct = False - validation_message += "\n\tProvided chunk formula does not have the right format. Were you trying to use another option?" - if filter_is_correct is True: - for startingDate in deserializedJson['sds']: - if startingDate['sd'] not in current_dates: - filter_is_correct = False - validation_message += "\n\tStarting date " + \ - startingDate['sd'] + \ - " does not exist in experiment." - for member in startingDate['ms']: - if member['m'] not in current_members and member['m'] != "Any": - filter_is_correct_ = False - validation_message += "\n\tMember " + \ - member['m'] + \ - " does not exist in experiment." - - # Ending validation - if filter_is_correct is False: - raise AutosubmitCritical( - "Error in the supplied input for -ftc.", 7011, section_validation_message+job_validation_message) - - # If input is valid, continue. - record = dict() - final_list = [] - # Get current list - working_list = job_list.get_job_list() - for section in selected_sections: - if section == "Any": - # Any section - section_selection = working_list - # Go through start dates - for starting_date in deserializedJson['sds']: - date = starting_date['sd'] - date_selection = [j for j in section_selection if date2str( - j.date) == date] - # Members for given start date - for member_group in starting_date['ms']: - member = member_group['m'] - if member == "Any": - # Any member - member_selection = date_selection - chunk_group = member_group['cs'] - for chunk in chunk_group: - filtered_job = [j for j in member_selection if j.chunk == int(chunk)] - for job in filtered_job: - final_list.append(job) - # From date filter and sync is not None - for job in [j for j in date_selection if - j.chunk == int(chunk) and j.synchronize is not None]: - final_list.append(job) - else: - # Selected members - member_selection = [j for j in date_selection if j.member == member] - chunk_group = member_group['cs'] - for chunk in chunk_group: - filtered_job = [j for j in member_selection if j.chunk == int(chunk)] - for job in filtered_job: - final_list.append(job) - # From date filter and sync is not None - for job in [j for j in date_selection if - j.chunk == int(chunk) and j.synchronize is not None]: - final_list.append(job) - else: - # Only given section - section_selection = [j for j in working_list if j.section == section] - # Go through start dates - for starting_date in deserializedJson['sds']: - date = starting_date['sd'] - date_selection = [j for j in section_selection if date2str( - j.date) == date] - # Members for given start date - for member_group in starting_date['ms']: - member = member_group['m'] - if member == "Any": - # Any member - member_selection = date_selection - chunk_group = member_group['cs'] - for chunk in chunk_group: - filtered_job = [j for j in member_selection if - j.chunk is None or j.chunk == int(chunk)] - for job in filtered_job: - final_list.append(job) - # From date filter and sync is not None - for job in [j for j in date_selection if - j.chunk == int(chunk) and j.synchronize is not None]: - final_list.append(job) - else: - # Selected members - member_selection = [j for j in date_selection if j.member == member] - chunk_group = member_group['cs'] - for chunk in chunk_group: - filtered_job = [j for j in member_selection if j.chunk == int(chunk)] - for job in filtered_job: - final_list.append(job) - # From date filter and sync is not None - for job in [j for j in date_selection if - j.chunk == int(chunk) and j.synchronize is not None]: - final_list.append(job) - status = Status() - for job in final_list: - if job.status in [Status.QUEUING, Status.RUNNING, - Status.SUBMITTED] and job.platform.name not in definitive_platforms: - Log.printlog("JOB: [{1}] is ignored as the [{0}] platform is currently offline".format( - job.platform.name, job.name), 6000) - continue - if job.status != final_status: - # Only real changes - performed_changes[job.name] = str( - Status.VALUE_TO_KEY[job.status]) + " -> " + str(final) - Autosubmit.change_status( - final, final_status, job, save) - # If changes have been performed - if len(list(performed_changes.keys())) > 0: - if detail: - Autosubmit.detail(job_list) - else: - Log.warning("No changes were performed.") - # End of New Feature - + final_list.append(job) if filter_chunks: + ft = filter_chunks.split(",")[1:] + # Any located in section part + if str(ft).upper() == "ANY": + for job in job_list.get_job_list(): + final_list.append(job) + for job in job_list.get_job_list(): + if job.section == section: + if filter_chunks: + jobs_filtered.append(job) if len(jobs_filtered) == 0: jobs_filtered = job_list.get_job_list() - fc = filter_chunks - Log.debug(fc) - - if fc == 'Any': + # Any located in chunks part + if str(fc).upper() == "ANY": for job in jobs_filtered: - Autosubmit.change_status( - final, final_status, job, save) + final_list.append(job) else: - # noinspection PyTypeChecker data = json.loads(Autosubmit._create_json(fc)) for date_json in data['sds']: date = date_json['sd'] @@ -5474,49 +5441,73 @@ class Autosubmit: for chunk_json in member_json['cs']: chunk = int(chunk_json) for job in [j for j in jobs_date if j.chunk == chunk and j.synchronize is not None]: - Autosubmit.change_status( - final, final_status, job, save) - + final_list.append(job) for job in [j for j in jobs_member if j.chunk == chunk]: - Autosubmit.change_status( - final, final_status, job, save) - + final_list.append(job) if filter_status: status_list = filter_status.split() - Log.debug("Filtering jobs with status {0}", filter_status) - if status_list == 'Any': + if str(status_list).upper() == 'ANY': for job in job_list.get_job_list(): - Autosubmit.change_status( - final, final_status, job, save) + final_list.append(job) else: for status in status_list: fs = Autosubmit._get_status(status) for job in [j for j in job_list.get_job_list() if j.status == fs]: - Autosubmit.change_status( - final, final_status, job, save) + final_list.append(job) - if lst: - jobs = lst.split() + if filter_list: + jobs = filter_list.split() expidJoblist = defaultdict(int) - for x in lst.split(): + for x in filter_list.split(): expidJoblist[str(x[0:4])] += 1 - if str(expid) in expidJoblist: wrongExpid = jobs.__len__() - expidJoblist[expid] if wrongExpid > 0: Log.warning( "There are {0} job.name with an invalid Expid", wrongExpid) - - if jobs == 'Any': + if str(jobs).upper() == 'ANY': for job in job_list.get_job_list(): - Autosubmit.change_status( - final, final_status, job, save) + final_list.append(job) else: for job in job_list.get_job_list(): if job.name in jobs: - Autosubmit.change_status( - final, final_status, job, save) + final_list.append(job) + # All filters should be in a function but no have time to do it + # filter_Type_chunk_split == filter_type_chunk, but with the split essencially is the same but not sure about of changing the name to the filter itself + if filter_type_chunk_split is not None: + final_list.extend(Autosubmit._apply_ftc(job_list,filter_type_chunk_split)) + if filter_type_chunk: + final_list.extend(Autosubmit._apply_ftc(job_list,filter_type_chunk)) + # Time to change status + final_list = list(set(final_list)) + performed_changes = {} + for job in final_list: + if job.status in [Status.QUEUING, Status.RUNNING, + Status.SUBMITTED] and job.platform.name not in definitive_platforms: + Log.printlog("JOB: [{1}] is ignored as the [{0}] platform is currently offline".format( + job.platform.name, job.name), 6000) + continue + if job.status != final_status: + # Only real changes + performed_changes[job.name] = str( + Status.VALUE_TO_KEY[job.status]) + " -> " + str(final) + Autosubmit.change_status( + final, final_status, job, save) + # If changes have been performed + if performed_changes: + if detail: + current_length = len(job_list.get_job_list()) + if current_length > 1000: + Log.warning( + "-d option: Experiment has too many jobs to be printed in the terminal. Maximum job quantity is 1000, your experiment has " + str( + current_length) + " jobs.") + else: + Log.info(job_list.print_with_status( + statusChange=performed_changes)) + else: + Log.warning("No changes were performed.") + job_list.update_list(as_conf, False, True) @@ -5533,37 +5524,25 @@ class Autosubmit: else: Log.printlog( "Changes NOT saved to the JobList!!!!: use -s option to save", 3000) - - if as_conf.get_wrapper_type() != 'none' and check_wrapper: - packages_persistence = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), - "job_packages_" + expid) - os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, - expid, "pkl", "job_packages_" + expid + ".db"), 0o775) - packages_persistence.reset_table(True) - referenced_jobs_to_remove = set() - job_list_wrappers = copy.deepcopy(job_list) - jobs_wr = copy.deepcopy(job_list.get_job_list()) - [job for job in jobs_wr if ( - job.status != Status.COMPLETED)] - for job in jobs_wr: - for child in job.children: - if child not in jobs_wr: - referenced_jobs_to_remove.add(child) - for parent in job.parents: - if parent not in jobs_wr: - referenced_jobs_to_remove.add(parent) - - for job in jobs_wr: - job.children = job.children - referenced_jobs_to_remove - job.parents = job.parents - referenced_jobs_to_remove - Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wrappers, jobs_wr, - packages_persistence, True) - - packages = packages_persistence.load(True) - else: - packages = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), - "job_packages_" + expid).load() + #Visualization stuff that should be in a function common to monitor , create, -cw flag, inspect and so on if not noplot: + if as_conf.get_wrapper_type() != 'none' and check_wrapper: + packages_persistence = JobPackagePersistence( + os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), + "job_packages_" + expid) + os.chmod(os.path.join(BasicConfig.LOCAL_ROOT_DIR, + expid, "pkl", "job_packages_" + expid + ".db"), 0o775) + packages_persistence.reset_table(True) + job_list_wr = Autosubmit.load_job_list( + expid, as_conf, notransitive=notransitive, monitor=True, new=False) + + Autosubmit.generate_scripts_andor_wrappers(as_conf, job_list_wr, job_list_wr.get_job_list(), + packages_persistence, True) + + packages = packages_persistence.load(True) + else: + packages = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), + "job_packages_" + expid).load() groups_dict = dict() if group_by: status = list() @@ -5587,11 +5566,7 @@ class Autosubmit: show=not hide, groups=groups_dict, job_list_object=job_list) - - if not filter_type_chunk and detail is True: - Log.warning("-d option only works with -ftc.") return True - except (portalocker.AlreadyLocked, portalocker.LockException) as e: message = "We have detected that there is another Autosubmit instance using the experiment\n. Stop other Autosubmit instances that are using the experiment or delete autosubmit.lock file located on tmp folder" raise AutosubmitCritical(message, 7000) @@ -5928,7 +5903,21 @@ class Autosubmit: open(as_conf.experiment_file, 'wb').write(content) @staticmethod - def load_job_list(expid, as_conf, notransitive=False, monitor=False): + def load_logs_from_previous_run(expid,as_conf): + logs = None + if Path(f'{BasicConfig.LOCAL_ROOT_DIR}/{expid}/pkl/job_list_{expid}.pkl').exists(): + job_list = JobList(expid, BasicConfig, YAMLParserFactory(),Autosubmit._get_job_list_persistence(expid, as_conf), as_conf) + with suppress(BaseException): + graph = job_list.load() + if len(graph.nodes) > 0: + # fast-look if graph existed, skips some steps + job_list._job_list = [job["job"] for _, job in graph.nodes.data() if + job.get("job", None)] + logs = job_list.get_logs() + del job_list + return logs + @staticmethod + def load_job_list(expid, as_conf, notransitive=False, monitor=False, new = True): rerun = as_conf.get_rerun() job_list = JobList(expid, BasicConfig, YAMLParserFactory(), @@ -5948,11 +5937,11 @@ class Autosubmit: if isinstance(wrapper_data, collections.abc.Mapping): wrapper_jobs[wrapper_section] = wrapper_data.get("JOBS_IN_WRAPPER", "") - job_list.generate(date_list, as_conf.get_member_list(), as_conf.get_num_chunks(), as_conf.get_chunk_ini(), + job_list.generate(as_conf, date_list, as_conf.get_member_list(), as_conf.get_num_chunks(), as_conf.get_chunk_ini(), as_conf.experiment_data, date_format, as_conf.get_retrials(), - as_conf.get_default_job_type(), as_conf.get_wrapper_type(), wrapper_jobs, - new=False, notransitive=notransitive, run_only_members=run_only_members, - jobs_data=as_conf.experiment_data, as_conf=as_conf) + as_conf.get_default_job_type(), wrapper_jobs, + new=new, run_only_members=run_only_members,monitor=monitor) + if str(rerun).lower() == "true": rerun_jobs = as_conf.get_rerun_jobs() job_list.rerun(rerun_jobs,as_conf, monitor=monitor) diff --git a/autosubmit/database/db_structure.py b/autosubmit/database/db_structure.py index b42854359edc2dea0c71c0b682057fcee0d28e89..31dc42740a56c6536355b06ce6dfaba255d4a77c 100644 --- a/autosubmit/database/db_structure.py +++ b/autosubmit/database/db_structure.py @@ -25,9 +25,6 @@ import sqlite3 from typing import Dict, List from log.log import Log -# from networkx import DiGraph - -# DB_FILE_AS_TIMES = "/esarchive/autosubmit/as_times.db" def get_structure(exp_id, structures_path): diff --git a/autosubmit/experiment/statistics.py b/autosubmit/experiment/statistics.py index 793210923d0e28c3325cc1ad098e2dc73621d51c..0188f00811c9f5dc0ec0af9520fe3b6cefefb25a 100644 --- a/autosubmit/experiment/statistics.py +++ b/autosubmit/experiment/statistics.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . +import math import datetime from autosubmit.job.job import Job from autosubmit.monitor.utils import FixedSizeList @@ -107,6 +108,31 @@ class ExperimentStats(object): def fail_run(self): return FixedSizeList(self._fail_run, 0.0) + def _estimate_requested_nodes(self,nodes,processors,tasks,processors_per_node) -> int: + if str(nodes).isdigit(): + return int(nodes) + elif str(tasks).isdigit(): + return math.ceil(int(processors) / int(tasks)) + elif str(processors_per_node).isdigit() and int(processors) > int(processors_per_node): + return math.ceil(int(processors) / int(processors_per_node)) + else: + return 1 + + def _calculate_processing_elements(self,nodes,processors,tasks,processors_per_node,exclusive) -> int: + if str(processors_per_node).isdigit(): + if str(nodes).isdigit(): + return int(nodes) * int(processors_per_node) + else: + estimated_nodes = self._estimate_requested_nodes(nodes,processors,tasks,processors_per_node) + if not exclusive and estimated_nodes <= 1 and int(processors) <= int(processors_per_node): + return int(processors) + else: + return estimated_nodes * int(processors_per_node) + elif (str(tasks).isdigit() or str(nodes).isdigit()): + Log.warning(f'Missing PROCESSORS_PER_NODE. Should be set if TASKS or NODES are defined. The PROCESSORS will used instead.') + return int(processors) + + def _calculate_stats(self): """ Main calculation @@ -116,6 +142,10 @@ class ExperimentStats(object): for i, job in enumerate(self._jobs_list): last_retrials = job.get_last_retrials() processors = job.total_processors + nodes = job.nodes + tasks = job.tasks + processors_per_node = job.processors_per_node + processors = self._calculate_processing_elements(nodes, processors, tasks, processors_per_node, job.exclusive) for retrial in last_retrials: if Job.is_a_completed_retrial(retrial): # The retrial has all necessary values and is status COMPLETED @@ -158,8 +188,7 @@ class ExperimentStats(object): self._total_jobs_run += len(last_retrials) self._total_jobs_failed += self.failed_jobs[i] self._threshold = max(self._threshold, job.total_wallclock) - self._expected_cpu_consumption += job.total_wallclock * \ - int(processors) + self._expected_cpu_consumption += job.total_wallclock * int(processors) self._expected_real_consumption += job.total_wallclock self._total_queueing_time += self._queued[i] diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 0527bf755ce9ba4fbf1545c98a365d8acb033133..9508340e486534ed6b78eb8c6ba0a41e9cdbd2c7 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -137,34 +137,45 @@ class Job(object): CHECK_ON_SUBMISSION = 'on_submission' + # TODO + # This is crashing the code + # I added it for the assertions of unit testing... since job obj != job obj when it was saved & load + # since it points to another section of the memory. + # Unfortunatelly, this is crashing the code everywhere else + + # def __eq__(self, other): + # return self.name == other.name and self.id == other.id + def __str__(self): return "{0} STATUS: {1}".format(self.name, self.status) + def __repr__(self): + return "{0} STATUS: {1}".format(self.name, self.status) + def __init__(self, name, job_id, status, priority): self.splits = None + self.rerun_only = False self.script_name_wrapper = None - self.delay_end = datetime.datetime.now() - self._delay_retrials = "0" + self.retrials = None + self.delay_end = None + self.delay_retrials = None self.wrapper_type = None self._wrapper_queue = None self._platform = None self._queue = None self._partition = None - - self.retry_delay = "0" - self.platform_name = None # type: str + self.retry_delay = None #: (str): Type of the job, as given on job configuration file. (job: TASKTYPE) self._section = None # type: str self._wallclock = None # type: str self.wchunkinc = None - self._tasks = '1' - self._nodes = "" - self.default_parameters = {'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', - 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'} - self._threads = '1' - self._processors = '1' - self._memory = '' - self._memory_per_task = '' + self._tasks = None + self._nodes = None + self.default_parameters = None + self._threads = None + self._processors = None + self._memory = None + self._memory_per_task = None self._chunk = None self._member = None self.date = None @@ -179,9 +190,9 @@ class Job(object): self.long_name = name self.date_format = '' self.type = Type.BASH - self._hyperthreading = "none" - self._scratch_free_space = None - self._custom_directives = [] + self.hyperthreading = None + self.scratch_free_space = None + self.custom_directives = [] self.undefined_variables = set() self.log_retries = 5 self.id = job_id @@ -202,7 +213,7 @@ class Job(object): #: (int) Number of failed attempts to run this job. (FAIL_COUNT) self._fail_count = 0 self.expid = name.split('_')[0] # type: str - self.parameters = dict() + self.parameters = None self._tmp_path = os.path.join( BasicConfig.LOCAL_ROOT_DIR, self.expid, BasicConfig.LOCAL_TMP_DIR) self.write_start = False @@ -215,25 +226,47 @@ class Job(object): self.level = 0 self._export = "none" self._dependencies = [] - self.running = "once" + self.running = None self.start_time = None - self.ext_header_path = '' - self.ext_tailer_path = '' + self.ext_header_path = None + self.ext_tailer_path = None self.edge_info = dict() self.total_jobs = None self.max_waiting_jobs = None self.exclusive = "" self._retrials = 0 - # internal self.current_checkpoint_step = 0 self.max_checkpoint_step = 0 - self.reservation= "" + self.reservation = "" + self.delete_when_edgeless = False # hetjobs - self.het = dict() - self.het['HETSIZE'] = 0 + self.het = None + def _init_runtime_parameters(self): + # hetjobs + self.het = {'HETSIZE': 0} + self.parameters = dict() + self._tasks = '1' + self._nodes = "" + self.default_parameters = {'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', + 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'} + self._threads = '1' + self._processors = '1' + self._memory = '' + self._memory_per_task = '' + def _clean_runtime_parameters(self): + # hetjobs + self.het = None + self.parameters = None + self._tasks = None + self._nodes = None + self.default_parameters = None + self._threads = None + self._processors = None + self._memory = None + self._memory_per_task = None @property @autosubmit_parameter(name='tasktype') def section(self): @@ -272,7 +305,8 @@ class Job(object): @retrials.setter def retrials(self, value): - self._retrials = int(value) + if value is not None: + self._retrials = int(value) @property @autosubmit_parameter(name='checkpoint') @@ -496,11 +530,8 @@ class Job(object): self._splits = value def __getstate__(self): - odict = self.__dict__ - if '_platform' in odict: - odict = odict.copy() # copy the dict since we change it - del odict['_platform'] # remove filehandle entry - return odict + return {k: v for k, v in self.__dict__.items() if k not in ["_platform", "_children", "_parents", "submitter"]} + def read_header_tailer_script(self, script_path: str, as_conf: AutosubmitConfig, is_header: bool): """ @@ -512,13 +543,15 @@ class Job(object): :param as_conf: Autosubmit configuration file :param is_header: boolean indicating if it is header extended script """ - + if not script_path: + return '' found_hashbang = False script_name = script_path.rsplit("/")[-1] # pick the name of the script for a more verbose error - script = '' # the value might be None string if the key has been set, but with no value - if script_path == '' or script_path == "None": - return script + if not script_name: + return '' + script = '' + # adjusts the error message to the type of the script if is_header: @@ -623,7 +656,7 @@ class Job(object): :return HPCPlatform object for the job to use :rtype: HPCPlatform """ - if self.is_serial: + if self.is_serial and self._platform: return self._platform.serial_platform else: return self._platform @@ -753,7 +786,7 @@ class Job(object): if ':' in str(self.processors): return reduce(lambda x, y: int(x) + int(y), self.processors.split(':')) elif self.processors == "" or self.processors == "1": - if int(self.nodes) <= 1: + if not self.nodes or int(self.nodes) <= 1: return 1 else: return "" @@ -776,6 +809,17 @@ class Job(object): def processors(self, value): self._processors = value + @property + @autosubmit_parameter(name=['processors_per_node']) + def processors_per_node(self): + """Number of processors per node that the job can use.""" + return self._processors_per_node + + @processors_per_node.setter + def processors_per_node(self, value): + """Number of processors per node that the job can use.""" + self._processors_per_node = value + def inc_fail_count(self): """ Increments fail count @@ -799,6 +843,16 @@ class Job(object): self._parents.add(new_parent) new_parent.__add_child(self) + def add_children(self, children): + """ + Add children for the job. It also adds current job as a parent for all the new children + + :param children: job's children to add + :type children: list of Job objects + """ + for child in (child for child in children if child.name != self.name): + self.__add_child(child) + child._parents.add(self) def __add_child(self, new_child): """ Adds a new child to the job @@ -808,19 +862,19 @@ class Job(object): """ self.children.add(new_child) - def add_edge_info(self, parent, special_variables): + def add_edge_info(self, parent, special_conditions): """ Adds edge information to the job :param parent: parent job :type parent: Job - :param special_variables: special variables - :type special_variables: dict + :param special_conditions: special variables + :type special_conditions: dict """ - if special_variables["STATUS"] not in self.edge_info: - self.edge_info[special_variables["STATUS"]] = {} + if special_conditions["STATUS"] not in self.edge_info: + self.edge_info[special_conditions["STATUS"]] = {} - self.edge_info[special_variables["STATUS"]][parent.name] = (parent,special_variables.get("FROM_STEP", 0)) + self.edge_info[special_conditions["STATUS"]][parent.name] = (parent,special_conditions.get("FROM_STEP", 0)) def delete_parent(self, parent): """ @@ -1585,37 +1639,40 @@ class Job(object): # Ignore the heterogeneous parameters if the cores or nodes are no specefied as a list if self.het['HETSIZE'] == 1: self.het = dict() - if self.wallclock is None and job_platform.type not in ['ps', "local", "PS", "LOCAL"]: - self.wallclock = "01:59" - elif self.wallclock is None and job_platform.type in ['ps', 'local', "PS", "LOCAL"]: - self.wallclock = "00:00" + if not self.wallclock: + if job_platform.type.lower() not in ['ps', "local"]: + self.wallclock = "01:59" + elif job_platform.type.lower() in ['ps', 'local']: + self.wallclock = "00:00" # Increasing according to chunk self.wallclock = increase_wallclock_by_chunk( self.wallclock, self.wchunkinc, chunk) def update_platform_associated_parameters(self,as_conf, parameters, job_platform, chunk): - self.ec_queue = str(as_conf.jobs_data[self.section].get("EC_QUEUE", as_conf.platforms_data.get(job_platform.name,{}).get("EC_QUEUE",""))) - - self.executable = as_conf.jobs_data[self.section].get("EXECUTABLE", as_conf.platforms_data.get(job_platform.name,{}).get("EXECUTABLE","")) - self.total_jobs = as_conf.jobs_data[self.section].get("TOTALJOBS", job_platform.total_jobs) - self.max_waiting_jobs = as_conf.jobs_data[self.section].get("MAXWAITINGJOBS", job_platform.max_waiting_jobs) - self.processors = as_conf.jobs_data[self.section].get("PROCESSORS",as_conf.platforms_data.get(job_platform.name,{}).get("PROCESSORS","1")) - self.nodes = as_conf.jobs_data[self.section].get("NODES",as_conf.platforms_data.get(job_platform.name,{}).get("NODES","")) - self.exclusive = as_conf.jobs_data[self.section].get("EXCLUSIVE",as_conf.platforms_data.get(job_platform.name,{}).get("EXCLUSIVE",False)) - self.threads = as_conf.jobs_data[self.section].get("THREADS",as_conf.platforms_data.get(job_platform.name,{}).get("THREADS","1")) - self.tasks = as_conf.jobs_data[self.section].get("TASKS",as_conf.platforms_data.get(job_platform.name,{}).get("TASKS","1")) - self.reservation = as_conf.jobs_data[self.section].get("RESERVATION",as_conf.platforms_data.get(job_platform.name, {}).get("RESERVATION", "")) - self.hyperthreading = as_conf.jobs_data[self.section].get("HYPERTHREADING",as_conf.platforms_data.get(job_platform.name,{}).get("HYPERTHREADING","none")) - self.queue = self.queue - self.partition = self.partition - self.scratch_free_space = int(as_conf.jobs_data[self.section].get("SCRATCH_FREE_SPACE",as_conf.platforms_data.get(job_platform.name,{}).get("SCRATCH_FREE_SPACE",0))) - - self.memory = as_conf.jobs_data[self.section].get("MEMORY",as_conf.platforms_data.get(job_platform.name,{}).get("MEMORY","")) - self.memory_per_task = as_conf.jobs_data[self.section].get("MEMORY_PER_TASK",as_conf.platforms_data.get(job_platform.name,{}).get("MEMORY_PER_TASK","")) - self.wallclock = as_conf.jobs_data[self.section].get("WALLCLOCK", + job_data = as_conf.jobs_data[self.section] + platform_data = as_conf.platforms_data.get(job_platform.name,{}) + self.ec_queue = str(job_data.get("EC_QUEUE", platform_data.get("EC_QUEUE",""))) + self.executable = job_data.get("EXECUTABLE", platform_data.get("EXECUTABLE","")) + self.total_jobs = job_data.get("TOTALJOBS",job_data.get("TOTAL_JOBS", job_platform.total_jobs)) + self.max_waiting_jobs = job_data.get("MAXWAITINGJOBS",job_data.get("MAX_WAITING_JOBS", job_platform.max_waiting_jobs)) + self.processors = job_data.get("PROCESSORS",platform_data.get("PROCESSORS","1")) + self.processors_per_node = job_data.get("PROCESSORS_PER_NODE",as_conf.platforms_data.get(job_platform.name,{}).get("PROCESSORS_PER_NODE","1")) + self.nodes = job_data.get("NODES",platform_data.get("NODES","")) + self.exclusive = job_data.get("EXCLUSIVE",platform_data.get("EXCLUSIVE",False)) + self.threads = job_data.get("THREADS",platform_data.get("THREADS","1")) + self.tasks = job_data.get("TASKS",platform_data.get("TASKS","1")) + self.reservation = job_data.get("RESERVATION",as_conf.platforms_data.get(job_platform.name, {}).get("RESERVATION", "")) + self.hyperthreading = job_data.get("HYPERTHREADING",platform_data.get("HYPERTHREADING","none")) + self.queue = job_data.get("QUEUE",platform_data.get("QUEUE","")) + self.partition = job_data.get("PARTITION",platform_data.get("PARTITION","")) + self.scratch_free_space = int(job_data.get("SCRATCH_FREE_SPACE",platform_data.get("SCRATCH_FREE_SPACE",0))) + + self.memory = job_data.get("MEMORY",platform_data.get("MEMORY","")) + self.memory_per_task = job_data.get("MEMORY_PER_TASK",platform_data.get("MEMORY_PER_TASK","")) + self.wallclock = job_data.get("WALLCLOCK", as_conf.platforms_data.get(self.platform_name, {}).get( "MAX_WALLCLOCK", None)) - self.custom_directives = as_conf.jobs_data[self.section].get("CUSTOM_DIRECTIVES", "") + self.custom_directives = job_data.get("CUSTOM_DIRECTIVES", "") self.process_scheduler_parameters(as_conf,parameters,job_platform,chunk) if self.het.get('HETSIZE',1) > 1: @@ -1625,7 +1682,12 @@ class Job(object): if indx == 0: parameters[name.upper()] = component parameters[f'{name.upper()}_{indx}'] = component - + parameters['TOTALJOBS'] = self.total_jobs + parameters['MAXWAITINGJOBS'] = self.max_waiting_jobs + parameters['PROCESSORS_PER_NODE'] = self.processors_per_node + parameters['EXECUTABLE'] = self.executable + parameters['EXCLUSIVE'] = self.exclusive + parameters['EC_QUEUE'] = self.ec_queue parameters['NUMPROC'] = self.processors parameters['PROCESSORS'] = self.processors parameters['MEMORY'] = self.memory @@ -1650,6 +1712,8 @@ class Job(object): parameters['CURRENT_QUEUE'] = self.queue parameters['RESERVATION'] = self.reservation parameters['CURRENT_EC_QUEUE'] = self.ec_queue + parameters['PARTITION'] = self.partition + return parameters @@ -1677,8 +1741,46 @@ class Job(object): as_conf.get_extensible_wallclock(as_conf.experiment_data["WRAPPERS"].get(wrapper_section))) return parameters - def update_job_parameters(self,as_conf, parameters): + def update_dict_parameters(self,as_conf): + self.retrials = as_conf.jobs_data.get(self.section,{}).get("RETRIALS", as_conf.experiment_data.get("CONFIG",{}).get("RETRIALS", 0)) + self.splits = as_conf.jobs_data.get(self.section,{}).get("SPLITS", None) + self.delete_when_edgeless = as_conf.jobs_data.get(self.section,{}).get("DELETE_WHEN_EDGELESS", True) + self.dependencies = str(as_conf.jobs_data.get(self.section,{}).get("DEPENDENCIES","")) + self.running = as_conf.jobs_data.get(self.section,{}).get("RUNNING", "once") + self.platform_name = as_conf.jobs_data.get(self.section,{}).get("PLATFORM", as_conf.experiment_data.get("DEFAULT",{}).get("HPCARCH", None)) + self.file = as_conf.jobs_data.get(self.section,{}).get("FILE", None) + self.additional_files = as_conf.jobs_data.get(self.section,{}).get("ADDITIONAL_FILES", []) + + type_ = str(as_conf.jobs_data.get(self.section,{}).get("TYPE", "bash")).lower() + if type_ == "bash": + self.type = Type.BASH + elif type_ == "python" or type_ == "python3": + self.type = Type.PYTHON + elif type_ == "r": + self.type = Type.R + elif type_ == "python2": + self.type = Type.PYTHON2 + else: + self.type = Type.BASH + self.ext_header_path = as_conf.jobs_data.get(self.section,{}).get('EXTENDED_HEADER_PATH', None) + self.ext_tailer_path = as_conf.jobs_data.get(self.section,{}).get('EXTENDED_TAILER_PATH', None) + if self.platform_name: + self.platform_name = self.platform_name.upper() + + def update_check_variables(self,as_conf): + job_data = as_conf.jobs_data.get(self.section, {}) + job_platform_name = job_data.get("PLATFORM", as_conf.experiment_data.get("DEFAULT",{}).get("HPCARCH", None)) + job_platform = job_data.get("PLATFORMS",{}).get(job_platform_name, {}) + self.check = job_data.get("CHECK", False) + self.check_warnings = job_data.get("CHECK_WARNINGS", False) + self.total_jobs = job_data.get("TOTALJOBS",job_data.get("TOTALJOBS", job_platform.get("TOTALJOBS", job_platform.get("TOTAL_JOBS", -1)))) + self.max_waiting_jobs = job_data.get("MAXWAITINGJOBS",job_data.get("MAXWAITINGJOBS", job_platform.get("MAXWAITINGJOBS", job_platform.get("MAX_WAITING_JOBS", -1)))) + def update_job_parameters(self,as_conf, parameters): + self.splits = as_conf.jobs_data[self.section].get("SPLITS", None) + self.delete_when_edgeless = as_conf.jobs_data[self.section].get("DELETE_WHEN_EDGELESS", True) + self.check = as_conf.jobs_data[self.section].get("CHECK", False) + self.check_warnings = as_conf.jobs_data[self.section].get("CHECK_WARNINGS", False) if self.checkpoint: # To activate placeholder sustitution per in the template parameters["AS_CHECKPOINT"] = self.checkpoint parameters['JOBNAME'] = self.name @@ -1692,10 +1794,11 @@ class Job(object): parameters['SYNCHRONIZE'] = self.synchronize parameters['PACKED'] = self.packed parameters['CHUNK'] = 1 - if hasattr(self, 'RETRIALS'): - parameters['RETRIALS'] = self.retrials - if hasattr(self, 'delay_retrials'): - parameters['DELAY_RETRIALS'] = self.delay_retrials + parameters['RETRIALS'] = self.retrials + parameters['DELAY_RETRIALS'] = self.delay_retrials + parameters['DELETE_WHEN_EDGELESS'] = self.delete_when_edgeless + + if self.date is not None and len(str(self.date)) > 0: if self.chunk is None and len(str(self.chunk)) > 0: chunk = 1 @@ -1705,7 +1808,7 @@ class Job(object): parameters['CHUNK'] = chunk total_chunk = int(parameters.get('EXPERIMENT.NUMCHUNKS', 1)) chunk_length = int(parameters.get('EXPERIMENT.CHUNKSIZE', 1)) - chunk_unit = str(parameters.get('EXPERIMENT.CHUNKSIZEUNIT', "")).lower() + chunk_unit = str(parameters.get('EXPERIMENT.CHUNKSIZEUNIT', "day")).lower() cal = str(parameters.get('EXPERIMENT.CALENDAR', "")).lower() chunk_start = chunk_start_date( self.date, chunk, chunk_length, chunk_unit, cal) @@ -1757,8 +1860,9 @@ class Job(object): else: parameters['CHUNK_LAST'] = 'FALSE' parameters['NUMMEMBERS'] = len(as_conf.get_member_list()) - parameters['DEPENDENCIES'] = str(as_conf.jobs_data[self.section].get("DEPENDENCIES","")) - self.dependencies = parameters['DEPENDENCIES'] + self.dependencies = as_conf.jobs_data[self.section].get("DEPENDENCIES", "") + self.dependencies = str(self.dependencies) + parameters['JOB_DEPENDENCIES'] = self.dependencies parameters['EXPORT'] = self.export parameters['PROJECT_TYPE'] = as_conf.get_project_type() self.wchunkinc = as_conf.get_wchunkinc(self.section) @@ -1766,6 +1870,37 @@ class Job(object): parameters["CURRENT_"+key.upper()] = value return parameters + + + def update_job_variables_final_values(self,parameters): + """ Jobs variables final values based on parameters dict instead of as_conf + This function is called to handle %CURRENT_% placeholders as they are filled up dynamically for each job + """ + self.splits = parameters["SPLITS"] + self.delete_when_edgeless = parameters["DELETE_WHEN_EDGELESS"] + self.dependencies = parameters["JOB_DEPENDENCIES"] + self.ec_queue = parameters["EC_QUEUE"] + self.executable = parameters["EXECUTABLE"] + self.total_jobs = parameters["TOTALJOBS"] + self.max_waiting_jobs = parameters["MAXWAITINGJOBS"] + self.processors = parameters["PROCESSORS"] + self.processors_per_node = parameters["PROCESSORS_PER_NODE"] + self.nodes = parameters["NODES"] + self.exclusive = parameters["EXCLUSIVE"] + self.threads = parameters["THREADS"] + self.tasks = parameters["TASKS"] + self.reservation = parameters["RESERVATION"] + self.hyperthreading = parameters["HYPERTHREADING"] + self.queue = parameters["CURRENT_QUEUE"] + self.partition = parameters["PARTITION"] + self.scratch_free_space = parameters["SCRATCH_FREE_SPACE"] + self.memory = parameters["MEMORY"] + self.memory_per_task = parameters["MEMORY_PER_TASK"] + self.wallclock = parameters["WALLCLOCK"] + self.custom_directives = parameters["CUSTOM_DIRECTIVES"] + self.retrials = parameters["RETRIALS"] + self.reservation = parameters["RESERVATION"] + def update_parameters(self, as_conf, parameters, default_parameters={'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'}): @@ -1780,6 +1915,9 @@ class Job(object): :type parameters: dict """ as_conf.reload() + self._init_runtime_parameters() + # Parameters that affect to all the rest of parameters + self.update_dict_parameters(as_conf) parameters = parameters.copy() parameters.update(as_conf.parameters) parameters.update(default_parameters) @@ -1787,14 +1925,16 @@ class Job(object): parameters['ROOTDIR'] = os.path.join( BasicConfig.LOCAL_ROOT_DIR, self.expid) parameters['PROJDIR'] = as_conf.get_project_dir() + # Set parameters dictionary + # Set final value parameters = self.update_job_parameters(as_conf,parameters) parameters = self.update_platform_parameters(as_conf, parameters, self._platform) parameters = self.update_platform_associated_parameters(as_conf, parameters, self._platform, parameters['CHUNK']) parameters = self.update_wrapper_parameters(as_conf, parameters) parameters = as_conf.normalize_parameters_keys(parameters,default_parameters) - parameters = as_conf.substitute_dynamic_variables(parameters,80) parameters = as_conf.normalize_parameters_keys(parameters,default_parameters) + self.update_job_variables_final_values(parameters) # For some reason, there is return but the assignee is also necessary self.parameters = parameters # This return is only being used by the mock , to change the mock @@ -1819,7 +1959,7 @@ class Job(object): :return: script code :rtype: str """ - parameters = self.parameters + self.update_parameters(as_conf, self.parameters) try: if as_conf.get_project_type().lower() != "none" and len(as_conf.get_project_type()) > 0: template_file = open(os.path.join(as_conf.get_project_dir(), self.file), 'r') @@ -1934,20 +2074,21 @@ class Job(object): #enumerate and get value #TODO regresion test for additional_file, additional_template_content in zip(self.additional_files, additional_templates): - for key, value in parameters.items(): - final_sub = str(value) - if "\\" in final_sub: - final_sub = re.escape(final_sub) - # Check if key is in the additional template - if "%(?. -from autosubmit.job.job import Job + from bscearth.utils.date import date2str -from autosubmit.job.job_common import Status, Type -from log.log import Log, AutosubmitError, AutosubmitCritical -from collections.abc import Iterable + +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status +import datetime + +import re + + class DicJobs: """ - Class to create jobs from conf file and to find jobs by start date, member and chunk - - :param jobs_list: jobs list to use - :type jobs_list: Joblist + Class to create and build jobs from conf file and to find jobs by start date, member and chunk :param date_list: start dates :type date_list: list - :param member_list: member + :param member_list: members :type member_list: list - :param chunk_list: chunks + :param chunk_list chunks :type chunk_list: list - :param date_format: option to format dates + :param date_format: H/M/D (hour, month, day) :type date_format: str - :param default_retrials: default retrials for ech job + :param default_retrials: 0 by default :type default_retrials: int - :type default_retrials: config_common + :param as_conf: Comes from config parser, contains all experiment yml info + :type as_conf: as_conf """ - def __init__(self, jobs_list, date_list, member_list, chunk_list, date_format, default_retrials,jobs_data,experiment_data): + def __init__(self, date_list, member_list, chunk_list, date_format, default_retrials, as_conf): self._date_list = date_list - self._jobs_list = jobs_list self._member_list = member_list self._chunk_list = chunk_list - self._jobs_data = jobs_data self._date_format = date_format self.default_retrials = default_retrials self._dic = dict() - self.experiment_data = experiment_data + self.as_conf = as_conf + self.experiment_data = as_conf.experiment_data + self.recreate_jobs = False + self.changes = {} + self._job_list = {} + self.workflow_jobs = [] + + @property + def job_list(self): + return self._job_list + + @job_list.setter + def job_list(self, job_list): + self._job_list = {job.name: job for job in job_list} + + def compare_section(self, current_section): + """ + Compare the current section metadata with the last run one to see if it has changed + :param current_section: current section + :type current_section: str + :rtype: bool + """ + self.changes[current_section] = self.as_conf.detailed_deep_diff( + self.as_conf.experiment_data["JOBS"].get(current_section, {}), + self.as_conf.last_experiment_data.get("JOBS", {}).get(current_section, {})) + # Only dependencies is relevant at this step, the rest is lookup by job name and if it inside the stored list + if "DEPENDENCIES" not in self.changes[current_section]: + del self.changes[current_section] + + def compare_backbone_sections(self): + """ + Compare the backbone sections metadata with the last run one to see if it has changed + """ + self.compare_experiment_section() + self.compare_jobs_section() + self.compare_config() + self.compare_default() + + def compare_experiment_section(self): + """ + Compare the experiment structure metadata with the last run one to see if it has changed + :return: + """ + self.changes["EXPERIMENT"] = self.as_conf.detailed_deep_diff(self.experiment_data.get("EXPERIMENT", {}), + self.as_conf.last_experiment_data.get("EXPERIMENT", + {})) + if not self.changes["EXPERIMENT"]: + del self.changes["EXPERIMENT"] + + def compare_default(self): + """ + Compare the default structure metadata with the last run one to see if it has changed + :return: + """ + self.changes["DEFAULT"] = self.as_conf.detailed_deep_diff(self.experiment_data.get("DEFAULT", {}), + self.as_conf.last_experiment_data.get("DEFAULT", {})) + if "HPCARCH" not in self.changes["DEFAULT"]: + del self.changes["DEFAULT"] - def read_section(self, section, priority, default_job_type, jobs_data=dict()): + def compare_config(self): + """ + Compare the config structure metadata with the last run one to see if it has changed + :return: + """ + self.changes["CONFIG"] = self.as_conf.detailed_deep_diff(self.experiment_data.get("CONFIG", {}), + self.as_conf.last_experiment_data.get("CONFIG", {})) + if "VERSION" not in self.changes["CONFIG"]: + del self.changes["CONFIG"] + + def compare_jobs_section(self): + """ + Compare the jobs structure metadata with the last run one to see if it has changed + :return: + """ + self.changes["JOBS"] = self.as_conf.detailed_deep_diff(self.experiment_data.get("JOBS", {}), + self.as_conf.last_experiment_data.get("JOBS", {})) + if not self.changes["JOBS"]: + del self.changes["JOBS"] + + def read_section(self, section, priority, default_job_type): """ Read a section from jobs conf and creates all jobs for it :param default_job_type: default type for jobs :type default_job_type: str - :param jobs_data: dictionary containing the plain data from jobs - :type jobs_data: dict :param section: section to read, and it's info :type section: tuple(str,dict) :param priority: priority for the jobs :type priority: int """ + self.compare_section(section) parameters = self.experiment_data["JOBS"] - splits = int(parameters[section].get("SPLITS", -1)) - running = str(parameters[section].get('RUNNING',"once")).lower() + running = str(parameters[section].get('RUNNING', "once")).lower() frequency = int(parameters[section].get("FREQUENCY", 1)) if running == 'once': - self._create_jobs_once(section, priority, default_job_type, jobs_data,splits) + self._create_jobs_once(section, priority, default_job_type, splits) elif running == 'date': - self._create_jobs_startdate(section, priority, frequency, default_job_type, jobs_data,splits) + self._create_jobs_startdate(section, priority, frequency, default_job_type, splits) elif running == 'member': - self._create_jobs_member(section, priority, frequency, default_job_type, jobs_data,splits) + self._create_jobs_member(section, priority, frequency, default_job_type, splits) elif running == 'chunk': synchronize = str(parameters[section].get("SYNCHRONIZE", "")) delay = int(parameters[section].get("DELAY", -1)) - self._create_jobs_chunk(section, priority, frequency, default_job_type, synchronize, delay, splits, jobs_data) - - + self._create_jobs_chunk(section, priority, frequency, default_job_type, synchronize, delay, splits) - pass - - def _create_jobs_startdate(self, section, priority, frequency, default_job_type, jobs_data=dict(), splits=-1): + def _create_jobs_startdate(self, section, priority, frequency, default_job_type, splits=-1): """ Create jobs to be run once per start date @@ -99,23 +171,15 @@ class DicJobs: :type frequency: int """ self._dic[section] = dict() - tmp_dic = dict() - tmp_dic[section] = dict() count = 0 for date in self._date_list: count += 1 if count % frequency == 0 or count == len(self._date_list): - if splits <= 0: - self._dic[section][date] = self.build_job(section, priority, date, None, None, default_job_type, - jobs_data) - self._jobs_list.graph.add_node(self._dic[section][date].name) - else: - tmp_dic[section][date] = [] - self._create_jobs_split(splits, section, date, None, None, priority, - default_job_type, jobs_data, tmp_dic[section][date]) - self._dic[section][date] = tmp_dic[section][date] + self._dic[section][date] = [] + self._create_jobs_split(splits, section, date, None, None, priority, default_job_type, + self._dic[section][date]) - def _create_jobs_member(self, section, priority, frequency, default_job_type, jobs_data=dict(),splits=-1): + def _create_jobs_member(self, section, priority, frequency, default_job_type, splits=-1): """ Create jobs to be run once per member @@ -131,23 +195,17 @@ class DicJobs: """ self._dic[section] = dict() - tmp_dic = dict() - tmp_dic[section] = dict() for date in self._date_list: self._dic[section][date] = dict() count = 0 for member in self._member_list: count += 1 if count % frequency == 0 or count == len(self._member_list): - if splits <= 0: - self._dic[section][date][member] = self.build_job(section, priority, date, member, None,default_job_type, jobs_data,splits) - self._jobs_list.graph.add_node(self._dic[section][date][member].name) - else: - self._create_jobs_split(splits, section, date, member, None, priority, - default_job_type, jobs_data, tmp_dic[section][date][member]) - self._dic[section][date][member] = tmp_dic[section][date][member] + self._dic[section][date][member] = [] + self._create_jobs_split(splits, section, date, member, None, priority, default_job_type, + self._dic[section][date][member]) - def _create_jobs_once(self, section, priority, default_job_type, jobs_data=dict(),splits=0): + def _create_jobs_once(self, section, priority, default_job_type, splits=0): """ Create jobs to be run once @@ -156,25 +214,10 @@ class DicJobs: :param priority: priority for the jobs :type priority: int """ + self._dic[section] = [] + self._create_jobs_split(splits, section, None, None, None, priority, default_job_type, self._dic[section]) - - if splits <= 0: - job = self.build_job(section, priority, None, None, None, default_job_type, jobs_data, -1) - self._dic[section] = job - self._jobs_list.graph.add_node(job.name) - else: - self._dic[section] = [] - total_jobs = 1 - while total_jobs <= splits: - job = self.build_job(section, priority, None, None, None, default_job_type, jobs_data, total_jobs) - self._dic[section].append(job) - self._jobs_list.graph.add_node(job.name) - total_jobs += 1 - pass - - #self._dic[section] = self.build_job(section, priority, None, None, None, default_job_type, jobs_data) - #self._jobs_list.graph.add_node(self._dic[section].name) - def _create_jobs_chunk(self, section, priority, frequency, default_job_type, synchronize=None, delay=0, splits=0, jobs_data=dict()): + def _create_jobs_chunk(self, section, priority, frequency, default_job_type, synchronize=None, delay=0, splits=0): """ Create jobs to be run once per chunk @@ -189,6 +232,7 @@ class DicJobs: :param delay: if this parameter is set, the job is only created for the chunks greater than the delay :type delay: int """ + self._dic[section] = dict() # Temporally creation for unified jobs in case of synchronize tmp_dic = dict() if synchronize is not None and len(str(synchronize)) > 0: @@ -197,35 +241,23 @@ class DicJobs: count += 1 if delay == -1 or delay < chunk: if count % frequency == 0 or count == len(self._chunk_list): - if splits > 1: - if synchronize == 'date': - tmp_dic[chunk] = [] - self._create_jobs_split(splits, section, None, None, chunk, priority, - default_job_type, jobs_data, tmp_dic[chunk]) - elif synchronize == 'member': - tmp_dic[chunk] = dict() - for date in self._date_list: - tmp_dic[chunk][date] = [] - self._create_jobs_split(splits, section, date, None, chunk, priority, - default_job_type, jobs_data, tmp_dic[chunk][date]) - - else: - if synchronize == 'date': - tmp_dic[chunk] = self.build_job(section, priority, None, None, - chunk, default_job_type, jobs_data) - elif synchronize == 'member': - tmp_dic[chunk] = dict() - for date in self._date_list: - tmp_dic[chunk][date] = self.build_job(section, priority, date, None, - chunk, default_job_type, jobs_data) + if synchronize == 'date': + tmp_dic[chunk] = [] + self._create_jobs_split(splits, section, None, None, chunk, priority, + default_job_type, tmp_dic[chunk]) + elif synchronize == 'member': + tmp_dic[chunk] = dict() + for date in self._date_list: + tmp_dic[chunk][date] = [] + self._create_jobs_split(splits, section, date, None, chunk, priority, + default_job_type, tmp_dic[chunk][date]) # Real dic jobs assignment/creation - self._dic[section] = dict() for date in self._date_list: self._dic[section][date] = dict() - for member in self._member_list: + for member in (member for member in self._member_list): self._dic[section][date][member] = dict() count = 0 - for chunk in self._chunk_list: + for chunk in (chunk for chunk in self._chunk_list): count += 1 if delay == -1 or delay < chunk: if count % frequency == 0 or count == len(self._chunk_list): @@ -235,25 +267,255 @@ class DicJobs: elif synchronize == 'member': if chunk in tmp_dic: self._dic[section][date][member][chunk] = tmp_dic[chunk][date] - - if splits > 1 and (synchronize is None or not synchronize): + else: self._dic[section][date][member][chunk] = [] - self._create_jobs_split(splits, section, date, member, chunk, priority, default_job_type, jobs_data, self._dic[section][date][member][chunk]) - pass - elif synchronize is None or not synchronize: - self._dic[section][date][member][chunk] = self.build_job(section, priority, date, member, - chunk, default_job_type, jobs_data) - self._jobs_list.graph.add_node(self._dic[section][date][member][chunk].name) - - def _create_jobs_split(self, splits, section, date, member, chunk, priority, default_job_type, jobs_data, dict_): - total_jobs = 1 - while total_jobs <= splits: - job = self.build_job(section, priority, date, member, chunk, default_job_type, jobs_data, total_jobs) - dict_.append(job) - self._jobs_list.graph.add_node(job.name) - total_jobs += 1 - - def get_jobs(self, section, date=None, member=None, chunk=None): + self._create_jobs_split(splits, section, date, member, chunk, priority, + default_job_type, + self._dic[section][date][member][chunk]) + + def _create_jobs_split(self, splits, section, date, member, chunk, priority, default_job_type, section_data): + splits_list = [-1] if splits <= 0 else range(1, splits + 1) + for split in splits_list: + self.build_job(section, priority, date, member, chunk, default_job_type, section_data, split) + + def update_jobs_filtered(self, current_jobs, next_level_jobs): + if type(next_level_jobs) is dict: + for key in next_level_jobs.keys(): + if key not in current_jobs: + current_jobs[key] = next_level_jobs[key] + else: + current_jobs[key] = self.update_jobs_filtered(current_jobs[key], next_level_jobs[key]) + elif type(next_level_jobs) is list: + current_jobs.extend(next_level_jobs) + else: + current_jobs.append(next_level_jobs) + return current_jobs + + def get_jobs_filtered(self, section, job, filters_to, natural_date, natural_member, natural_chunk, + filters_to_of_parent): + # datetime.strptime("20020201", "%Y%m%d") + jobs = self._dic.get(section, {}) + final_jobs_list = [] + # values replace original dict + jobs_aux = {} + if len(jobs) > 0: + if type(jobs) is list: + final_jobs_list.extend(jobs) + jobs = {} + else: + if filters_to.get('DATES_TO', None): + if "none" in filters_to['DATES_TO'].lower(): + jobs_aux = {} + elif "all" in filters_to['DATES_TO'].lower(): + for date in jobs.keys(): + if jobs.get(date, None): + if type(jobs.get(date, None)) == list: + for aux_job in jobs[date]: + final_jobs_list.append(aux_job) + elif type(jobs.get(date, None)) == Job: + final_jobs_list.append(jobs[date]) + elif type(jobs.get(date, None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[date]) + else: + for date in filters_to.get('DATES_TO', "").split(","): + if jobs.get(datetime.datetime.strptime(date, "%Y%m%d"), None): + if type(jobs.get(datetime.datetime.strptime(date, "%Y%m%d"), None)) == list: + for aux_job in jobs[datetime.datetime.strptime(date, "%Y%m%d")]: + final_jobs_list.append(aux_job) + elif type(jobs.get(datetime.datetime.strptime(date, "%Y%m%d"), None)) == Job: + final_jobs_list.append(jobs[datetime.datetime.strptime(date, "%Y%m%d")]) + elif type(jobs.get(datetime.datetime.strptime(date, "%Y%m%d"), None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[ + datetime.datetime.strptime(date, "%Y%m%d")]) + else: + if job.running == "once": + for key in jobs.keys(): + if type(jobs.get(key, None)) == list: # TODO + for aux_job in jobs[key]: + final_jobs_list.append(aux_job) + elif type(jobs.get(key, None)) == Job: # TODO + final_jobs_list.append(jobs[key]) + elif type(jobs.get(key, None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[key]) + elif jobs.get(job.date, None): + if type(jobs.get(natural_date, None)) == list: # TODO + for aux_job in jobs[natural_date]: + final_jobs_list.append(aux_job) + elif type(jobs.get(natural_date, None)) == Job: # TODO + final_jobs_list.append(jobs[natural_date]) + elif type(jobs.get(natural_date, None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[natural_date]) + else: + jobs_aux = {} + jobs = jobs_aux + if len(jobs) > 0: + if type(jobs) == list: # TODO check the other todo, maybe this is not neccesary, https://earth.bsc.es/gitlab/es/autosubmit/-/merge_requests/387#note_243751 + final_jobs_list.extend(jobs) + jobs = {} + else: + # pass keys to uppercase to normalize the member name as it can be whatever the user wants + jobs = {k.upper(): v for k, v in jobs.items()} + jobs_aux = {} + if filters_to.get('MEMBERS_TO', None): + if "none" in filters_to['MEMBERS_TO'].lower(): + jobs_aux = {} + elif "all" in filters_to['MEMBERS_TO'].lower(): + for member in jobs.keys(): + if jobs.get(member.upper(), None): + if type(jobs.get(member.upper(), None)) == list: + for aux_job in jobs[member.upper()]: + final_jobs_list.append(aux_job) + elif type(jobs.get(member.upper(), None)) == Job: + final_jobs_list.append(jobs[member.upper()]) + elif type(jobs.get(member.upper(), None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[member.upper()]) + + else: + for member in filters_to.get('MEMBERS_TO', "").split(","): + if jobs.get(member.upper(), None): + if type(jobs.get(member.upper(), None)) == list: + for aux_job in jobs[member.upper()]: + final_jobs_list.append(aux_job) + elif type(jobs.get(member.upper(), None)) == Job: + final_jobs_list.append(jobs[member.upper()]) + elif type(jobs.get(member.upper(), None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[member.upper()]) + else: + if job.running == "once" or not job.member: + for key in jobs.keys(): + if type(jobs.get(key, None)) == list: + for aux_job in jobs[key.upper()]: + final_jobs_list.append(aux_job) + elif type(jobs.get(key.upper(), None)) == Job: + final_jobs_list.append(jobs[key]) + elif type(jobs.get(key.upper(), None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[key.upper()]) + + elif jobs.get(job.member.upper(), None): + if type(jobs.get(natural_member.upper(), None)) == list: + for aux_job in jobs[natural_member.upper()]: + final_jobs_list.append(aux_job) + elif type(jobs.get(natural_member.upper(), None)) == Job: + final_jobs_list.append(jobs[natural_member.upper()]) + elif type(jobs.get(natural_member.upper(), None)) == dict: + jobs_aux = self.update_jobs_filtered(jobs_aux, jobs[natural_member.upper()]) + else: + jobs_aux = {} + jobs = jobs_aux + if len(jobs) > 0: + if type(jobs) == list: + final_jobs_list.extend(jobs) + else: + if filters_to.get('CHUNKS_TO', None): + if "none" in filters_to['CHUNKS_TO'].lower(): + pass + elif "all" in filters_to['CHUNKS_TO'].lower(): + for chunk in jobs.keys(): + if type(jobs.get(chunk, None)) == list: + for aux_job in jobs[chunk]: + final_jobs_list.append(aux_job) + elif type(jobs.get(chunk, None)) == Job: + final_jobs_list.append(jobs[chunk]) + else: + for chunk in filters_to.get('CHUNKS_TO', "").split(","): + chunk = int(chunk) + if type(jobs.get(chunk, None)) == list: + for aux_job in jobs[chunk]: + final_jobs_list.append(aux_job) + elif type(jobs.get(chunk, None)) == Job: + final_jobs_list.append(jobs[chunk]) + else: + if job.running == "once" or not job.chunk: + for chunk in jobs.keys(): + if type(jobs.get(chunk, None)) == list: + final_jobs_list += [aux_job for aux_job in jobs[chunk]] + elif type(jobs.get(chunk, None)) == Job: + final_jobs_list.append(jobs[chunk]) + elif jobs.get(job.chunk, None): + if type(jobs.get(natural_chunk, None)) == list: + final_jobs_list += [aux_job for aux_job in jobs[natural_chunk]] + elif type(jobs.get(natural_chunk, None)) == Job: + final_jobs_list.append(jobs[natural_chunk]) + + if len(final_jobs_list) > 0: + split_filter = filters_to.get("SPLITS_TO", None) + if split_filter: + split_filter = split_filter.split(",") + one_to_one_splits = [split for split in split_filter if "*" in split] + one_to_one_splits = ",".join(one_to_one_splits).lower() + normal_splits = [split for split in split_filter if "*" not in split] + normal_splits = ",".join(normal_splits).lower() + skip_one_to_one = False + if "none" in normal_splits: + final_jobs_list_normal = [f_job for f_job in final_jobs_list if ( + f_job.split is None or f_job.split == -1 or f_job.split == 0) and f_job.name != job.name] + skip_one_to_one = True + elif "all" in normal_splits: + final_jobs_list_normal = final_jobs_list + skip_one_to_one = True + elif "previous" in normal_splits: + final_jobs_list_normal = [f_job for f_job in final_jobs_list if ( + f_job.split is None or job.split is None or f_job.split == job.split - 1) and f_job.name != job.name] + skip_one_to_one = True + else: + final_jobs_list_normal = [f_job for f_job in final_jobs_list if ( + f_job.split is None or f_job.split == -1 or f_job.split == 0 or str(f_job.split) in + normal_splits.split(',')) and f_job.name != job.name] + final_jobs_list_special = [] + if "*" in one_to_one_splits and not skip_one_to_one: + easier_to_filter = "," + one_to_one_splits + "," + matches = re.findall(rf"\\[0-9]+", easier_to_filter) + if len(matches) > 0: # get *\\ + + split_slice = int(matches[0].split("\\")[1]) + if int(job.splits) <= int(final_jobs_list[0].splits): # get N-1 ( child - parent ) + # (parent) -> (child) + # 1 -> 1,2 + # 2 -> 3,4 + # 3 -> 5 # but 5 is not enough to make another group, so it must be included in the previous one ( did in part two ) + matches = re.findall(rf",{(job.split - 1) * split_slice + 1}\*\\?[0-9]*,", easier_to_filter) + else: # get 1-N ( child - parent ) + # (parent) -> (child) + # 1,2 -> 1 + # 3,4 -> 2 + # 5 -> 3 # but 5 is not enough to make another group, so it must be included in the previous one + group = (job.split - 1) // split_slice + 1 + matches = re.findall(rf",{group}\*\\?[0-9]*,", easier_to_filter) + if len(matches) == 0: + matches = re.findall(rf",{group - 1}\*\\?[0-9]*,", easier_to_filter) + else: # get * (1-1) + split_slice = 1 + # get current index 1-1 + matches = re.findall(rf",{job.split}\*\\?[0-9]*,", easier_to_filter) + if len(matches) > 0: + if int(job.splits) <= int(final_jobs_list[0].splits): # get 1-1,N-1 (part 1) + my_complete_slice = matches[0].strip(",").split("*") + split_index = int(my_complete_slice[0]) - 1 + end = split_index + split_slice + if split_slice > 1: + if len(final_jobs_list) < end + split_slice: + end = len(final_jobs_list) + final_jobs_list_special = final_jobs_list[split_index:end] + if "previous" in filters_to_of_parent.get("SPLITS_TO", ""): + final_jobs_list_special = [final_jobs_list_special[-1]] + else: # get 1-N (part 2) + my_complete_slice = matches[0].strip(",").split("*") + split_index = int(my_complete_slice[0]) - 1 + final_jobs_list_special = final_jobs_list[split_index] + if "previous" in filters_to_of_parent.get("SPLITS_TO", ""): + final_jobs_list_special = [final_jobs_list_special[-1]] + else: + final_jobs_list_special = [] + if type(final_jobs_list_special) is not list: + final_jobs_list_special = [final_jobs_list_special] + if type(final_jobs_list_normal) is not list: + final_jobs_list_normal = [final_jobs_list_normal] + final_jobs_list = list(set(final_jobs_list_normal + final_jobs_list_special)) + if type(final_jobs_list) is not list: + return [final_jobs_list] + return list(set(final_jobs_list)) + + def get_jobs(self, section, date=None, member=None, chunk=None, sort_string=False): """ Return all the jobs matching section, date, member and chunk provided. If any parameter is none, returns all the jobs without checking that parameter value. If a job has one parameter to None, is returned if all the @@ -276,7 +538,7 @@ class DicJobs: return jobs dic = self._dic[section] - #once jobs + # once jobs if type(dic) is list: jobs = dic elif type(dic) is not dict: @@ -293,6 +555,16 @@ class DicJobs: jobs = jobs_flattened except TypeError as e: pass + if sort_string: + # I want to have first chunks then member then date to easily filter later on + if len(jobs) > 0: + if jobs[0].chunk is not None: + jobs = sorted(jobs, key=lambda x: x.chunk) + elif jobs[0].member is not None: + jobs = sorted(jobs, key=lambda x: x.member) + elif jobs[0].date is not None: + jobs = sorted(jobs, key=lambda x: x.date) + return jobs def _get_date(self, jobs, dic, date, member, chunk): @@ -330,111 +602,33 @@ class DicJobs: jobs.append(dic[c]) return jobs - def build_job(self, section, priority, date, member, chunk, default_job_type, jobs_data=dict(), split=-1): - parameters = self.experiment_data["JOBS"] - name = self._jobs_list.expid - if date is not None and len(str(date)) > 0: + def build_job(self, section, priority, date, member, chunk, default_job_type, section_data, split=-1): + name = self.experiment_data.get("DEFAULT", {}).get("EXPID", "") + if date: name += "_" + date2str(date, self._date_format) - if member is not None and len(str(member)) > 0: + if member: name += "_" + member - if chunk is not None and len(str(chunk)) > 0: + if chunk: name += "_{0}".format(chunk) - if split > -1: + if split > 0: name += "_{0}".format(split) name += "_" + section - if name in jobs_data: - job = Job(name, jobs_data[name][1], jobs_data[name][2], priority) - job.local_logs = (jobs_data[name][8], jobs_data[name][9]) - job.remote_logs = (jobs_data[name][10], jobs_data[name][11]) - - else: + if not self._job_list.get(name, None): job = Job(name, 0, Status.WAITING, priority) - - - job.section = section - job.date = date - job.member = member - job.chunk = chunk - job.splits = self.experiment_data["JOBS"].get(job.section,{}).get("SPLITS", None) - job.date_format = self._date_format - job.delete_when_edgeless = str(parameters[section].get("DELETE_WHEN_EDGELESS", "true")).lower() - - if split > -1: + job.type = default_job_type + job.section = section + job.date = date + job.date_format = self._date_format + job.member = member + job.chunk = chunk job.split = split - - job.frequency = int(parameters[section].get( "FREQUENCY", 1)) - job.delay = int(parameters[section].get( "DELAY", -1)) - job.wait = str(parameters[section].get( "WAIT", True)).lower() - job.rerun_only = str(parameters[section].get( "RERUN_ONLY", False)).lower() - job_type = str(parameters[section].get( "TYPE", default_job_type)).lower() - - job.dependencies = parameters[section].get( "DEPENDENCIES", "") - if job.dependencies and type(job.dependencies) is not dict: - job.dependencies = str(job.dependencies).split() - if job_type == 'bash': - job.type = Type.BASH - elif job_type == 'python' or job_type == 'python3': - job.type = Type.PYTHON3 - elif job_type == 'python2': - job.type = Type.PYTHON2 - elif job_type == 'r': - job.type = Type.R - hpcarch = self.experiment_data.get("DEFAULT",{}) - hpcarch = hpcarch.get("HPCARCH","") - job.platform_name = str(parameters[section].get("PLATFORM", hpcarch)).upper() - if self.experiment_data["PLATFORMS"].get(job.platform_name, "") == "" and job.platform_name.upper() != "LOCAL": - raise AutosubmitCritical("Platform does not exists, check the value of %JOBS.{0}.PLATFORM% = {1} parameter".format(job.section,job.platform_name),7000,"List of platforms: {0} ".format(self.experiment_data["PLATFORMS"].keys()) ) - job.file = str(parameters[section].get( "FILE", "")) - job.additional_files = parameters[section].get( "ADDITIONAL_FILES", []) - - job.executable = str(parameters[section].get("EXECUTABLE", self.experiment_data["PLATFORMS"].get(job.platform_name,{}).get("EXECUTABLE",""))) - job.queue = str(parameters[section].get( "QUEUE", "")) - - job.ec_queue = str(parameters[section].get("EC_QUEUE", "")) - if job.ec_queue == "" and job.platform_name != "LOCAL": - job.ec_queue = str(self.experiment_data["PLATFORMS"][job.platform_name].get("EC_QUEUE","hpc")) - - job.partition = str(parameters[section].get( "PARTITION", "")) - job.check = str(parameters[section].get( "CHECK", "true")).lower() - job.export = str(parameters[section].get( "EXPORT", "")) - job.processors = str(parameters[section].get( "PROCESSORS", "")) - job.threads = str(parameters[section].get( "THREADS", "")) - job.tasks = str(parameters[section].get( "TASKS", "")) - job.memory = str(parameters[section].get("MEMORY", "")) - job.memory_per_task = str(parameters[section].get("MEMORY_PER_TASK", "")) - remote_max_wallclock = self.experiment_data["PLATFORMS"].get(job.platform_name,{}) - remote_max_wallclock = remote_max_wallclock.get("MAX_WALLCLOCK",None) - job.wallclock = parameters[section].get("WALLCLOCK", remote_max_wallclock) - for wrapper_section in self.experiment_data.get("WRAPPERS",{}).values(): - if job.section in wrapper_section.get("JOBS_IN_WRAPPER",""): - job.retrials = int(wrapper_section.get("RETRIALS", wrapper_section.get("INNER_RETRIALS",parameters[section].get('RETRIALS',self.experiment_data["CONFIG"].get("RETRIALS", 0))))) - break + job.update_dict_parameters(self.as_conf) + section_data.append(job) + self.changes["NEWJOBS"] = True else: - job.retrials = int(parameters[section].get('RETRIALS', self.experiment_data["CONFIG"].get("RETRIALS", 0))) - job.delay_retrials = int(parameters[section].get( 'DELAY_RETRY_TIME', "-1")) - if job.wallclock is None and job.platform_name.upper() != "LOCAL": - job.wallclock = "01:59" - elif job.wallclock is None and job.platform_name.upper() != "LOCAL": - job.wallclock = "00:00" - elif job.wallclock is None: - job.wallclock = "00:00" - if job.retrials == -1: - job.retrials = None - notify_on = parameters[section].get("NOTIFY_ON",None) - if type(notify_on) == str: - job.notify_on = [x.upper() for x in notify_on.split(' ')] - else: - job.notify_on = "" - job.synchronize = str(parameters[section].get( "SYNCHRONIZE", "")) - job.check_warnings = str(parameters[section].get("SHOW_CHECK_WARNINGS", False)).lower() - job.running = str(parameters[section].get( 'RUNNING', 'once')) - job.x11 = str(parameters[section].get( 'X11', False )).lower() - job.skippable = str(parameters[section].get( "SKIPPABLE", False)).lower() - # store from within the relative path to the project - job.ext_header_path = str(parameters[section].get('EXTENDED_HEADER_PATH', '')) - job.ext_tailer_path = str(parameters[section].get('EXTENDED_TAILER_PATH', '')) - self._jobs_list.get_job_list().append(job) - - return job - - + self._job_list[name].status = Status.WAITING if self._job_list[name].status in [Status.DELAYED, + Status.PREPARED, + Status.READY] else \ + self._job_list[name].status + section_data.append(self._job_list[name]) + self.workflow_jobs.append(name) diff --git a/autosubmit/job/job_grouping.py b/autosubmit/job/job_grouping.py index bcddaf038371b1708fee4d8eb198c77e0855a136..63a064719876118d1df6fce2611c7ecee0645a8a 100644 --- a/autosubmit/job/job_grouping.py +++ b/autosubmit/job/job_grouping.py @@ -169,7 +169,7 @@ class JobGrouping(object): groups = [] if not self._check_synchronized_job(job, groups): if self.group_by == 'split': - if job.split is not None and len(str(job.split)) > 0: + if job.split is not None and job.split > 0: idx = job.name.rfind("_") groups.append(job.name[:idx - 1] + job.name[idx + 1:]) elif self.group_by == 'chunk': diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index edf58fa094f5676659d21dded6efd3bc7684a6b9..e3a9c9708e4305256bc25cff4af5d3bd259b4c23 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -17,18 +17,21 @@ # along with Autosubmit. If not, see . import copy import datetime -import math import os import pickle import re import traceback -from bscearth.utils.date import date2str, parse_date -from networkx import DiGraph +from contextlib import suppress from shutil import move from threading import Thread -from time import localtime, strftime, mktime from typing import List, Dict +import math +import networkx as nx +from bscearth.utils.date import date2str, parse_date +from networkx import DiGraph +from time import localtime, strftime, mktime + import autosubmit.database.db_structure as DbStructure from autosubmit.helpers.data_transfer import JobRow from autosubmit.job.job import Job @@ -82,8 +85,6 @@ class JobList(object): self._chunk_list = [] self._dic_jobs = dict() self._persistence = job_list_persistence - self._graph = DiGraph() - self.packages_dict = dict() self._ordered_jobs_by_date_member = dict() @@ -93,6 +94,9 @@ class JobList(object): self._run_members = None self.jobs_to_run_first = list() self.rerun_job_list = list() + self.graph = DiGraph() + self.depends_on_previous_chunk = dict() + self.depends_on_previous_split = dict() @property def expid(self): @@ -104,24 +108,10 @@ class JobList(object): """ return self._expid - @property - def graph(self): - """ - Returns the graph - - :return: graph - :rtype: networkx graph - """ - return self._graph - @property def jobs_data(self): return self.experiment_data["JOBS"] - @graph.setter - def graph(self, value): - self._graph = value - @property def run_members(self): return self._run_members @@ -134,8 +124,7 @@ class JobList(object): found_member = False processed_job_list = [] for job in self._job_list: # We are assuming that the jobs are sorted in topological order (which is the default) - if ( - job.member is None and not found_member) or job.member in self._run_members or job.status not in [ + if (job.member is None and not found_member) or job.member in self._run_members or job.status not in [ Status.WAITING, Status.READY]: processed_job_list.append(job) if job.member is not None and len(str(job.member)) > 0: @@ -146,13 +135,10 @@ class JobList(object): # job.parents) == 0 or len(set(old_job_list_names).intersection(set([jobp.name for jobp in job.parents]))) == len(job.parents)] def create_dictionary(self, date_list, member_list, num_chunks, chunk_ini, date_format, default_retrials, - wrapper_jobs): + wrapper_jobs, as_conf): chunk_list = list(range(chunk_ini, num_chunks + 1)) - jobs_parser = self._get_jobs_parser() - dic_jobs = DicJobs(self, date_list, member_list, - chunk_list, date_format, default_retrials, jobs_data={}, - experiment_data=self.experiment_data) + dic_jobs = DicJobs(date_list, member_list, chunk_list, date_format, default_retrials, as_conf) self._dic_jobs = dic_jobs for wrapper_section in wrapper_jobs: if str(wrapper_jobs[wrapper_section]).lower() != 'none': @@ -163,100 +149,118 @@ class JobList(object): pass def _delete_edgeless_jobs(self): - jobs_to_delete = [] # indices to delete - for i, job in enumerate(self._job_list): - if job.dependencies is not None: - if (( - len(job.dependencies) > 0 and not job.has_parents()) and not job.has_children()) and job.delete_when_edgeless in [ - "true", True, 1]: - jobs_to_delete.append(job) - # delete jobs by indices - for i in jobs_to_delete: - self._job_list.remove(i) - - def generate(self, date_list, member_list, num_chunks, chunk_ini, parameters, date_format, default_retrials, - default_job_type, wrapper_type=None, wrapper_jobs=dict(), new=True, notransitive=False, - update_structure=False, run_only_members=[], show_log=True, jobs_data={}, as_conf=""): - """ - Creates all jobs needed for the current workflow - - :param as_conf: - :param jobs_data: - :param show_log: - :param run_only_members: - :param update_structure: - :param notransitive: - :param default_job_type: default type for jobs - :type default_job_type: str - :param date_list: start dates + for job in self._job_list[:]: + if job.dependencies is not None and job.dependencies not in ["{}", "[]"]: + if (len(job.dependencies) > 0 and not job.has_parents() and not job.has_children()) and str( + job.delete_when_edgeless).casefold() == "true".casefold(): + self._job_list.remove(job) + self.graph.remove_node(job.name) + + def generate(self, as_conf, date_list, member_list, num_chunks, chunk_ini, parameters, date_format, + default_retrials, + default_job_type, wrapper_jobs=dict(), new=True, run_only_members=[], show_log=True, monitor=False, + force=False): + """ + Creates all jobs needed for the current workflow. + :param as_conf: AutosubmitConfig object + :type as_conf: AutosubmitConfig + :param date_list: list of dates :type date_list: list - :param member_list: members + :param member_list: list of members :type member_list: list - :param num_chunks: number of chunks to run + :param num_chunks: number of chunks :type num_chunks: int - :param chunk_ini: the experiment will start by the given chunk + :param chunk_ini: initial chunk :type chunk_ini: int - :param parameters: experiment parameters + :param parameters: parameters :type parameters: dict - :param date_format: option to format dates + :param date_format: date format ( D/M/Y ) :type date_format: str - :param default_retrials: default retrials for ech job + :param default_retrials: default number of retrials :type default_retrials: int - :param new: is it a new generation? - :type new: bool \n - :param wrapper_type: Type of wrapper defined by the user in ``autosubmit_.yml`` [wrapper] section. \n - :param wrapper_jobs: Job types defined in ``autosubmit_.yml`` [wrapper sections] to be wrapped. \n - :type wrapper_jobs: String \n + :param default_job_type: default job type + :type default_job_type: str + :param wrapper_jobs: wrapper jobs + :type wrapper_jobs: dict + :param new: new + :type new: bool + :param run_only_members: run only members + :type run_only_members: list + :param show_log: show log + :type show_log: bool + :param monitor: monitor + :type monitor: bool """ + if force: + if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + ".pkl")): + os.remove(os.path.join(self._persistence_path, self._persistence_file + ".pkl")) + if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")): + os.remove(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")) self._parameters = parameters self._date_list = date_list self._member_list = member_list chunk_list = list(range(chunk_ini, num_chunks + 1)) self._chunk_list = chunk_list - - dic_jobs = DicJobs(self, date_list, member_list, chunk_list, date_format, default_retrials, jobs_data, - experiment_data=self.experiment_data) - self._dic_jobs = dic_jobs - priority = 0 + try: + self.graph = self.load() + if type(self.graph) is not DiGraph: + self.graph = nx.DiGraph() + except: + self.graph = nx.DiGraph() + self._dic_jobs = DicJobs(date_list, member_list, chunk_list, date_format, default_retrials, as_conf) + self._dic_jobs.graph = self.graph if show_log: Log.info("Creating jobs...") - # jobs_data includes the name of the .our and .err files of the job in LOG_expid - jobs_data = dict() - if not new: - try: - jobs_data = {row[0]: row for row in self.load()} - except Exception as e: - try: - jobs_data = {row[0]: row for row in self.backup_load()} - except Exception as e: - pass - Log.info("Deleting previous pkl due being incompatible with current AS version") - if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + ".pkl")): - os.remove(os.path.join(self._persistence_path, self._persistence_file + ".pkl")) - if os.path.exists(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")): - os.remove(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")) - - self._create_jobs(dic_jobs, priority, default_job_type, jobs_data) + if len(self.graph.nodes) > 0: + if show_log: + Log.info("Load finished") + if monitor: + as_conf.experiment_data = as_conf.last_experiment_data + as_conf.data_changed = False + if not as_conf.data_changed: + self._dic_jobs._job_list = {job["job"].name: job["job"] for _, job in self.graph.nodes.data() if + job.get("job", None)} + else: + self._dic_jobs.compare_backbone_sections() + # fast-look if graph existed, skips some steps + # If VERSION in CONFIG or HPCARCH in DEFAULT it will exist, if not it won't. + if not new and not self._dic_jobs.changes.get("EXPERIMENT", {}) and not self._dic_jobs.changes.get( + "CONFIG", {}) and not self._dic_jobs.changes.get("DEFAULT", {}): + self._dic_jobs._job_list = {job["job"].name: job["job"] for _, job in self.graph.nodes.data() if + job.get("job", None)} + + # Force to use the last known job_list when autosubmit monitor is running. + self._dic_jobs.last_experiment_data = as_conf.last_experiment_data + else: + # Remove the previous pkl, if it exists. + if not new: + Log.info( + "Removing previous pkl file due to empty graph, likely due using an Autosubmit 4.0.XXX version") + with suppress(FileNotFoundError): + os.remove(os.path.join(self._persistence_path, self._persistence_file + ".pkl")) + with suppress(FileNotFoundError): + os.remove(os.path.join(self._persistence_path, self._persistence_file + "_backup.pkl")) + new = True + # This generates the job object and also finds if dic_jobs has modified from previous iteration in order to expand the workflow + self._create_jobs(self._dic_jobs, 0, default_job_type) + # not needed anymore all data is inside their correspondent sections in dic_jobs + # This dic_job is key to the dependencies management as they're ordered by date[member[chunk]] + del self._dic_jobs._job_list if show_log: - Log.info("Adding dependencies...") - self._add_dependencies(date_list, member_list, chunk_list, dic_jobs, self.graph) + Log.info("Adding dependencies to the graph..") + # del all nodes that are only in the current graph + if len(self.graph.nodes) > 0: + gen = (name for name in set(self.graph.nodes).symmetric_difference(set(self._dic_jobs.workflow_jobs)) if + name in self.graph.nodes) + for name in gen: + self.graph.remove_node(name) + # This actually, also adds the node to the graph if it isn't already there + self._add_dependencies(date_list, member_list, chunk_list, self._dic_jobs) if show_log: - Log.info("Removing redundant dependencies...") - self.update_genealogy( - new, notransitive, update_structure=update_structure) - for job in self._job_list: - job.parameters = parameters - job_data = jobs_data.get(job.name, "none") - try: - if job_data != "none": - job.wrapper_type = job_data[12] - else: - job.wrapper_type = "none" - except BaseException as e: - job.wrapper_type = "none" - + Log.info("Adding dependencies to the job..") + self.update_genealogy() # Checking for member constraints if len(run_only_members) > 0: # Found @@ -278,6 +282,15 @@ class JobList(object): if show_log: Log.info("Looking for edgeless jobs...") self._delete_edgeless_jobs() + if new: + for job in self._job_list: + job.parameters = parameters + if not job.has_parents(): + job.status = Status.READY + job.packed = False + else: + job.status = Status.WAITING + for wrapper_section in wrapper_jobs: try: if wrapper_jobs[wrapper_section] is not None and len(str(wrapper_jobs[wrapper_section])) > 0: @@ -290,46 +303,140 @@ class JobList(object): "Some section jobs of the wrapper:{0} are not in the current job_list defined in jobs.conf".format( wrapper_section), 7014, str(e)) - def _add_dependencies(self, date_list, member_list, chunk_list, dic_jobs, graph, option="DEPENDENCIES"): - jobs_data = dic_jobs._jobs_data.get("JOBS", {}) - for job_section in jobs_data.keys(): + def _add_all_jobs_edge_info(self, dic_jobs, option="DEPENDENCIES"): + jobs_data = dic_jobs.experiment_data.get("JOBS", {}) + sections_gen = (section for section in jobs_data.keys()) + for job_section in sections_gen: + jobs_gen = (job for job in dic_jobs.get_jobs(job_section)) + dependencies_keys = jobs_data.get(job_section, {}).get(option, None) + dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs) if dependencies_keys else {} + for job in jobs_gen: + self._apply_jobs_edge_info(job, dependencies) + + + def _deep_map_dependencies(self, section, jobs_data, option, dependency_list = set(), strip_keys = True): + """ + Recursive function to map dependencies of dependencies + """ + if section in dependency_list: + return dependency_list + dependency_list.add(section) + if not strip_keys: + if "+" in section: + section = section.split("+")[0] + elif "-" in section: + section = section.split("-")[0] + dependencies_keys = jobs_data.get(section, {}).get(option, {}) + for dependency in dependencies_keys: + if strip_keys: + if "+" in dependency: + dependency = dependency.split("+")[0] + elif "-" in dependency: + dependency = dependency.split("-")[0] + dependency_list = self._deep_map_dependencies(dependency, jobs_data, option, dependency_list, strip_keys) + dependency_list.add(dependency) + return dependency_list + + + + def _add_dependencies(self, date_list, member_list, chunk_list, dic_jobs, option="DEPENDENCIES"): + jobs_data = dic_jobs.experiment_data.get("JOBS", {}) + problematic_jobs = {} + # map dependencies + self.dependency_map = dict() + for section in jobs_data.keys(): + self.dependency_map[section] = self._deep_map_dependencies(section, jobs_data, option, set(), strip_keys = True) + self.dependency_map[section].remove(section) + # map dependencies + self.dependency_map_with_distances = dict() + for section in jobs_data.keys(): + self.dependency_map_with_distances[section] = self._deep_map_dependencies(section, jobs_data, option, set(), + strip_keys=False) + self.dependency_map_with_distances[section].remove(section) + + changes = False + if len(self.graph.out_edges) > 0: + sections_gen = (section for section in jobs_data.keys()) + for job_section in sections_gen: # Room for improvement: Do changes only to jobs affected by the yaml changes + if dic_jobs.changes.get(job_section, None) or dic_jobs.changes.get("EXPERIMENT", None) or dic_jobs.changes.get("NEWJOBS", False): + changes = True + break + Log.debug("Looking if there are changes in the workflow") + if changes: + Log.debug("Changes detected, removing all dependencies") + self.graph.clear_edges() # reset edges of all jobs as they need to be recalculated + Log.debug("Dependencies deleted, recalculating dependencies") + else: + Log.debug("No changes detected, keeping edges") + else: + changes = True + Log.debug("No dependencies detected, calculating dependencies") + sections_gen = (section for section in jobs_data.keys()) + for job_section in sections_gen: + # Changes when all jobs of a section are added + self.depends_on_previous_chunk = dict() + self.depends_on_previous_split = dict() + self.depends_on_previous_special_section = dict() + self.actual_job_depends_on_previous_chunk = False + self.actual_job_depends_on_previous_member = False + self.actual_job_depends_on_special_chunk = False + # No changes, no need to recalculate dependencies Log.debug("Adding dependencies for {0} jobs".format(job_section)) - # If it does not have dependencies, do nothing - if not (job_section, option): - continue + # If it does not have dependencies, just append it to job_list and continue + dependencies_keys = jobs_data.get(job_section, {}).get(option, None) + # call function if dependencies_key is not None + dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs) if dependencies_keys else {} + jobs_gen = (job for job in dic_jobs.get_jobs(job_section,sort_string=True)) + for job in jobs_gen: + if job.name not in self.graph.nodes: + self.graph.add_node(job.name, job=job) + elif job.name in self.graph.nodes and self.graph.nodes.get(job.name).get("job", None) is None: # Old versions of autosubmit needs re-adding the job to the graph + self.graph.nodes.get(job.name)["job"] = job + if dependencies and changes: + job = self.graph.nodes.get(job.name)['job'] + problematic_dependencies = self._manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, + dependencies_keys, + dependencies, self.graph) + if len(problematic_dependencies) > 1: + if job_section not in problematic_jobs.keys(): + problematic_jobs[job_section] = {} + problematic_jobs[job_section].update({job.name: problematic_dependencies}) + if changes: + self.find_and_delete_redundant_relations(problematic_jobs) + self._add_all_jobs_edge_info(dic_jobs, option) + + def find_and_delete_redundant_relations(self, problematic_jobs): + ''' + Jobs with intrisic rules than can't be safelty not added without messing other workflows. + The graph will have the least amount of edges added as much as safely possible before this function. + Structure: + problematic_jobs structure is {section: {child_name: [parent_names]}} + + :return: + ''' + delete_relations = set() + for section, jobs in problematic_jobs.items(): + for child_name, parents in jobs.items(): + for parent_name in parents: + for another_parent_name in list(parents)[1:]: + if self.graph.has_successor(parent_name, another_parent_name): + delete_relations.add((parent_name, child_name)) + elif self.graph.has_successor(another_parent_name, parent_name): + delete_relations.add((another_parent_name, child_name)) + + for relation_to_delete in delete_relations: + with suppress(Exception): + self.graph.remove_edge(relation_to_delete[0], relation_to_delete[1]) - dependencies_keys = jobs_data[job_section].get(option, {}) - if type(dependencies_keys) is str: - if "," in dependencies_keys: - dependencies_list = dependencies_keys.split(",") - else: - dependencies_list = dependencies_keys.split(" ") - dependencies_keys = {} - for dependency in dependencies_list: - dependencies_keys[dependency] = {} - if dependencies_keys is None: - dependencies_keys = {} - dependencies = self._manage_dependencies(dependencies_keys, dic_jobs, job_section) - - for job in dic_jobs.get_jobs(job_section): - num_jobs = 1 - if isinstance(job, list): - num_jobs = len(job) - for i in range(num_jobs): - _job = job[i] if num_jobs > 1 else job - self._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys, - dependencies, graph) - pass @staticmethod - def _manage_dependencies(dependencies_keys, dic_jobs, job_section): - parameters = dic_jobs._jobs_data["JOBS"] + def _manage_dependencies(dependencies_keys, dic_jobs): + parameters = dic_jobs.experiment_data["JOBS"] dependencies = dict() - for key in dependencies_keys: + for key in list(dependencies_keys): distance = None splits = None sign = None - if '-' not in key and '+' not in key and '*' not in key and '?' not in key: section = key else: @@ -346,26 +453,14 @@ class JobList(object): key_split = key.split(sign) section = key_split[0] distance = int(key_split[1]) - - if '[' in section: - # Todo check what is this because we never enter this - try: - section_name = section[0:section.find("[")] - splits_section = int( - dic_jobs.experiment_data["JOBS"][section_name].get('SPLITS', -1)) - splits = JobList._calculate_splits_dependencies( - section, splits_section) - section = section_name - except Exception as e: - pass - if parameters.get(section, None) is None: - continue - # raise AutosubmitCritical("Section:{0} doesn't exists.".format(section),7014) - dependency_running_type = str(parameters[section].get('RUNNING', 'once')).lower() - delay = int(parameters[section].get('DELAY', -1)) - dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits, - relationships=dependencies_keys[key]) - dependencies[key] = dependency + if parameters.get(section, None): + dependency_running_type = str(parameters[section].get('RUNNING', 'once')).lower() + delay = int(parameters[section].get('DELAY', -1)) + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits, + relationships=dependencies_keys[key]) + dependencies[key] = dependency + else: + dependencies_keys.pop(key) return dependencies @staticmethod @@ -384,114 +479,13 @@ class JobList(object): splits.append(int(str_split)) return splits - - @staticmethod - def _apply_filter(parent_value, filter_value, associative_list, level_to_check="DATES_FROM", child=None, parent=None): - """ - Check if the current_job_value is included in the filter_value - :param parent_value: - :param filter_value: filter - :param associative_list: dates, members, chunks, splits. - :param filter_type: dates, members, chunks, splits . - :param level_to_check: Can be dates,members, chunks, splits. - :return: - """ - if "NONE".casefold() in str(parent_value).casefold(): - return True - if parent and child and level_to_check.casefold() == "splits".casefold(): - if not parent.splits: - parent_splits = -1 - else: - parent_splits = int(parent.splits) - if not child.splits: - child_splits = -1 - else: - child_splits = int(child.splits) - if parent_splits == child_splits: - to_look_at_lesser = associative_list - lesser_group = -1 - lesser = str(parent_splits) - greater = str(child_splits) - lesser_value = "parent" - else: - if parent_splits > child_splits: - lesser = str(child_splits) - greater = str(parent_splits) - lesser_value = "child" - else: - lesser = str(parent_splits) - greater = str(child_splits) - lesser_value = "parent" - to_look_at_lesser = [associative_list[i:i + 1] for i in range(0, int(lesser), 1)] - for lesser_group in range(len(to_look_at_lesser)): - if lesser_value == "parent": - if str(parent_value) in to_look_at_lesser[lesser_group]: - break - else: - if str(child.split) in to_look_at_lesser[lesser_group]: - break - else: - to_look_at_lesser = associative_list - lesser_group = -1 - if "?" in filter_value: - # replace all ? for "" - filter_value = filter_value.replace("?", "") - if "*" in filter_value: - aux_filter = filter_value - filter_value = "" - for filter_ in aux_filter.split(","): - if "*" in filter_: - filter_,split_info = filter_.split("*") - if "\\" in split_info: - split_info = int(split_info.split("\\")[-1]) - else: - split_info = 1 - # split_info: if a value is 1, it means that the filter is 1-to-1, if it is 2, it means that the filter is 1-to-2, etc. - if child and parent: - if (split_info == 1 or level_to_check.casefold() != "splits".casefold()) and str(parent_value).casefold() == str(filter_).casefold(): - if child.split == parent_value: - return True - elif split_info > 1 and level_to_check.casefold() == "splits".casefold(): - # 1-to-X filter - to_look_at_greater = [associative_list[i:i + split_info] for i in - range(0, int(greater), split_info)] - if lesser_value == "parent": - if str(child.split) in to_look_at_greater[lesser_group]: - return True - else: - if str(parent_value) in to_look_at_greater[lesser_group]: - return True - else: - filter_value += filter_ + "," - else: - filter_value += filter_ + "," - filter_value = filter_value[:-1] - to_filter = JobList._parse_filters_to_check(filter_value,associative_list,level_to_check) - if to_filter is None: - return False - elif len(to_filter) == 0: - return False - elif "ALL".casefold() == str(to_filter[0]).casefold(): - return True - elif "NATURAL".casefold() == str(to_filter[0]).casefold(): - if parent_value is None or parent_value in associative_list: - return True - elif "NONE".casefold() == str(to_filter[0]).casefold(): - return False - elif len( [ filter_ for filter_ in to_filter if str(parent_value).strip(" ").casefold() == str(filter_).strip(" ").casefold() ] )>0: - return True - else: - return False - - - @staticmethod - def _parse_filters_to_check(list_of_values_to_check,value_list=[],level_to_check="DATES_FROM"): + def _parse_filters_to_check(list_of_values_to_check, value_list=[], level_to_check="DATES_FROM"): final_values = [] list_of_values_to_check = str(list_of_values_to_check).upper() if list_of_values_to_check is None: return None - elif list_of_values_to_check.casefold() == "ALL".casefold() : + elif list_of_values_to_check.casefold() == "ALL".casefold(): return ["ALL"] elif list_of_values_to_check.casefold() == "NONE".casefold(): return ["NONE"] @@ -499,14 +493,13 @@ class JobList(object): return ["NATURAL"] elif "," in list_of_values_to_check: for value_to_check in list_of_values_to_check.split(","): - final_values.extend(JobList._parse_filter_to_check(value_to_check,value_list,level_to_check)) + final_values.extend(JobList._parse_filter_to_check(value_to_check, value_list, level_to_check)) else: - final_values = JobList._parse_filter_to_check(list_of_values_to_check,value_list,level_to_check) + final_values = JobList._parse_filter_to_check(list_of_values_to_check, value_list, level_to_check) return final_values - @staticmethod - def _parse_filter_to_check(value_to_check,value_list=[],level_to_check="DATES_FROM"): + def _parse_filter_to_check(value_to_check, value_list=[], level_to_check="DATES_FROM"): """ Parse the filter to check and return the value to check. Selection process: @@ -526,13 +519,13 @@ class JobList(object): # Find N index in the list start = None end = value_to_check.split(":")[1].strip("[]") - if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: + if level_to_check in ["CHUNKS_FROM", "SPLITS_FROM"]: end = int(end) elif value_to_check[-2] == ":": # [N:] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") - if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: + if level_to_check in ["CHUNKS_FROM", "SPLITS_FROM"]: start = int(start) end = None else: @@ -541,7 +534,7 @@ class JobList(object): start = value_to_check.split(":")[0].strip("[]") end = value_to_check.split(":")[1].strip("[]") step = 1 - if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: + if level_to_check in ["CHUNKS_FROM", "SPLITS_FROM"]: start = int(start) end = int(end) elif value_to_check.count(":") == 2: @@ -551,7 +544,7 @@ class JobList(object): start = value_to_check.split(":")[0].strip("[]") end = None step = 1 - if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: + if level_to_check in ["CHUNKS_FROM", "SPLITS_FROM"]: start = int(start) elif value_to_check[1] == ":" and value_to_check[2] == ":": # [::S] # Find N index in the list @@ -560,20 +553,20 @@ class JobList(object): step = value_to_check.split(":")[-1].strip("[]") # get index in the value_list step = int(step) - elif value_to_check[1] == ":" and value_to_check[-2] == ":": # [:M:] + elif value_to_check[1] == ":" and value_to_check[-2] == ":": # [:M:] # Find N index in the list start = None end = value_to_check.split(":")[1].strip("[]") - if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: + if level_to_check in ["CHUNKS_FROM", "SPLITS_FROM"]: end = int(end) step = 1 - else: # [N:M:S] + else: # [N:M:S] # Find N index in the list start = value_to_check.split(":")[0].strip("[]") end = value_to_check.split(":")[1].strip("[]") step = value_to_check.split(":")[2].strip("[]") step = int(step) - if level_to_check in ["CHUNKS_FROM","SPLITS_FROM"]: + if level_to_check in ["CHUNKS_FROM", "SPLITS_FROM"]: start = int(start) end = int(end) else: @@ -589,15 +582,15 @@ class JobList(object): if level_to_check == "CHUNKS_TO": start = int(start) end = int(end) - return value_list[slice(value_list.index(start), value_list.index(end)+1, int(step))] + return value_list[slice(value_list.index(start), value_list.index(end) + 1, int(step))] except ValueError: - return value_list[slice(0,len(value_list)-1,int(step))] + return value_list[slice(0, len(value_list) - 1, int(step))] else: if not start: start = 0 if end is None: return [] - return [number_gen for number_gen in range(int(start), int(end)+1, int(step))] + return [number_gen for number_gen in range(int(start), int(end) + 1, int(step))] def _check_relationship(self, relationships, level_to_check, value_to_check): """ @@ -609,27 +602,26 @@ class JobList(object): """ filters = [] if level_to_check == "DATES_FROM": + if type(value_to_check) != str: + value_to_check = date2str(value_to_check, "%Y%m%d") # need to convert in some cases try: - value_to_check = date2str(value_to_check, "%Y%m%d") # need to convert in some cases - except: - pass - try: - values_list = [date2str(date_, "%Y%m%d") for date_ in self._date_list] # need to convert in some cases + values_list = [date2str(date_, "%Y%m%d") for date_ in self._date_list] # need to convert in some cases except: values_list = self._date_list elif level_to_check == "MEMBERS_FROM": - values_list = self._member_list # Str list + values_list = self._member_list # Str list elif level_to_check == "CHUNKS_FROM": - values_list = self._chunk_list # int list + values_list = self._chunk_list # int list else: - values_list = [] # splits, int list ( artificially generated later ) + values_list = [] # splits, int list ( artificially generated later ) relationship = relationships.get(level_to_check, {}) status = relationship.pop("STATUS", relationships.get("STATUS", None)) from_step = relationship.pop("FROM_STEP", relationships.get("FROM_STEP", None)) for filter_range, filter_data in relationship.items(): selected_filter = JobList._parse_filters_to_check(filter_range, values_list, level_to_check) - if filter_range.casefold() in ["ALL".casefold(),"NATURAL".casefold(),"NONE".casefold()] or not value_to_check: + if filter_range.casefold() in ["ALL".casefold(), "NATURAL".casefold(), + "NONE".casefold()] or not value_to_check: included = True else: included = False @@ -648,7 +640,6 @@ class JobList(object): filters = [{}] return filters - def _check_dates(self, relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value @@ -666,7 +657,8 @@ class JobList(object): # Will enter chunks_from, and obtain [{DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"] if "CHUNKS_FROM" in filter: filters_to_apply_c = self._check_chunks({"CHUNKS_FROM": (filter.pop("CHUNKS_FROM"))}, current_job) - if len(filters_to_apply_c) > 0 and len(filters_to_apply_c[0]) > 0: + if len(filters_to_apply_c) > 0 and (type(filters_to_apply_c) != list or ( + type(filters_to_apply_c) == list and len(filters_to_apply_c[0]) > 0)): filters_to_apply[i].update(filters_to_apply_c) # IGNORED if "SPLITS_FROM" in filter: @@ -678,8 +670,7 @@ class JobList(object): # {DATES_TO: "20020201", MEMBERS_TO: "fc2", CHUNKS_TO: "ALL", SPLITS_TO: "2"} return filters_to_apply - - def _check_members(self,relationships, current_job): + def _check_members(self, relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -699,7 +690,7 @@ class JobList(object): filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply - def _check_chunks(self,relationships, current_job): + def _check_chunks(self, relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -716,7 +707,7 @@ class JobList(object): filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply - def _check_splits(self,relationships, current_job): + def _check_splits(self, relationships, current_job): """ Check if the current_job_value is included in the filter_from and retrieve filter_to value :param relationships: Remaining filters to apply. @@ -729,7 +720,7 @@ class JobList(object): filters_to_apply = self._unify_to_filters(filters_to_apply) return filters_to_apply - def _unify_to_filter(self,unified_filter, filter_to, filter_type): + def _unify_to_filter(self, unified_filter, filter_to, filter_type): """ Unify filter_to filters into a single dictionary :param unified_filter: Single dictionary with all filters_to @@ -752,16 +743,20 @@ class JobList(object): value_list = [] level_to_check = "SPLITS_FROM" if "all".casefold() not in unified_filter[filter_type].casefold(): - aux = filter_to.pop(filter_type, None) + aux = str(filter_to.pop(filter_type, None)) if aux: - aux = aux.split(",") + if "," in aux: + aux = aux.split(",") + else: + aux = [aux] for element in aux: if element == "": continue # Get only the first alphanumeric part and [:] chars - parsed_element = re.findall(r"([\[:\]a-zA-Z0-9]+)", element)[0].lower() + parsed_element = re.findall(r"([\[:\]a-zA-Z0-9._-]+)", element)[0].lower() extra_data = element[len(parsed_element):] - parsed_element = JobList._parse_filter_to_check(parsed_element, value_list = value_list, level_to_check = filter_type) + parsed_element = JobList._parse_filter_to_check(parsed_element, value_list=value_list, + level_to_check=filter_type) # convert list to str skip = False if isinstance(parsed_element, list): @@ -777,9 +772,9 @@ class JobList(object): else: for ele in parsed_element: if extra_data: - check_whole_string = str(ele)+extra_data+"," + check_whole_string = str(ele) + extra_data + "," else: - check_whole_string = str(ele)+"," + check_whole_string = str(ele) + "," if str(check_whole_string) not in unified_filter[filter_type]: unified_filter[filter_type] += check_whole_string return unified_filter @@ -804,7 +799,7 @@ class JobList(object): if "," in filter_to[filter_type][0]: filter_to[filter_type] = filter_to[filter_type][1:] - def _unify_to_filters(self,filter_to_apply): + def _unify_to_filters(self, filter_to_apply): """ Unify all filter_to filters into a single dictionary ( of current selection ) :param filter_to_apply: Filters to apply @@ -826,10 +821,16 @@ class JobList(object): JobList._normalize_to_filters(unified_filter, "MEMBERS_TO") JobList._normalize_to_filters(unified_filter, "CHUNKS_TO") JobList._normalize_to_filters(unified_filter, "SPLITS_TO") + only_none_values = [filters for filters in unified_filter.values() if "none" == filters.lower()] + if len(only_none_values) != 4: + # remove all none filters if not all is none + unified_filter = {key: value for key, value in unified_filter.items() if "none" != value.lower()} + return unified_filter - def _filter_current_job(self,current_job, relationships): - ''' This function will filter the current job based on the relationships given + def _filter_current_job(self, current_job, relationships): + ''' + This function will filter the current job based on the relationships given :param current_job: Current job to filter :param relationships: Relationships to apply :return: dict() with the filters to apply, or empty dict() if no filters to apply @@ -863,6 +864,7 @@ class JobList(object): elif "SPLITS_FROM" in relationships: filters_to_apply = self._check_splits(relationships, current_job) else: + relationships.pop("CHUNKS_FROM", None) relationships.pop("MEMBERS_FROM", None) relationships.pop("DATES_FROM", None) @@ -870,67 +872,6 @@ class JobList(object): filters_to_apply = relationships return filters_to_apply - @staticmethod - def _valid_parent(parent, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child): - ''' - Check if the parent is valid for the current job - :param parent: job to check - :param member_list: list of members - :param date_list: list of dates - :param chunk_list: list of chunks - :param is_a_natural_relation: if the relation is natural or not - :return: True if the parent is valid, False otherwise - ''' - # check if current_parent is listed on dependency.relationships - associative_list = {} - associative_list["dates"] = date_list - associative_list["members"] = member_list - associative_list["chunks"] = chunk_list - - if not child.splits: - child_splits = 0 - else: - child_splits = int(child.splits) - if not parent.splits: - parent_splits = 0 - else: - parent_splits = int(parent.splits) - splits = max(child_splits, parent_splits) - if splits > 0: - associative_list["splits"] = [str(split) for split in range(1, int(splits) + 1)] - else: - associative_list["splits"] = None - dates_to = str(filter_.get("DATES_TO", "natural")).lower() - members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() - chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() - splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() - - if not is_a_natural_relation: - if dates_to == "natural": - dates_to = "none" - if members_to == "natural": - members_to = "none" - if chunks_to == "natural": - chunks_to = "none" - if splits_to == "natural": - splits_to = "none" - if "natural" in dates_to: - associative_list["dates"] = [date2str(parent.date)] if parent.date is not None else date_list - if "natural" in members_to: - associative_list["members"] = [parent.member] if parent.member is not None else member_list - if "natural" in chunks_to: - associative_list["chunks"] = [parent.chunk] if parent.chunk is not None else chunk_list - if "natural" in splits_to: - associative_list["splits"] = [parent.split] if parent.split is not None else parent.splits - parsed_parent_date = date2str(parent.date) if parent.date is not None else None - valid_dates = JobList._apply_filter(parsed_parent_date, dates_to, associative_list["dates"], "dates") - valid_members = JobList._apply_filter(parent.member, members_to, associative_list["members"], "members") - valid_chunks = JobList._apply_filter(parent.chunk, chunks_to, associative_list["chunks"], "chunks") - valid_splits = JobList._apply_filter(parent.split, splits_to, associative_list["splits"], "splits", child, parent) - if valid_dates and valid_members and valid_chunks and valid_splits: - return True - return False - def _add_edge_info(self, job, special_status): """ Special relations to be check in the update_list method @@ -945,113 +886,470 @@ class JobList(object): self.jobs_edges["ALL"] = set() self.jobs_edges["ALL"].add(job) + def add_special_conditions(self, job, special_conditions, filters_to_apply, parent): + """ + Add special conditions to the job edge + :param job: Job + :param special_conditions: dict + :param filters_to_apply: dict + :param parent: parent job + :return: + """ + if special_conditions.get("STATUS", None): + + if special_conditions.get("FROM_STEP", None): + job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int( + special_conditions.get("FROM_STEP", 0)) > job.max_checkpoint_step else job.max_checkpoint_step + self._add_edge_info(job, special_conditions["STATUS"]) # job_list map + job.add_edge_info(parent, special_conditions) # this job + + def _apply_jobs_edge_info(self, job, dependencies): + # prune first + job.edge_info = {} + # get dependency that has special conditions set + filters_to_apply_by_section = dict() + for key, dependency in dependencies.items(): + filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) + if "STATUS" in filters_to_apply: + if "-" in key: + key = key.split("-")[0] + elif "+" in key: + key = key.split("+")[0] + filters_to_apply_by_section[key] = filters_to_apply + if not filters_to_apply_by_section: + return + # divide edge per section name + parents_by_section = dict() + for parent, _ in self.graph.in_edges(job.name): + if self.graph.nodes[parent]['job'].section in filters_to_apply_by_section.keys(): + if self.graph.nodes[parent]['job'].section not in parents_by_section: + parents_by_section[self.graph.nodes[parent]['job'].section] = set() + parents_by_section[self.graph.nodes[parent]['job'].section].add(self.graph.nodes[parent]['job']) + for key, list_of_parents in parents_by_section.items(): + special_conditions = dict() + special_conditions["STATUS"] = filters_to_apply_by_section[key].pop("STATUS", None) + special_conditions["FROM_STEP"] = filters_to_apply_by_section[key].pop("FROM_STEP", None) + for parent in list_of_parents: + self.add_special_conditions(job, special_conditions, filters_to_apply_by_section[key], + parent) + + def find_current_section(self, job_section, section, dic_jobs, distance, visited_section=[]): + sections = dic_jobs.as_conf.jobs_data[section].get("DEPENDENCIES", {}).keys() + if len(sections) == 0: + return distance + sections_str = str("," + ",".join(sections) + ",").upper() + matches = re.findall(rf",{job_section}[+-]*[0-9]*,", sections_str) + if not matches: + for key in [dependency_keys for dependency_keys in sections if job_section not in dependency_keys]: + if "-" in key: + stripped_key = key.split("-")[0] + elif "+" in key: + stripped_key = key.split("+")[0] + else: + stripped_key = key + if stripped_key not in visited_section: + distance = max(self.find_current_section(job_section, stripped_key, dic_jobs, distance, + visited_section + [stripped_key]), distance) + else: + for key in [dependency_keys for dependency_keys in sections if job_section in dependency_keys]: + if "-" in key: + distance = int(key.split("-")[1]) + elif "+" in key: + distance = int(key.split("+")[1]) + if distance > 0: + return distance + return distance + + def _calculate_natural_dependencies(self, dic_jobs, job, dependency, date, member, chunk, graph, + dependencies_keys_without_special_chars, distances_of_current_section, distances_of_current_section_members, + key, dependencies_of_that_section, + chunk_list, date_list, member_list, special_dependencies, max_distance, problematic_dependencies): + """ + Calculate natural dependencies and add them to the graph if they're neccesary. + :param dic_jobs: JobList + :param job: Current job + :param dependency: Dependency + :param date: Date + :param member: Member + :param chunk: Chunk + :param graph: Graph + :param dependencies_keys_without_special_chars: Dependencies of current job without special chars ( without SIM-10 -> SIM ) + :param distances_of_current_section: Distances of current section + :param distances_of_current_section_members: Distances of current section members + :param key: Key + :param dependencies_of_that_section: Dependencies of that section ( Dependencies of target parent ) + :param chunk_list: Chunk list + :param date_list: Date list + :param member_list: Member list + :param special_dependencies: Special dependencies ( dependencies that comes from dependency: special_filters ) + :param max_distance: Max distance ( if a dependency has CLEAN-5 SIM-10, this value would be 10 ) + :param problematic_dependencies: Problematic dependencies + :return: + """ + if key != job.section and not date and not member and not chunk: + if key in dependencies_of_that_section and str(dic_jobs.as_conf.jobs_data[key].get("RUNNING","once")) == "chunk": + natural_parents = [natural_parent for natural_parent in + dic_jobs.get_jobs(dependency.section, date, member, chunk_list[-1]) if + natural_parent.name != job.name] + + elif key in dependencies_of_that_section and str(dic_jobs.as_conf.jobs_data[key].get("RUNNING","once")) == "member": + natural_parents = [natural_parent for natural_parent in + dic_jobs.get_jobs(dependency.section, date, member_list[-1], chunk) if + natural_parent.name != job.name] + else: + natural_parents = [natural_parent for natural_parent in + dic_jobs.get_jobs(dependency.section, date, member, chunk) if + natural_parent.name != job.name] + + else: + natural_parents = [natural_parent for natural_parent in dic_jobs.get_jobs(dependency.section, date, member, chunk) if natural_parent.name != job.name] + # Natural jobs, no filters to apply we can safely add the edge + for parent in natural_parents: + if parent.name in special_dependencies: + continue + if dependency.relationships: # If this section has filter, selects.. + found = [ aux for aux in dic_jobs.as_conf.jobs_data[parent.section].get("DEPENDENCIES",{}).keys() if job.section == aux ] + if found: + continue + if distances_of_current_section.get(dependency.section, 0) == 0: + if job.section == parent.section: + if not self.actual_job_depends_on_previous_chunk: + if parent.section not in self.dependency_map[job.section]: + graph.add_edge(parent.name, job.name) + else: + if self.actual_job_depends_on_special_chunk and not self.actual_job_depends_on_previous_chunk: + if parent.section not in self.dependency_map[job.section]: + if parent.running == job.running: + graph.add_edge(parent.name, job.name) + elif not self.actual_job_depends_on_previous_chunk: + + graph.add_edge(parent.name, job.name) + elif not self.actual_job_depends_on_special_chunk and self.actual_job_depends_on_previous_chunk: + if job.running == "chunk" and job.chunk == 1: + graph.add_edge(parent.name, job.name) + else: + if job.section == parent.section: + if self.actual_job_depends_on_previous_chunk: + skip = False + for aux in [ aux for aux in self.dependency_map[job.section] if aux != job.section]: + distance = 0 + for aux_ in self.dependency_map_with_distances.get(aux,[]): + if "-" in aux_: + if job.section == aux_.split("-")[0]: + distance = int(aux_.split("-")[1]) + elif "+" in aux_: + if job.section == aux_.split("+")[0]: + distance = int(aux_.split("+")[1]) + if distance >= max_distance: + skip = True + if not skip: + # get max value in distances_of_current_section.values + if job.running == "chunk": + if parent.chunk <= (len(chunk_list) - max_distance): + skip = False + if not skip: + problematic_dependencies.add(parent.name) + graph.add_edge(parent.name, job.name) + else: + if job.running == parent.running: + skip = False + problematic_dependencies.add(parent.name) + graph.add_edge(parent.name, job.name) + if parent.running == "chunk": + if parent.chunk > (len(chunk_list) - max_distance): + graph.add_edge(parent.name, job.name) + JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, + member, + member_list, dependency.section, natural_parents) + return problematic_dependencies + + def _calculate_filter_dependencies(self, filters_to_apply, dic_jobs, job, dependency, date, member, chunk, graph, + dependencies_keys_without_special_chars, distances_of_current_section, distances_of_current_section_members, + key, dependencies_of_that_section, + chunk_list, date_list, member_list, special_dependencies, problematic_dependencies): + """ + Calculate dependencies that has any kind of filter set and add them to the graph if they're neccesary. + :param filters_to_apply: Filters to apply + :param dic_jobs: JobList + :param job: Current job + :param dependency: Dependency + :param date: Date + :param member: Member + :param chunk: Chunk + :param graph: Graph + :param dependencies_keys_without_special_chars: Dependencies keys without special chars + :param distances_of_current_section: Distances of current section + :param distances_of_current_section_members: Distances of current section members + :param key: Key + :param dependencies_of_that_section: Dependencies of that section + :param chunk_list: Chunk list + :param date_list: Date list + :param member_list: Member list + :param special_dependencies: Special dependencies + :param problematic_dependencies: Problematic dependencies + :return: + + """ + all_none = True + for filter_value in filters_to_apply.values(): + if str(filter_value).lower() != "none": + all_none = False + break + if all_none: + return special_dependencies, problematic_dependencies + any_all_filter = False + for filter_value in filters_to_apply.values(): + if str(filter_value).lower() == "all": + any_all_filter = True + break + if job.section != dependency.section: + filters_to_apply_of_parent = self._filter_current_job(job, copy.deepcopy( + dependencies_of_that_section.get(dependency.section))) + else: + filters_to_apply_of_parent = {} + possible_parents = [ possible_parent for possible_parent in dic_jobs.get_jobs_filtered(dependency.section, job, filters_to_apply, date, member, chunk, + filters_to_apply_of_parent) if possible_parent.name != job.name] + for parent in possible_parents: + edge_added = False + if any_all_filter: + if ( + (parent.chunk and parent.chunk != self.depends_on_previous_chunk.get(parent.section, + parent.chunk)) or + (parent.running == "chunk" and parent.chunk != chunk_list[-1] and not filters_to_apply_of_parent) or + self.actual_job_depends_on_previous_chunk or + self.actual_job_depends_on_special_chunk or + (parent.name in special_dependencies) + ): + continue + + if parent.section == job.section: + if not job.splits or int(job.splits) > 0: + self.depends_on_previous_split[job.section] = int(parent.split) + if self.actual_job_depends_on_previous_chunk and parent.section == job.section: + graph.add_edge(parent.name, job.name) + edge_added = True + else: + if parent.name not in self.depends_on_previous_special_section.get(job.section,set()) or job.split > 0: + graph.add_edge(parent.name, job.name) + edge_added = True + if parent.section == job.section: + self.actual_job_depends_on_special_chunk = True + if edge_added: + if job.name not in self.depends_on_previous_special_section: + self.depends_on_previous_special_section[job.name] = set() + if job.section not in self.depends_on_previous_special_section: + self.depends_on_previous_special_section[job.section] = set() + if parent.name in self.depends_on_previous_special_section.keys(): + special_dependencies.update(self.depends_on_previous_special_section[parent.name]) + self.depends_on_previous_special_section[job.name].add(parent.name) + self.depends_on_previous_special_section[job.section].add(parent.name) + problematic_dependencies.add(parent.name) + + + JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, + member_list, dependency.section, possible_parents) + + return special_dependencies, problematic_dependencies + def _manage_job_dependencies(self, dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): - ''' - Manage the dependencies of a job - :param dic_jobs: - :param job: - :param date_list: - :param member_list: - :param chunk_list: - :param dependencies_keys: - :param dependencies: - :param graph: - :return: - ''' + """ + Manage job dependencies + :param dic_jobs: JobList + :param job: Current job + :param date_list: Date list + :param member_list: Member list + :param chunk_list: Chunk list + :param dependencies_keys: Dependencies keys + :param dependencies: Dependencies + :param graph: Graph + :return: problematic_dependencies + """ + # self.depends_on_previous_chunk = dict() + depends_on_previous_section = set() + distances_of_current_section = {} + distances_of_current_section_member = {} + problematic_dependencies = set() + special_dependencies = set() + dependencies_to_del = set() + dependencies_non_natural_to_del = set() + max_distance = 0 + dependencies_keys_aux = [] + dependencies_keys_without_special_chars = [] + depends_on_itself = None + if not job.splits: + child_splits = 0 + else: + child_splits = int(job.splits) parsed_date_list = [] for dat in date_list: parsed_date_list.append(date2str(dat)) - special_conditions = dict() - for key in dependencies_keys: - dependency = dependencies.get(key, None) - if dependency is None: - Log.printlog("WARNING: SECTION {0} is not defined in jobs.conf. Dependency skipped".format(key), - Log.WARNING) - continue + # It is faster to check the conf instead of calculate 90000000 tasks + # Prune number of dependencies to check, to reduce the transitive reduction complexity + for dependency in dependencies_keys.keys(): + if ("-" in dependency and job.section == dependency.split("-")[0]) or ( + "+" in dependency and job.section == dependency.split("+")[0]) or (job.section == dependency): + depends_on_itself = dependency + else: + dependencies_keys_aux.append(dependency) + if depends_on_itself: + dependencies_keys_aux = dependencies_keys_aux + [depends_on_itself] + + for key_aux_stripped in dependencies_keys_aux: + if "-" in key_aux_stripped: + key_aux_stripped = key_aux_stripped.split("-")[0] + elif "+" in key_aux_stripped: + key_aux_stripped = key_aux_stripped.split("+")[0] + dependencies_keys_without_special_chars.append(key_aux_stripped) + self.dependency_map[job.section] = self.dependency_map[job.section].difference(set(dependencies_keys_aux)) + # If parent already has defined that dependency, skip it to reduce the transitive reduction complexity + # Calcule distances ( SIM-1, ClEAN-2..) + for dependency_key in dependencies_keys_aux: + if "-" in dependency_key: + aux_key = dependency_key.split("-")[0] + distance = int(dependency_key.split("-")[1]) + elif "+" in dependency_key: + aux_key = dependency_key.split("+")[0] + distance = int(dependency_key.split("+")[1]) + else: + aux_key = dependency_key + distance = 0 + if dic_jobs.as_conf.jobs_data.get(aux_key, {}).get("RUNNING", + "once") == "chunk": + distances_of_current_section[aux_key] = distance + elif dic_jobs.as_conf.jobs_data.get(aux_key, {}).get("RUNNING","once") == "member": + distances_of_current_section_member[aux_key] = distance + if distance != 0: + if job.running == "chunk": + if int(job.chunk) > 1: + if job.section == aux_key or dic_jobs.as_conf.jobs_data.get(aux_key, {}).get("RUNNING", + "once") == "chunk": + self.actual_job_depends_on_previous_chunk = True + if job.running == "member" or job.running == "chunk": + # find member in member_list + if job.member: + if member_list.index(job.member) > 0: + if job.section == aux_key or dic_jobs.as_conf.jobs_data.get(aux_key, {}).get("RUNNING", + "once") == "member": + self.actual_job_depends_on_previous_member = True + if aux_key != job.section: + dependencies_of_that_section = dic_jobs.as_conf.jobs_data[aux_key].get("DEPENDENCIES", {}) + for key in dependencies_of_that_section.keys(): + if "-" in key: + stripped_key = key.split("-")[0] + distance_ = int(key.split("-")[1]) + elif "+" in key: + stripped_key = key.split("+")[0] + distance_ = int(key.split("+")[1]) + else: + stripped_key = key + distance_ = 0 + if stripped_key in dependencies_keys_without_special_chars and stripped_key != job.section: + # Fix delay + if job.running == "chunk" and dic_jobs.as_conf.jobs_data[aux_key].get("DELAY", None): + if job.chunk <= int(dic_jobs.as_conf.jobs_data[aux_key].get("DELAY", 0)): + continue + # check doc example + if dependencies.get(stripped_key,None) and not dependencies[stripped_key].relationships: + dependencies_to_del.add(key) + + max_distance = 0 + for key in self.dependency_map_with_distances[job.section]: + if "-" in key: + aux_key = key.split("-")[0] + distance = int(key.split("-")[1]) + elif "+" in key: + aux_key = key.split("+")[0] + distance = int(key.split("+")[1]) + else: + distance = 0 + max_distance = max(max_distance, distance) + if dic_jobs.as_conf.jobs_data.get(aux_key, {}).get("RUNNING","once") == "chunk": + if aux_key in distances_of_current_section: + if distance > distances_of_current_section[aux_key]: + distances_of_current_section[aux_key] = distance + elif dic_jobs.as_conf.jobs_data.get(aux_key, {}).get("RUNNING","once") == "member": + if aux_key in distances_of_current_section_member: + if distance > distances_of_current_section_member[aux_key]: + distances_of_current_section_member[aux_key] = distance + sections_to_calculate = [key for key in dependencies_keys_aux if key not in dependencies_to_del] + natural_sections = list() + # Parse first sections with special filters if any + for key in sections_to_calculate: + dependency = dependencies[key] skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, chunk_list, job.member, member_list, job.date, date_list, dependency) if skip: continue - - other_parents = dic_jobs.get_jobs(dependency.section, None, None, None) - parents_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) - natural_jobs = dic_jobs.get_jobs(dependency.section, date, member, chunk) - all_parents = list(set(other_parents + parents_jobs)) - # Get dates_to, members_to, chunks_to of the deepest level of the relationship. filters_to_apply = self._filter_current_job(job, copy.deepcopy(dependency.relationships)) - if "?" in filters_to_apply.get("SPLITS_TO", "") or "?" in filters_to_apply.get("DATES_TO","") or "?" in filters_to_apply.get("MEMBERS_TO", "") or "?" in filters_to_apply.get("CHUNKS_TO", ""): - only_marked_status = True + filters_to_apply.pop("STATUS", None) + filters_to_apply.pop("FROM_STEP", None) + if len(filters_to_apply) > 0: + dependencies_of_that_section = dic_jobs.as_conf.jobs_data[dependency.section].get("DEPENDENCIES", {}) + special_dependencies, problematic_dependencies = self._calculate_filter_dependencies(filters_to_apply, dic_jobs, job, dependency, date, member, chunk, + graph, + dependencies_keys_without_special_chars, + distances_of_current_section, distances_of_current_section_member, + key, + dependencies_of_that_section, chunk_list, date_list, member_list, special_dependencies, problematic_dependencies) else: - only_marked_status = False - special_conditions["STATUS"] = filters_to_apply.pop("STATUS", None) - special_conditions["FROM_STEP"] = filters_to_apply.pop("FROM_STEP", None) - for parent in all_parents: - # If splits is not None, the job is a list of jobs - if parent.name == job.name: - continue - # Check if it is a natural relation. The only difference is that a chunk can depend on a chunks <= than the current chunk - if parent in natural_jobs and (job.chunk is None or parent.chunk is None or parent.chunk <= job.chunk): - natural_relationship = True + if key in dependencies_non_natural_to_del: + continue else: - natural_relationship = False - # Check if the current parent is a valid parent based on the dependencies set on expdef.conf - # If the parent is valid, add it to the graph - - if JobList._valid_parent(parent, member_list, parsed_date_list, chunk_list, natural_relationship, - filters_to_apply,job): - job.add_parent(parent) - self._add_edge(graph, job, parent) - # Could be more variables in the future - # Do parse checkpoint - if special_conditions.get("STATUS", None): - if only_marked_status: - if str(job.split) + "?" in filters_to_apply.get("SPLITS_TO", "") or str( - job.chunk) + "?" in filters_to_apply.get("CHUNKS_TO", "") or str( - job.member) + "?" in filters_to_apply.get("MEMBERS_TO", "") or str( - job.date) + "?" in filters_to_apply.get("DATES_TO", ""): - selected = True - else: - selected = False - else: - selected = True - if selected: - if special_conditions.get("FROM_STEP", None): - job.max_checkpoint_step = int(special_conditions.get("FROM_STEP", 0)) if int( - special_conditions.get("FROM_STEP", - 0)) > job.max_checkpoint_step else job.max_checkpoint_step - self._add_edge_info(job, special_conditions["STATUS"]) - job.add_edge_info(parent, special_conditions) - JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, - member_list, dependency.section, graph, other_parents) - + natural_sections.append(key) + for key in natural_sections: + dependency = dependencies[key] + skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, chunk_list, + job.member, member_list, + job.date, date_list, + dependency) + if skip: + continue + aux = dic_jobs.as_conf.jobs_data[dependency.section].get("DEPENDENCIES", {}) + dependencies_of_that_section = [] + for key_aux_stripped in aux.keys(): + if "-" in key_aux_stripped: + key_aux_stripped = key_aux_stripped.split("-")[0] + elif "+" in key_aux_stripped: + key_aux_stripped = key_aux_stripped.split("+")[0] + + dependencies_of_that_section.append(key_aux_stripped) + + problematic_dependencies = self._calculate_natural_dependencies(dic_jobs, job, dependency, date, + member, chunk, graph, + dependencies_keys_without_special_chars, + distances_of_current_section, distances_of_current_section_member, + key, + dependencies_of_that_section, chunk_list, date_list, member_list, special_dependencies, max_distance, problematic_dependencies) + return problematic_dependencies @staticmethod def _calculate_dependency_metadata(chunk, chunk_list, member, member_list, date, date_list, dependency): skip = False if dependency.sign == '-': if chunk is not None and len(str(chunk)) > 0 and dependency.running == 'chunk': - chunk_index = chunk_list.index(chunk) + chunk_index = chunk - 1 if chunk_index >= dependency.distance: chunk = chunk_list[chunk_index - dependency.distance] else: skip = True elif member is not None and len(str(member)) > 0 and dependency.running in ['chunk', 'member']: + # improve this TODO member_index = member_list.index(member) if member_index >= dependency.distance: member = member_list[member_index - dependency.distance] else: skip = True elif date is not None and len(str(date)) > 0 and dependency.running in ['chunk', 'member', 'startdate']: + # improve this TODO date_index = date_list.index(date) if date_index >= dependency.distance: date = date_list[date_index - dependency.distance] else: skip = True - - if dependency.sign == '+': + elif dependency.sign == '+': if chunk is not None and len(str(chunk)) > 0 and dependency.running == 'chunk': chunk_index = chunk_list.index(chunk) if (chunk_index + dependency.distance) < len(chunk_list): @@ -1063,7 +1361,6 @@ class JobList(object): if (chunk_index + temp_distance) < len(chunk_list): chunk = chunk_list[chunk_index + temp_distance] break - elif member is not None and len(str(member)) > 0 and dependency.running in ['chunk', 'member']: member_index = member_list.index(member) if (member_index + dependency.distance) < len(member_list): @@ -1080,8 +1377,8 @@ class JobList(object): @staticmethod def handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, - section_name, graph, visited_parents): - if job.wait and job.frequency > 1: + section_name, visited_parents): + if job.frequency and job.frequency > 1: if job.chunk is not None and len(str(job.chunk)) > 0: max_distance = (chunk_list.index(chunk) + 1) % job.frequency if max_distance == 0: @@ -1090,7 +1387,6 @@ class JobList(object): for parent in dic_jobs.get_jobs(section_name, date, member, chunk - distance): if parent not in visited_parents: job.add_parent(parent) - JobList._add_edge(graph, job, parent) elif job.member is not None and len(str(job.member)) > 0: member_index = member_list.index(job.member) max_distance = (member_index + 1) % job.frequency @@ -1101,7 +1397,6 @@ class JobList(object): member_list[member_index - distance], chunk): if parent not in visited_parents: job.add_parent(parent) - JobList._add_edge(graph, job, parent) elif job.date is not None and len(str(job.date)) > 0: date_index = date_list.index(job.date) max_distance = (date_index + 1) % job.frequency @@ -1112,23 +1407,12 @@ class JobList(object): member, chunk): if parent not in visited_parents: job.add_parent(parent) - JobList._add_edge(graph, job, parent) @staticmethod - def _add_edge(graph, job, parents): - num_parents = 1 - if isinstance(parents, list): - num_parents = len(parents) - for i in range(num_parents): - parent = parents[i] if isinstance(parents, list) else parents - graph.add_edge(parent.name, job.name) - pass - - @staticmethod - def _create_jobs(dic_jobs, priority, default_job_type, jobs_data=dict()): - for section in dic_jobs._jobs_data.get("JOBS", {}).keys(): + def _create_jobs(dic_jobs, priority, default_job_type): + for section in (job for job in dic_jobs.experiment_data.get("JOBS", {}).keys()): Log.debug("Creating {0} jobs".format(section)) - dic_jobs.read_section(section, priority, default_job_type, jobs_data) + dic_jobs.read_section(section, priority, default_job_type) priority += 1 def _create_sorted_dict_jobs(self, wrapper_jobs): @@ -1185,11 +1469,9 @@ class JobList(object): str_date = self._get_date(date) for member in self._member_list: # Filter list of fake jobs according to date and member, result not sorted at this point - sorted_jobs_list = list(filter(lambda job: job.name.split("_")[1] == str_date and - job.name.split("_")[2] == member, - filtered_jobs_fake_date_member)) - # sorted_jobs_list = [job for job in filtered_jobs_fake_date_member if job.name.split("_")[1] == str_date and - # job.name.split("_")[2] == member] + sorted_jobs_list = [job for job in filtered_jobs_fake_date_member if + job.name.split("_")[1] == str_date and + job.name.split("_")[2] == member] # There can be no jobs for this member when select chunk/member is enabled if not sorted_jobs_list or len(sorted_jobs_list) == 0: @@ -1509,7 +1791,7 @@ class JobList(object): """ unsubmitted = [job for job in self._job_list if (platform is None or job.platform.name == platform.name) and ( - job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status == Status.RUNNING and job.status == Status.COMPLETED)] + job.status != Status.SUBMITTED and job.status != Status.QUEUING and job.status == Status.RUNNING and job.status == Status.COMPLETED)] if wrapper: return [job for job in unsubmitted if job.packed is False] @@ -1670,11 +1952,11 @@ class JobList(object): if section_chunks != "" or section_members != "": jobs_final = [job for job in jobs_date if ( - section_chunks == "" or re.search("(^|[^0-9a-z_])" + str(job.chunk) + "([^a-z0-9_]|$)", - section_chunks) is not None) and ( - section_members == "" or re.search( - "(^|[^0-9a-z_])" + str(job.member) + "([^a-z0-9_]|$)", - section_members.lower()) is not None)] + section_chunks == "" or re.search("(^|[^0-9a-z_])" + str(job.chunk) + "([^a-z0-9_]|$)", + section_chunks) is not None) and ( + section_members == "" or re.search( + "(^|[^0-9a-z_])" + str(job.member) + "([^a-z0-9_]|$)", + section_members.lower()) is not None)] ultimate_jobs_list.extend(jobs_final) # Duplicates out ultimate_jobs_list = list(set(ultimate_jobs_list)) @@ -2033,7 +2315,12 @@ class JobList(object): :rtype: JobList """ Log.info("Loading JobList") - return self._persistence.load(self._persistence_path, self._persistence_file) + try: + return self._persistence.load(self._persistence_path, self._persistence_file) + except: + Log.printlog( + "Autosubmit will use a backup for recover the job_list", 6010) + return self.backup_load() def backup_load(self): """ @@ -2063,8 +2350,8 @@ class JobList(object): try: self._persistence.save(self._persistence_path, self._persistence_file, - self._job_list if self.run_members is None or job_list is None else job_list) - pass + self._job_list if self.run_members is None or job_list is None else job_list, + self.graph) except BaseException as e: raise AutosubmitError(str(e), 6040, "Failure while saving the job_list") except AutosubmitError as e: @@ -2095,14 +2382,15 @@ class JobList(object): Log.status_failed("\n{0:<35}{1:<15}{2:<15}{3:<20}{4:<15}", "Job Name", "Job Id", "Job Status", "Job Platform", "Job Queue") for job in job_list: - if len(job.queue) > 0 and str(job.platform.queue).lower() != "none": + if job.platform and len(job.queue) > 0 and str(job.platform.queue).lower() != "none": queue = job.queue - elif len(job.platform.queue) > 0 and str(job.platform.queue).lower() != "none": + elif job.platform and len(job.platform.queue) > 0 and str(job.platform.queue).lower() != "none": queue = job.platform.queue else: queue = job.queue + platform_name = job.platform.name if job.platform else "no-platform" Log.status("{0:<35}{1:<15}{2:<15}{3:<20}{4:<15}", job.name, job.id, Status( - ).VALUE_TO_KEY[job.status], job.platform.name, queue) + ).VALUE_TO_KEY[job.status], platform_name, queue) for job in failed_job_list: if len(job.queue) < 1: queue = "no-scheduler" @@ -2176,7 +2464,7 @@ class JobList(object): continue if status in ["RUNNING", "FAILED"]: # check checkpoint if any - if job.platform.connected: # This will be true only when used under setstatus/run + if job.platform and job.platform.connected: # This will be true only when used under setstatus/run job.get_checkpoint_files() non_completed_parents_current = 0 completed_parents = len([parent for parent in job.parents if parent.status == Status.COMPLETED]) @@ -2187,12 +2475,11 @@ class JobList(object): status_str = Status.VALUE_TO_KEY[parent[0].status] if Status.LOGICAL_ORDER.index(status_str) >= Status.LOGICAL_ORDER.index(status): non_completed_parents_current += 1 - if ( non_completed_parents_current + completed_parents ) == len(job.parents): + if (non_completed_parents_current + completed_parents) == len(job.parents): jobs_to_check.append(job) return jobs_to_check - def update_list(self, as_conf, store_change=True, fromSetStatus=False, submitter=None, first_time=False): # type: (AutosubmitConfig, bool, bool, object, bool) -> bool """ @@ -2215,6 +2502,7 @@ class JobList(object): write_log_status = False if not first_time: for job in self.get_failed(): + job.packed = False if self.jobs_data[job.section].get("RETRIALS", None) is None: retrials = int(as_conf.get_retrials()) else: @@ -2224,10 +2512,13 @@ class JobList(object): tmp = [ parent for parent in job.parents if parent.status == Status.COMPLETED] if len(tmp) == len(job.parents): - if "+" == str(job.delay_retrials)[0] or "*" == str(job.delay_retrials)[0]: - aux_job_delay = int(job.delay_retrials[1:]) - else: - aux_job_delay = int(job.delay_retrials) + aux_job_delay = 0 + if job.delay_retrials: + + if "+" == str(job.delay_retrials)[0] or "*" == str(job.delay_retrials)[0]: + aux_job_delay = int(job.delay_retrials[1:]) + else: + aux_job_delay = int(job.delay_retrials) if self.jobs_data[job.section].get("DELAY_RETRY_TIME", None) or aux_job_delay <= 0: delay_retry_time = str(as_conf.get_delay_retry_time()) @@ -2248,6 +2539,7 @@ class JobList(object): "Resetting job: {0} status to: DELAYED for retrial...".format(job.name)) else: job.status = Status.READY + job.packed = False Log.debug( "Resetting job: {0} status to: READY for retrial...".format(job.name)) job.id = None @@ -2274,6 +2566,7 @@ class JobList(object): Log.debug(f"Special condition fullfilled for job {job.name}") # if waiting jobs has all parents completed change its State to READY for job in self.get_completed(): + job.packed = False if job.synchronize is not None and len(str(job.synchronize)) > 0: tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED] if len(tmp) != len(job.parents): @@ -2300,6 +2593,7 @@ class JobList(object): for job in self.get_delayed(): if datetime.datetime.now() >= job.delay_end: job.status = Status.READY + job.packed = False for job in self.get_waiting(): tmp = [parent for parent in job.parents if parent.status == Status.COMPLETED or parent.status == Status.SKIPPED] @@ -2310,6 +2604,7 @@ class JobList(object): failed_ones = [parent for parent in job.parents if parent.status == Status.FAILED] if job.parents is None or len(tmp) == len(job.parents): job.status = Status.READY + job.packed = False job.hold = False Log.debug( "Setting job: {0} status to: READY (all parents completed)...".format(job.name)) @@ -2329,6 +2624,7 @@ class JobList(object): break if not strong_dependencies_failure and weak_dependencies_failure: job.status = Status.READY + job.packed = False job.hold = False Log.debug( "Setting job: {0} status to: READY (conditional jobs are completed/failed)...".format( @@ -2341,6 +2637,7 @@ class JobList(object): for parent in job.parents: if parent.name in job.edge_info and job.edge_info[parent.name].get('optional', False): job.status = Status.READY + job.packed = False job.hold = False Log.debug( "Setting job: {0} status to: READY (conditional jobs are completed/failed)...".format( @@ -2368,7 +2665,7 @@ class JobList(object): if job.name not in all_parents_completed: tmp = [parent for parent in job.parents if ( ( - parent.status == Status.SKIPPED or parent.status == Status.COMPLETED or parent.status == Status.QUEUING or parent.status == Status.RUNNING) and "setup" not in parent.name.lower())] + parent.status == Status.SKIPPED or parent.status == Status.COMPLETED or parent.status == Status.QUEUING or parent.status == Status.RUNNING) and "setup" not in parent.name.lower())] if len(tmp) == len(job.parents): job.status = Status.PREPARED job.hold = True @@ -2452,92 +2749,20 @@ class JobList(object): Log.debug('Update finished') return save - def update_genealogy(self, new=True, notransitive=False, update_structure=False): + def update_genealogy(self): """ When we have created the job list, every type of job is created. Update genealogy remove jobs that have no templates - :param update_structure: - :param notransitive: - :param new: if it is a new job list or not - :type new: bool """ - - # Use a copy of job_list because original is modified along iterations - for job in self._job_list[:]: - if job.file is None or job.file == '': - self._remove_job(job) - - # Simplifying dependencies: if a parent is already an ancestor of another parent, - # we remove parent dependency - if not notransitive: - # Transitive reduction required - current_structure = None - db_path = os.path.join( - self._config.STRUCTURES_DIR, "structure_" + self.expid + ".db") - m_time_db = None - jobs_conf_path = os.path.join( - self._config.LOCAL_ROOT_DIR, self.expid, "conf", "jobs_{0}.yml".format(self.expid)) - m_time_job_conf = None - if os.path.exists(db_path): - try: - current_structure = DbStructure.get_structure( - self.expid, self._config.STRUCTURES_DIR) - m_time_db = os.stat(db_path).st_mtime - if os.path.exists(jobs_conf_path): - m_time_job_conf = os.stat(jobs_conf_path).st_mtime - except Exception as exp: - pass - structure_valid = False - # If there is a current structure, and the number of jobs in JobList is equal to the number of jobs in the structure - if (current_structure) and (len(self._job_list) == len(current_structure)) and update_structure is False: - structure_valid = True - # Further validation - # Structure exists and is valid, use it as a source of dependencies - if m_time_job_conf: - if m_time_job_conf > m_time_db: - Log.info( - "File jobs_{0}.yml has been modified since the last time the structure persistence was saved.".format( - self.expid)) - structure_valid = False - else: - Log.info( - "File jobs_{0}.yml was not found.".format(self.expid)) - - if structure_valid is True: - for job in self._job_list: - if current_structure.get(job.name, None) is None: - structure_valid = False - break - - if structure_valid is True: - Log.info("Using existing valid structure.") - for job in self._job_list: - children_to_remove = [ - child for child in job.children if child.name not in current_structure[job.name]] - for child in children_to_remove: - job.children.remove(child) - child.parents.remove(job) - if structure_valid is False: - # Structure does not exist, or it is not be updated, attempt to create it. - Log.info("Updating structure persistence...") - self.graph = transitive_reduction(self.graph) # add threads for large experiments? todo - if self.graph: - for job in self._job_list: - children_to_remove = [ - child for child in job.children if child.name not in self.graph.neighbors(job.name)] - for child in children_to_remove: - job.children.remove(child) - child.parents.remove(job) - try: - DbStructure.save_structure( - self.graph, self.expid, self._config.STRUCTURES_DIR) - except Exception as exp: - Log.warning(str(exp)) - pass - - for job in self._job_list: - if not job.has_parents() and new: - job.status = Status.READY + Log.info("Transitive reduction...") + # This also adds the jobs edges to the job itself (job._parents and job._children) + self.graph = transitive_reduction(self.graph) + # update job list view as transitive_Reduction also fills job._parents and job._children if recreate is set + self._job_list = [job["job"] for job in self.graph.nodes().values()] + try: + DbStructure.save_structure(self.graph, self.expid, self._config.STRUCTURES_DIR) + except Exception as exp: + Log.warning(str(exp)) @threaded def check_scripts_threaded(self, as_conf): @@ -2585,20 +2810,15 @@ class JobList(object): out = True # Implementing checking scripts feedback to the users in a minimum of 4 messages count = stage = 0 - for job in self._job_list: + for job in (job for job in self._job_list): + job.update_check_variables(as_conf) count += 1 if (count >= len(self._job_list) / 4 * (stage + 1)) or count == len(self._job_list): stage += 1 Log.info("{} of {} checked".format(count, len(self._job_list))) show_logs = str(job.check_warnings).lower() - if job.check == 'on_submission': - Log.info( - 'Template {0} will be checked in running time'.format(job.section)) - continue - elif job.check == "false": - Log.info( - 'Template {0} will not be checked'.format(job.section)) + if job.check in ['on_submission','false']: continue else: if job.section in self.sections_checked: @@ -2676,7 +2896,7 @@ class JobList(object): dependencies_keys = dependencies_keys.upper().split() if dependencies_keys is None: dependencies_keys = [] - dependencies = JobList._manage_dependencies(dependencies_keys, self._dic_jobs, job_section) + dependencies = JobList._manage_dependencies(dependencies_keys, self._dic_jobs) for job in self.get_jobs_by_section(job_section): for key in dependencies_keys: dependency = dependencies[key] @@ -2706,13 +2926,13 @@ class JobList(object): Removes all jobs to be run only in reruns """ flag = False - for job in set(self._job_list): + for job in self._job_list[:]: if job.rerun_only == "true": self._remove_job(job) flag = True if flag: - self.update_genealogy(notransitive=notransitive) + self.update_genealogy() del self._dic_jobs def print_with_status(self, statusChange=None, nocolor=False, existingList=None): @@ -2742,7 +2962,6 @@ class JobList(object): result += " ## " # Find root - root = None roots = [] for job in allJobs: if len(job.parents) == 0: @@ -2759,31 +2978,28 @@ class JobList(object): return result - def __str__(self): + def __repr__(self): """ Returns the string representation of the class. - Usage print(class) - :return: String representation. :rtype: String """ - allJobs = self.get_all() - result = "## String representation of Job List [" + str( - len(allJobs)) + "] ##" - - # Find root - root = None - for job in allJobs: - if job.has_parents() is False: - root = job - - # root exists - if root is not None and len(str(root)) > 0: - result += self._recursion_print(root, 0) - else: - result += "\nCannot find root." - - return result + try: + results = [f"## String representation of Job List [{len(self.jobs)}] ##"] + # Find root + roots = [job for job in self.get_all() + if len(job.parents) == 0 + and job is not None and len(str(job)) > 0] + visited = list() + # root exists + for root in roots: + if root is not None and len(str(root)) > 0: + results.append(self._recursion_print(root, 0, visited, nocolor=True)) + else: + results.append("Cannot find root.") + except: + return f'Job List object' + return "\n".join(results) def _recursion_print(self, job, level, visited=[], statusChange=None, nocolor=False): """ @@ -2971,17 +3187,17 @@ class JobList(object): # Using standard procedure if status_code in [Status.RUNNING, Status.SUBMITTED, Status.QUEUING, - Status.FAILED] or make_exception is True: + Status.FAILED] or make_exception is True: # COMPLETED adds too much overhead so these values are now stored in a database and retrieved separately submit_time, start_time, finish_time, status = JobList._job_running_check( status_code, name, tmp_path) - if status_code in [Status.RUNNING, Status.FAILED]: + if status_code in [Status.RUNNING, Status.FAILED, Status.COMPLETED]: running_for_min = (finish_time - start_time) queuing_for_min = (start_time - submit_time) submit_time = mktime(submit_time.timetuple()) start_time = mktime(start_time.timetuple()) finish_time = mktime(finish_time.timetuple()) if status_code in [ - Status.FAILED] else 0 + Status.FAILED, Status.COMPLETED] else 0 else: queuing_for_min = ( datetime.datetime.now() - submit_time) diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 7554ddad746eeee5bcdbbb6920b78080a8024e68..592fcc1961fa16be4f8b8513e96906e0d662752f 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -13,16 +13,15 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. - # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import pickle -from sys import setrecursionlimit import os +import pickle +from sys import setrecursionlimit -from log.log import Log from autosubmit.database.db_manager import DbManager +from log.log import Log class JobListPersistence(object): @@ -31,7 +30,7 @@ class JobListPersistence(object): """ - def save(self, persistence_path, persistence_file, job_list): + def save(self, persistence_path, persistence_file, job_list , graph): """ Persists a job list :param job_list: JobList @@ -68,13 +67,22 @@ class JobListPersistencePkl(JobListPersistence): """ path = os.path.join(persistence_path, persistence_file + '.pkl') if os.path.exists(path): - fd = open(path, 'rb') - return pickle.load(fd) + with open(path, 'rb') as fd: + graph = pickle.load(fd) + for u in ( node for node in graph ): + # Set after the dependencies are set + graph.nodes[u]["job"].children = set() + graph.nodes[u]["job"].parents = set() + # Set in recovery/run + graph.nodes[u]["job"]._platform = None + graph.nodes[u]["job"]._serial_platform = None + graph.nodes[u]["job"].submitter = None + return graph else: Log.printlog('File {0} does not exist'.format(path),Log.WARNING) return list() - def save(self, persistence_path, persistence_file, job_list): + def save(self, persistence_path, persistence_file, job_list, graph): """ Persists a job list in a pkl file :param job_list: JobList @@ -83,15 +91,10 @@ class JobListPersistencePkl(JobListPersistence): """ path = os.path.join(persistence_path, persistence_file + '.pkl') - fd = open(path, 'wb') - setrecursionlimit(50000) + setrecursionlimit(500000000) Log.debug("Saving JobList: " + path) - jobs_data = [(job.name, job.id, job.status, - job.priority, job.section, job.date, - job.member, job.chunk, - job.local_logs[0], job.local_logs[1], - job.remote_logs[0], job.remote_logs[1],job.wrapper_type) for job in job_list] - pickle.dump(jobs_data, fd, protocol=2) + with open(path, 'wb') as fd: + pickle.dump(graph, fd, pickle.HIGHEST_PROTOCOL) Log.debug('Job list saved') @@ -120,7 +123,7 @@ class JobListPersistenceDb(JobListPersistence): """ return self.db_manager.select_all(self.JOB_LIST_TABLE) - def save(self, persistence_path, persistence_file, job_list): + def save(self, persistence_path, persistence_file, job_list, graph): """ Persists a job list in a database :param job_list: JobList @@ -131,7 +134,7 @@ class JobListPersistenceDb(JobListPersistence): self._reset_table() jobs_data = [(job.name, job.id, job.status, job.priority, job.section, job.date, - job.member, job.chunk, + job.member, job.chunk, job.split, job.local_logs[0], job.local_logs[1], job.remote_logs[0], job.remote_logs[1],job.wrapper_type) for job in job_list] self.db_manager.insertMany(self.JOB_LIST_TABLE, jobs_data) diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index 67e833e2719049822cd43220dc0ea68a96bc7ad5..5bec964355fdbd6429146e0a80341c65d4ec885f 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -26,8 +26,7 @@ from operator import attrgetter from math import ceil import operator from typing import List -import copy - +from contextlib import suppress class JobPackager(object): @@ -42,44 +41,6 @@ class JobPackager(object): :type jobs_list: JobList object. """ - def calculate_job_limits(self,platform,job=None): - jobs_list = self._jobs_list - # Submitted + Queuing Jobs for specific Platform - queuing_jobs = jobs_list.get_queuing(platform) - # We now consider the running jobs count - running_jobs = jobs_list.get_running(platform) - running_by_id = dict() - for running_job in running_jobs: - running_by_id[running_job.id] = running_job - running_jobs_len = len(running_by_id.keys()) - - queued_by_id = dict() - for queued_job in queuing_jobs: - queued_by_id[queued_job.id] = queued_job - queuing_jobs_len = len(list(queued_by_id.keys())) - - submitted_jobs = jobs_list.get_submitted(platform) - submitted_by_id = dict() - for submitted_job in submitted_jobs: - submitted_by_id[submitted_job.id] = submitted_job - submitted_jobs_len = len(list(submitted_by_id.keys())) - - waiting_jobs = submitted_jobs_len + queuing_jobs_len - # Calculate available space in Platform Queue - if job is not None and job.max_waiting_jobs and platform.max_waiting_jobs and int(job.max_waiting_jobs) != int(platform.max_waiting_jobs): - self._max_wait_jobs_to_submit = int(job.max_waiting_jobs) - int(waiting_jobs) - else: - self._max_wait_jobs_to_submit = int(platform.max_waiting_jobs) - int(waiting_jobs) - # .total_jobs is defined in each section of platforms_.yml, if not from there, it comes form autosubmit_.yml - # .total_jobs Maximum number of jobs at the same time - if job is not None and job.total_jobs != platform.total_jobs: - self._max_jobs_to_submit = job.total_jobs - queuing_jobs_len - else: - self._max_jobs_to_submit = platform.total_jobs - queuing_jobs_len - # Subtracting running jobs - self._max_jobs_to_submit = self._max_jobs_to_submit - running_jobs_len - self._max_jobs_to_submit = self._max_jobs_to_submit if self._max_jobs_to_submit > 0 else 0 - self.max_jobs = min(self._max_wait_jobs_to_submit,self._max_jobs_to_submit) def __init__(self, as_config, platform, jobs_list, hold=False): self.current_wrapper_section = "WRAPPERS" @@ -88,6 +49,8 @@ class JobPackager(object): self._jobs_list = jobs_list self._max_wait_jobs_to_submit = 9999999 self.hold = hold + self.max_jobs = None + self._max_jobs_to_submit = None # These are defined in the [wrapper] section of autosubmit_,conf self.wrapper_type = dict() self.wrapper_policy = dict() @@ -159,23 +122,217 @@ class JobPackager(object): highest_completed.append(job) for job in highest_completed: job.distance_weight = job.distance_weight - 1 - def _special_variables(self,job): - special_variables = dict() - if job.section not in self.special_variables: - special_variables[job.section] = dict() - if job.total_jobs != self._platform.total_jobs: - special_variables[job.section]["TOTAL_JOBS"] = job - self.special_variables.update(special_variables) - def build_packages(self): - # type: () -> List[JobPackageBase] + + def calculate_wrapper_bounds(self, section_list): + """ - Returns the list of the built packages to be submitted + Returns the minimum and maximum number of jobs that can be wrapped - :return: List of packages depending on type of package, JobPackageVertical Object for 'vertical'. - :rtype: List() of JobPackageVertical + :param section_list: List of sections to be wrapped + :type section_list: List of strings + :return: Minimum and Maximum number of jobs that can be wrapped + :rtype: Dictionary with keys: min, max, min_v, max_v, min_h, max_h, max_by_section + """ + wrapper_limits = {'min': 1, 'max': 9999999, 'min_v': 1, 'max_v': 9999999, 'min_h': 1, 'max_h': 9999999, 'max_by_section': dict()} + + # Calculate the min and max based in the wrapper_section wrappers: min_wrapped:2, max_wrapped: 2 { wrapper_section: {min_wrapped: 6, max_wrapped: 6} } + wrapper_data = self._as_config.experiment_data.get("WRAPPERS",{}) + current_wrapper_data = wrapper_data.get(self.current_wrapper_section,{}) + if len(self._jobs_list.jobs_to_run_first) == 0: + wrapper_limits['min'] = int(current_wrapper_data.get("MIN_WRAPPED", wrapper_data.get("MIN_WRAPPED", 2))) + wrapper_limits['max'] = int(current_wrapper_data.get("MAX_WRAPPED", wrapper_data.get("MAX_WRAPPED", 9999999))) + wrapper_limits['min_v'] = int(current_wrapper_data.get("MIN_WRAPPED_V", wrapper_data.get("MIN_WRAPPED_V", 1))) + wrapper_limits['max_v'] = int(current_wrapper_data.get("MAX_WRAPPED_V", wrapper_data.get("MAX_WRAPPED_V", 1))) + wrapper_limits['min_h'] = int(current_wrapper_data.get("MIN_WRAPPED_H", wrapper_data.get("MIN_WRAPPED_H", 1))) + wrapper_limits['max_h'] = int(current_wrapper_data.get("MAX_WRAPPED_H", wrapper_data.get("MAX_WRAPPED_H", 1))) + # Max and min calculations + if wrapper_limits['max'] < wrapper_limits['max_v'] * wrapper_limits['max_h']: + wrapper_limits['max'] = wrapper_limits['max_v'] * wrapper_limits['max_h'] + if wrapper_limits['min'] < wrapper_limits['min_v'] * wrapper_limits['min_h']: + wrapper_limits['min'] = max(wrapper_limits['min_v'],wrapper_limits['min_h']) + # if one dimensional wrapper or value is the default + if wrapper_limits['max_v'] == 1 or current_wrapper_data.get("TYPE", "") == "vertical": + wrapper_limits['max_v'] = wrapper_limits['max'] + + if wrapper_limits['max_h'] == 1 or current_wrapper_data.get("TYPE", "") == "horizontal": + wrapper_limits['max_h'] = wrapper_limits['max'] + + if wrapper_limits['min_v'] == 1 and current_wrapper_data.get("TYPE", "") == "vertical": + wrapper_limits['min_v'] = wrapper_limits['min'] + + if wrapper_limits['min_h'] == 1 and current_wrapper_data.get("TYPE", "") == "horizontal": + wrapper_limits['min_h'] = wrapper_limits['min'] + + + # Calculate the max by section by looking at jobs_data[section].max_wrapped + for section in section_list: + wrapper_limits['max_by_section'][section] = self._as_config.jobs_data.get(section,{}).get("MAX_WRAPPED",wrapper_limits['max']) + + return wrapper_limits + + + def check_jobs_to_run_first(self, package): + """ + Check if the jobs to run first are in the package + :param package: + :return: + """ + run_first = False + if len(self._jobs_list.jobs_to_run_first) > 0: + for job in package.jobs[:]: + job.wrapper_type = package.wrapper_type + if job in self._jobs_list.jobs_to_run_first: + job.packed = False + run_first = True + else: + job.packed = False + package.jobs.remove(job) + if self.wrapper_type[self.current_wrapper_section] not in ["horizontal", "vertical", "vertical-mixed"]: + for seq in range(0, len(package.jobs_lists)): + with suppress(ValueError): + package.jobs_lists[seq].remove(job) + return package, run_first + + def check_real_package_wrapper_limits(self,package): + balanced = True + if self.wrapper_type[self.current_wrapper_section] == 'vertical-horizontal': + i = 0 + min_h = len(package.jobs_lists) + min_v = len(package.jobs_lists[0]) + for list_of_jobs in package.jobs_lists[1:-1]: + min_v = min(min_v, len(list_of_jobs)) + for list_of_jobs in package.jobs_lists[:]: + i = i + 1 + if min_v != len(list_of_jobs) and i < len(package.jobs_lists): + balanced = False + elif self.wrapper_type[self.current_wrapper_section] == 'horizontal-vertical': + min_v = len(package.jobs_lists) + min_h = len(package.jobs_lists[0]) + i = 0 + for list_of_jobs in package.jobs_lists[1:-1]: + min_h = min(min_h, len(list_of_jobs)) + for list_of_jobs in package.jobs_lists[:]: + i = i + 1 + if min_h != len(list_of_jobs) and i < len(package.jobs_lists): + balanced = False + elif self.wrapper_type[self.current_wrapper_section] == 'horizontal': + min_h = len(package.jobs) + min_v = 1 + elif self.wrapper_type[self.current_wrapper_section] == 'vertical': + min_v = len(package.jobs) + min_h = 1 + else: + min_v = len(package.jobs) + min_h = len(package.jobs) + return min_v, min_h, balanced + + def check_packages_respect_wrapper_policy(self,built_packages_tmp,packages_to_submit,max_jobs_to_submit,wrapper_limits, any_simple_packages = False): + """ + Check if the packages respect the wrapper policy and act in base of it ( submit wrapper, submit sequential, wait for more jobs to form a wrapper) + :param built_packages_tmp: List of packages to be submitted + :param packages_to_submit: List of packages to be submitted + :param max_jobs_to_submit: Maximum number of jobs to submit + :param wrapper_limits: Dictionary with keys: min, max, min_v, max_v, min_h, max_h, max_by_section + :return: packages_to_submit, max_jobs_to_submit + :rtype: List of packages to be submitted, int + :return: packages_to_submit, max_jobs_to_submit + """ + not_wrappeable_package_info = list() + for p in built_packages_tmp: + if max_jobs_to_submit == 0: + break + failed_innerjobs = False + # Check if the user is using the option to run first some jobs. if so, remove non-first jobs from the package and submit them sequentially following a flexible policy + if len(self._jobs_list.jobs_to_run_first) > 0: + p,run_first = self.check_jobs_to_run_first(p) + if run_first: + for job in p.jobs: + if max_jobs_to_submit == 0: + break + job.packed = False + if job.status == Status.READY: + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped([job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) + max_jobs_to_submit = max_jobs_to_submit - 1 + continue + for job in p.jobs: + if job.fail_count > 0: + failed_innerjobs = True + break + min_v, min_h, balanced = self.check_real_package_wrapper_limits(p) + # if the quantity is enough, make the wrapper + if len(p.jobs) >= wrapper_limits["min"] and min_v >= wrapper_limits["min_v"] and min_h >= wrapper_limits["min_h"] and not failed_innerjobs: + for job in p.jobs: + job.packed = True + packages_to_submit.append(p) + max_jobs_to_submit = max_jobs_to_submit - 1 + else: + not_wrappeable_package_info.append([p, min_v, min_h, balanced]) + # It is a deadlock when: + # 1. There are no more non-wrapped jobs in ready status + # 2. And there are no more jobs in the queue ( submitted, queuing, running, held ) + # 3. And all current packages are not wrappable. + if not any_simple_packages and len(self._jobs_list.get_in_queue()) == 0 and len(not_wrappeable_package_info) == len(built_packages_tmp): + for p, min_v, min_h, balanced in not_wrappeable_package_info: + if self.wrapper_policy[self.current_wrapper_section] == "strict": + for job in p.jobs: + job.packed = False + raise AutosubmitCritical(self.error_message_policy(min_h, min_v, wrapper_limits, p.wallclock, balanced), 7014) + elif self.wrapper_policy[self.current_wrapper_section] == "mixed": + error = True + for job in p.jobs: + if max_jobs_to_submit == 0: + break + if job.fail_count > 0 and job.status == Status.READY: + job.packed = False + Log.printlog( + "Wrapper policy is set to mixed, there is a failed job that will be sent sequential") + error = False + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped( + [job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) + max_jobs_to_submit = max_jobs_to_submit - 1 + if error: + if len(self._jobs_list.get_in_queue()) == 0: # When there are not more possible jobs, autosubmit will stop the execution + raise AutosubmitCritical(self.error_message_policy(min_h, min_v, wrapper_limits, p.wallclock, balanced), 7014) + else: + Log.info( + "Wrapper policy is set to flexible and there is a deadlock, Autosubmit will submit the jobs sequentially") + for job in p.jobs: + if max_jobs_to_submit == 0: + break + job.packed = False + if job.status == Status.READY: + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped( + [job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) + max_jobs_to_submit = max_jobs_to_submit - 1 + return packages_to_submit, max_jobs_to_submit + + def error_message_policy(self,min_h,min_v,wrapper_limits,wallclock_sum,balanced): + message = f"Wrapper couldn't be formed under {self.wrapper_policy[self.current_wrapper_section]} POLICY due minimum limit not being reached: [wrappable:{wrapper_limits['min']} < defined_min:{min_h*min_v}] [wrappable_h:{min_h} < defined_min_h:{wrapper_limits['min_h']}]|[wrappeable_v:{min_v} < defined_min_v:{wrapper_limits['min_v']}] " + if min_v > 1: + message += f"\nCheck your configuration: Check if current {wallclock_sum} vertical wallclock has reached the max defined on platforms.conf." + else: + message += "\nCheck your configuration: Only jobs_in_wrappers are active, check your jobs_in_wrapper dependencies." + if not balanced: + message += "\nPackages are not well balanced! (This is not the main cause of the Critical error)" + return message + + def check_if_packages_are_ready_to_build(self): + """ + Check if the packages are ready to be built + :return: List of jobs ready to be built, boolean indicating if packages can't be built for other reasons ( max_total_jobs...) """ - packages_to_submit = list() - # only_wrappers = False when coming from Autosubmit.submit_ready_jobs, jobs_filtered empty jobs_ready = list() if len(self._jobs_list.jobs_to_run_first) > 0: jobs_ready = [job for job in self._jobs_list.jobs_to_run_first if @@ -214,339 +371,124 @@ class JobPackager(object): pass if len(jobs_ready) == 0: # If there are no jobs ready, result is tuple of empty - return packages_to_submit + return jobs_ready,False #check if there are jobs listed on calculate_job_limits - for job in jobs_ready: - self._special_variables(job) - if len(self.special_variables) > 0: - for section in self.special_variables: - if "TOTAL_JOBS" in self.special_variables[section]: - self.calculate_job_limits(self._platform,self.special_variables[section]["TOTAL_JOBS"]) - if not (self._max_wait_jobs_to_submit > 0 and self._max_jobs_to_submit > 0): - # If there is no more space in platform, result is tuple of empty - Log.debug("No more space in platform {0} for jobs {1}".format(self._platform.name, - [job.name for job in jobs_ready])) - return packages_to_submit - self.calculate_job_limits(self._platform) + self.calculate_job_limits(self._platform) + if not (self._max_wait_jobs_to_submit > 0 and self._max_jobs_to_submit > 0): + # If there is no more space in platform, result is tuple of empty + Log.debug('Max jobs to submit reached, waiting for more space in platform {0}'.format(self._platform.name)) + return jobs_ready,False + return jobs_ready,True + + def calculate_job_limits(self,platform,job=None): + jobs_list = self._jobs_list + # Submitted + Queuing Jobs for specific Platform + queuing_jobs = jobs_list.get_queuing(platform) + # We now consider the running jobs count + running_jobs = jobs_list.get_running(platform) + running_by_id = dict() + for running_job in running_jobs: + running_by_id[running_job.id] = running_job + running_jobs_len = len(running_by_id.keys()) + queued_by_id = dict() + for queued_job in queuing_jobs: + queued_by_id[queued_job.id] = queued_job + queuing_jobs_len = len(list(queued_by_id.keys())) + + submitted_jobs = jobs_list.get_submitted(platform) + submitted_by_id = dict() + for submitted_job in submitted_jobs: + submitted_by_id[submitted_job.id] = submitted_job + submitted_jobs_len = len(list(submitted_by_id.keys())) + + waiting_jobs = submitted_jobs_len + queuing_jobs_len + # Calculate available space in Platform Queue + if job is not None and job.max_waiting_jobs and platform.max_waiting_jobs and int(job.max_waiting_jobs) != int(platform.max_waiting_jobs): + self._max_wait_jobs_to_submit = int(job.max_waiting_jobs) - int(waiting_jobs) + else: + self._max_wait_jobs_to_submit = int(platform.max_waiting_jobs) - int(waiting_jobs) + # .total_jobs is defined in each section of platforms_.yml, if not from there, it comes form autosubmit_.yml + # .total_jobs Maximum number of jobs at the same time + if job is not None and job.total_jobs != platform.total_jobs: + self._max_jobs_to_submit = job.total_jobs - queuing_jobs_len else: - self.calculate_job_limits(self._platform) - if not (self._max_wait_jobs_to_submit > 0 and self._max_jobs_to_submit > 0): - # If there is no more space in platform, result is tuple of empty - Log.debug("No more space in platform {0} for jobs {1}".format(self._platform.name, [job.name for job in jobs_ready])) - return packages_to_submit - - - # Sort by 6 first digits of date - available_sorted = sorted( - jobs_ready, key=lambda k: k.long_name.split('_')[1][:6]) - # Sort by Priority, the highest first - list_of_available = sorted( - available_sorted, key=lambda k: k.priority, reverse=True) - num_jobs_to_submit = min(self._max_wait_jobs_to_submit, len(jobs_ready), self._max_jobs_to_submit) - # Take the first num_jobs_to_submit from the list of available - jobs_to_submit_tmp = list_of_available[0:num_jobs_to_submit] - #jobs_to_submit = [ - # fresh_job for fresh_job in jobs_to_submit_tmp if fresh_job.fail_count == 0] - jobs_to_submit = [fresh_job for fresh_job in jobs_to_submit_tmp] - failed_wrapped_jobs = [failed_job for failed_job in jobs_to_submit_tmp if failed_job.fail_count > 0] - for job in failed_wrapped_jobs: + self._max_jobs_to_submit = platform.total_jobs - queuing_jobs_len + # Subtracting running jobs + self._max_jobs_to_submit = self._max_jobs_to_submit - running_jobs_len + self._max_jobs_to_submit = self._max_jobs_to_submit if self._max_jobs_to_submit > 0 else 0 + self.max_jobs = min(self._max_wait_jobs_to_submit,self._max_jobs_to_submit) + + def build_packages(self): + # type: () -> List[JobPackageBase] + """ + Returns the list of the built packages to be submitted + + :return: List of packages depending on type of package, JobPackageVertical Object for 'vertical'. + :rtype: List() of JobPackageVertical + """ + packages_to_submit = list() + jobs_ready,ready = self.check_if_packages_are_ready_to_build() + if not ready: + return [] + max_jobs_to_submit = min(self._max_wait_jobs_to_submit, self._max_jobs_to_submit) + jobs_to_submit = sorted( + jobs_ready, key=lambda k: k.priority, reverse=True) + for job in [failed_job for failed_job in jobs_to_submit if failed_job.fail_count > 0]: job.packed = False - jobs_to_submit_by_section = self._divide_list_by_section(jobs_to_submit) - # create wrapped package jobs Wrapper building starts here - for wrapper_name,section_jobs in jobs_to_submit_by_section.items(): + jobs_to_wrap = self._divide_list_by_section(jobs_to_submit) + non_wrapped_jobs = jobs_to_wrap.pop("SIMPLE",[]) + if len(non_wrapped_jobs) > 0: + any_simple_packages = True + else: + any_simple_packages = False + # Prepare packages for wrapped jobs + for wrapper_name, jobs in jobs_to_wrap.items(): + if max_jobs_to_submit == 0: + break self.current_wrapper_section = wrapper_name - for section,jobs in section_jobs.items(): - if len(jobs) > 0: - if self.current_wrapper_section != "SIMPLE" and not self._platform.allow_wrappers: - Log.warning("Platform {0} does not allow wrappers, submitting jobs individually".format(self._platform.name)) - if wrapper_name != "SIMPLE" and self._platform.allow_wrappers and self.wrapper_type[self.current_wrapper_section] in ['horizontal', 'vertical','vertical-horizontal', 'horizontal-vertical'] : - # Trying to find the value in jobs_parser, if not, default to an autosubmit_.yml value (Looks first in [wrapper] section) - wrapper_limits = dict() - wrapper_limits["max_by_section"] = dict() - wrapper_limits["max"] = int(self._as_config.get_max_wrapped_jobs(self._as_config.experiment_data["WRAPPERS"][self.current_wrapper_section])) - wrapper_limits["max_v"] = int(self._as_config.get_max_wrapped_jobs_vertical(self._as_config.experiment_data["WRAPPERS"][self.current_wrapper_section])) - wrapper_limits["max_h"] = int(self._as_config.get_max_wrapped_jobs_horizontal(self._as_config.experiment_data["WRAPPERS"][self.current_wrapper_section])) - if wrapper_limits["max"] < wrapper_limits["max_v"] * wrapper_limits["max_h"]: - wrapper_limits["max"] = wrapper_limits["max_v"] * wrapper_limits["max_h"] - if wrapper_limits["max_v"] == -1: - wrapper_limits["max_v"] = wrapper_limits["max"] - if wrapper_limits["max_h"] == -1: - wrapper_limits["max_h"] = wrapper_limits["max"] - if '&' not in section: - dependencies_keys = self._as_config.jobs_data[section].get('DEPENDENCIES', "") - wrapper_limits["max_by_section"][section] = wrapper_limits["max"] - wrapper_limits["min"] = min(self._as_config.jobs_data[section].get( - "MIN_WRAPPED", 99999999), 0) - else: - multiple_sections = section.split('&') - dependencies_keys = [] - min_value = int(self._as_config.get_min_wrapped_jobs(self._as_config.experiment_data["WRAPPERS"][self.current_wrapper_section])) - for sectionN in multiple_sections: - if self._as_config.jobs_data[sectionN].get('DEPENDENCIES',"") != "": - dependencies_keys += self._as_config.jobs_data.get("DEPENDENCIES", "").split() - if self._as_config.jobs_data[sectionN].get('MAX_WRAPPED',None) is not None and len(str(self._as_config.jobs_data[sectionN].get('MAX_WRAPPED',None))) > 0: - wrapper_limits["max_by_section"][sectionN] = int(self._as_config.jobs_data[sectionN].get("MAX_WRAPPED")) - else: - wrapper_limits["max_by_section"][sectionN] = wrapper_limits["max"] - wrapper_limits["min"] = min(self._as_config.jobs_data[sectionN].get("MIN_WRAPPED",min_value),min_value) - hard_limit_wrapper = wrapper_limits["max"] - wrapper_limits["min"] = min(wrapper_limits["min"], hard_limit_wrapper) - wrapper_limits["min_v"] = self._as_config.get_min_wrapped_jobs_vertical(self._as_config.experiment_data["WRAPPERS"][self.current_wrapper_section]) - wrapper_limits["min_h"] = self._as_config.get_min_wrapped_jobs_horizontal(self._as_config.experiment_data["WRAPPERS"][self.current_wrapper_section]) - wrapper_limits["max"] = hard_limit_wrapper - if wrapper_limits["min"] < wrapper_limits["min_v"] * wrapper_limits["min_h"]: - wrapper_limits["min"] = max(wrapper_limits["min_v"],wrapper_limits["min_h"]) - if len(self._jobs_list.jobs_to_run_first) > 0: - wrapper_limits["min"] = 2 - current_info = list() - for param in self.wrapper_info: - current_info.append(param[self.current_wrapper_section]) - if self.wrapper_type[self.current_wrapper_section] == 'vertical': - built_packages_tmp = self._build_vertical_packages(jobs, wrapper_limits,wrapper_info=current_info) - elif self.wrapper_type[self.current_wrapper_section] == 'horizontal': - built_packages_tmp = self._build_horizontal_packages(jobs, wrapper_limits, section,wrapper_info=current_info) - elif self.wrapper_type[self.current_wrapper_section] in ['vertical-horizontal', 'horizontal-vertical']: - built_packages_tmp = list() - built_packages_tmp.append(self._build_hybrid_package(jobs, wrapper_limits, section,wrapper_info=current_info)) - else: - built_packages_tmp = self._build_vertical_packages(jobs, wrapper_limits) - - for p in built_packages_tmp: - infinite_deadlock = False # This will raise an autosubmit critical if true - failed_innerjobs = False - job_has_to_run_first = False - aux_jobs = [] - # Check failed jobs first - for job in p.jobs: - job.wrapper_type = p.wrapper_type - if len(self._jobs_list.jobs_to_run_first) > 0: - if job not in self._jobs_list.jobs_to_run_first: - job.packed = False - aux_jobs.append(job) - if job.fail_count > 0: - failed_innerjobs = True - if len(self._jobs_list.jobs_to_run_first) > 0: - job_has_to_run_first = True - for job in aux_jobs: - job.packed = False - p.jobs.remove(job) - if self.wrapper_type[self.current_wrapper_section] != "horizontal" and self.wrapper_type[self.current_wrapper_section] != "vertical" and self.wrapper_type[self.current_wrapper_section] != "vertical-mixed": - for seq in range(0,len(p.jobs_lists)): - try: - p.jobs_lists[seq].remove(job) - except Exception as e: - pass - if self.wrapper_type[self.current_wrapper_section] != "horizontal" and self.wrapper_type[self.current_wrapper_section] != "vertical" and self.wrapper_type[self.current_wrapper_section] != "vertical-mixed": - aux = p.jobs_lists - p.jobs_lists = [] - for seq in range(0,len(aux)): - if len(aux[seq]) > 0: - p.jobs_lists.append(aux[seq]) - if len(p.jobs) > 0: - balanced = True - if self.wrapper_type[self.current_wrapper_section] == 'vertical-horizontal': - min_h = len(p.jobs_lists) - min_v = len(p.jobs_lists[0]) - for list_of_jobs in p.jobs_lists[1:-1]: - min_v = min(min_v, len(list_of_jobs)) - - elif self.wrapper_type[self.current_wrapper_section] == 'horizontal-vertical': - min_v = len(p.jobs_lists) - min_h = len(p.jobs_lists[0]) - i = 0 - for list_of_jobs in p.jobs_lists[1:-1]: - min_h = min(min_h, len(list_of_jobs)) - for list_of_jobs in p.jobs_lists[:]: - i = i+1 - if min_h != len(list_of_jobs) and i < len(p.jobs_lists): - balanced = False - elif min_h != len(list_of_jobs) and i == len(p.jobs_lists): - if balanced: - for job in list_of_jobs: - job.packed = False - p.jobs.remove(job) - package = JobPackageSimple([job]) - packages_to_submit.append(package) - p.jobs_lists = p.jobs_lists[:-1] - - - - elif self.wrapper_type[self.current_wrapper_section] == 'horizontal': - min_h = len(p.jobs) - min_v = 1 - elif self.wrapper_type[self.current_wrapper_section] == 'vertical': - min_v = len(p.jobs) - min_h = 1 - else: - min_v = len(p.jobs) - min_h = len(p.jobs) - # if the quantity is enough, make the wrapper - - if (len(p.jobs) >= wrapper_limits["min"] and min_v >= wrapper_limits["min_v"] and min_h >= wrapper_limits["min_h"] and (not failed_innerjobs or self.wrapper_policy[self.current_wrapper_section] not in ["mixed","strict"] ) ) or job_has_to_run_first: - for job in p.jobs: - job.packed = True - packages_to_submit.append(p) - else: - deadlock = True - if deadlock: # Remaining jobs if chunk is the last one - for job in p.jobs: - if ( job.running == "chunk" and job.chunk == int(job.parameters["EXPERIMENT.NUMCHUNKS"]) ) and balanced: - deadlock = False - break - if not deadlock: # Submit package if deadlock has been liberated - for job in p.jobs: - job.packed = True - packages_to_submit.append(p) - else: - wallclock_sum = p.jobs[0].wallclock - for seq in range(1, min_v): - wallclock_sum = sum_str_hours(wallclock_sum, p.jobs[0].wallclock) - next_wrappable_jobs = self._jobs_list.get_jobs_by_section(self.jobs_in_wrapper[self.current_wrapper_section]) - next_wrappable_jobs = [job for job in next_wrappable_jobs if job.status == Status.WAITING and job not in p.jobs ] # Get only waiting jobs - active_jobs = list() - aux_active_jobs = list() - for job in next_wrappable_jobs: # Prone tree by looking only the closest children - direct_children = False - for related in job.parents: - if related in p.jobs: - direct_children = True - break - if direct_children: # Get parent of direct children that aren't in wrapper - aux_active_jobs += [aux_parent for aux_parent in job.parents if ( aux_parent.status != Status.COMPLETED and aux_parent.status != Status.FAILED) and ( aux_parent.section not in self.jobs_in_wrapper[self.current_wrapper_section] or ( aux_parent.section in self.jobs_in_wrapper[self.current_wrapper_section] and aux_parent.status != Status.COMPLETED and aux_parent.status != Status.FAILED and aux_parent.status != Status.WAITING and aux_parent.status != Status.READY ) ) ] - aux_active_jobs = list(set(aux_active_jobs)) - track = [] # Tracker to prone tree for avoid the checking of the same parent from different nodes. - active_jobs_names = [ job.name for job in p.jobs ] # We want to search if the actual wrapped jobs needs to run for add more jobs to this wrapper - hard_deadlock = False - for job in aux_active_jobs: - parents_to_check = [] - if job.status == Status.WAITING: # We only want to check uncompleted parents - aux_job = job - for parent in aux_job.parents: # First case - if parent.name in active_jobs_names: - hard_deadlock = True - infinite_deadlock = True - break - if (parent.status == Status.WAITING ) and parent.name != aux_job.name: - parents_to_check.append(parent) - track.extend(parents_to_check) - while len(parents_to_check) > 0 and not infinite_deadlock: # We want to look deeper on the tree until all jobs are completed, or we find an unresolvable deadlock. - aux_job = parents_to_check.pop(0) - for parent in aux_job.parents: - if parent.name in active_jobs_names: - hard_deadlock = True - infinite_deadlock = True - break - if (parent.status == Status.WAITING ) and parent.name != aux_job.name and parent not in track: - parents_to_check.append(parent) - track.extend(parents_to_check) - if not infinite_deadlock: - active_jobs.append(job) # List of jobs that can continue to run without run this wrapper - # Act in base of active_jobs and Policies - if self.wrapper_policy[self.current_wrapper_section] == "strict": - error = True - for job in p.jobs: - job.packed = False - if job in self._jobs_list.jobs_to_run_first: - error = False - if job.status == Status.READY: - if job.type == Type.PYTHON and not self._platform.allow_python_jobs: - package = JobPackageSimpleWrapped( - [job]) - else: - package = JobPackageSimple([job]) - packages_to_submit.append(package) - if error: - if len(active_jobs) > 0: - Log.printlog( - "Wrapper policy is set to MIXED and there are not enough jobs to form a wrapper.[wrappable:{4} <= defined_min:{5}] [wrappeable_h:{0} <= defined_min_h:{1}]|[wrappeable_v:{2} <= defined_min_v:{3}] waiting until the wrapper can be formed.\nIf all values are <=, some innerjob has failed under strict policy".format( - min_h, wrapper_limits["min_h"], min_v, - wrapper_limits["min_v"], wrapper_limits["min"], len(active_jobs)), - 6013) - else: - message = "Wrapper couldn't be formed under {0} POLICY due minimum limit not being reached: [wrappable:{4} < defined_min:{5}] [wrappable_h:{1} < defined_min_h:{2}]|[wrappeable_v:{3} < defined_min_v:{4}] ".format( - self.wrapper_policy[self.current_wrapper_section], min_h, - wrapper_limits["min_h"], min_v, wrapper_limits["min_v"], - wrapper_limits["min"], len(active_jobs)) - if hard_deadlock: - message += "\nCheck your configuration: The next wrappable job can't be wrapped until some of inner jobs of current packages finishes which is imposible" - if min_v > 1: - message += "\nCheck your configuration: Check if current {0} vertical wallclock has reached the max defined on platforms.conf.".format(wallclock_sum) - else: - message += "\nCheck your configuration: Only jobs_in_wrappers are active, check their dependencies." - if not balanced: - message += "\nPackages are not well balanced: Check your dependencies(This is not the main cause of the Critical error)" - if len(self._jobs_list.get_in_queue()) == 0: - raise AutosubmitCritical(message, 7014) - elif self.wrapper_policy[self.current_wrapper_section] == "mixed": - error = True - show_log = True - for job in p.jobs: - if job in self._jobs_list.jobs_to_run_first: - job.packed = False - error = False - if job.status == Status.READY: - if job.type == Type.PYTHON and not self._platform.allow_python_jobs: - package = JobPackageSimpleWrapped( - [job]) - else: - package = JobPackageSimple([job]) - packages_to_submit.append(package) - if job.fail_count > 0 and job.status == Status.READY: - job.packed = False - Log.printlog( - "Wrapper policy is set to mixed, there is a failed job that will be sent sequential") - error = False - show_log = False - if job.type == Type.PYTHON and not self._platform.allow_python_jobs: - package = JobPackageSimpleWrapped( - [job]) - else: - package = JobPackageSimple([job]) - packages_to_submit.append(package) - if error: - if len(active_jobs) > 0: - if show_log: - Log.printlog( - "Wrapper policy is set to MIXED and there are not enough jobs to form a wrapper.[wrappable:{4} < defined_min:{5}] [wrappable_h:{0} < defined_min_h:{1}]|[wrappeable_v:{2} < defined_min_v:{3}] waiting until the wrapper can be formed.".format( - min_h, wrapper_limits["min_h"], min_v, - wrapper_limits["min_v"],wrapper_limits["min"],len(active_jobs)), 6013) - else: - message = "Wrapper couldn't be formed under {0} POLICY due minimum limit not being reached: [wrappable:{4} < defined_min:{5}] [wrappable_h:{1} < defined_min_h:{2}]|[wrappeable_v:{3} < defined_min_v:{4}] ".format( - self.wrapper_policy[self.current_wrapper_section], min_h, - wrapper_limits["min_h"], min_v, wrapper_limits["min_v"],wrapper_limits["min"],len(active_jobs)) - if hard_deadlock: - message += "\nCheck your configuration: The next wrappable job can't be wrapped until some of inner jobs of current packages finishes which is impossible" - if min_v > 1: - message += "\nCheck your configuration: Check if current {0} vertical wallclock has reached the max defined on platforms.conf.".format( - wallclock_sum) - else: - message += "\nCheck your configuration: Only jobs_in_wrappers are active, check your jobs_in_wrapper dependencies." - if not balanced: - message += "\nPackages are not well balanced! (This is not the main cause of the Critical error)" - - if len(self._jobs_list.get_in_queue()) == 0: # When there are not more possible jobs, autosubmit will stop the execution - raise AutosubmitCritical(message, 7014) - else: - for job in p.jobs: - job.packed = False - if job.status == Status.READY: - if job.type == Type.PYTHON and not self._platform.allow_python_jobs: - package = JobPackageSimpleWrapped( - [job]) - else: - package = JobPackageSimple([job]) - packages_to_submit.append(package) - Log.info("Wrapper policy is set to flexible and there is a deadlock, Autosubmit will submit the jobs sequentially") - else: - for job in jobs: - job.packed = False - if job.type == Type.PYTHON and not self._platform.allow_python_jobs: - package = JobPackageSimpleWrapped([job]) - else: - package = JobPackageSimple([job]) - packages_to_submit.append(package) + section = self._as_config.experiment_data.get("WRAPPERS",{}).get(self.current_wrapper_section,{}).get("JOBS_IN_WRAPPER", "") + if not self._platform.allow_wrappers and self.wrapper_type[self.current_wrapper_section] in ['horizontal', 'vertical','vertical-horizontal', 'horizontal-vertical']: + Log.warning( + "Platform {0} does not allow wrappers, submitting jobs individually".format(self._platform.name)) + for job in jobs: + non_wrapped_jobs.append(job) + continue + if "&" in section: + section_list = section.split("&") + elif "," in section: + section_list = section.split(",") + else: + section_list = section.split(" ") + wrapper_limits = self.calculate_wrapper_bounds(section_list) + current_info = list() + built_packages_tmp = list() + for param in self.wrapper_info: + current_info.append(param[self.current_wrapper_section]) + if self.wrapper_type[self.current_wrapper_section] == 'vertical': + built_packages_tmp = self._build_vertical_packages(jobs, wrapper_limits,wrapper_info=current_info) + elif self.wrapper_type[self.current_wrapper_section] == 'horizontal': + built_packages_tmp = self._build_horizontal_packages(jobs, wrapper_limits, section,wrapper_info=current_info) + elif self.wrapper_type[self.current_wrapper_section] in ['vertical-horizontal', 'horizontal-vertical']: + built_packages_tmp.append(self._build_hybrid_package(jobs, wrapper_limits, section,wrapper_info=current_info)) + else: + built_packages_tmp = self._build_vertical_packages(jobs, wrapper_limits) + packages_to_submit,max_jobs_to_submit = self.check_packages_respect_wrapper_policy(built_packages_tmp,packages_to_submit,max_jobs_to_submit,wrapper_limits,any_simple_packages) + # Now, prepare the packages for non-wrapper jobs + for job in non_wrapped_jobs: + if max_jobs_to_submit == 0: + break + if len(self._jobs_list.jobs_to_run_first) > 0: # if user wants to run first some jobs, submit them first + if job not in self._jobs_list.jobs_to_run_first: + continue + job.packed = False + if job.type == Type.PYTHON and not self._platform.allow_python_jobs: + package = JobPackageSimpleWrapped([job]) + else: + package = JobPackageSimple([job]) + packages_to_submit.append(package) + max_jobs_to_submit = max_jobs_to_submit - 1 for package in packages_to_submit: self.max_jobs = self.max_jobs - 1 @@ -572,21 +514,20 @@ class JobPackager(object): section_name += section+"&" section_name = section_name[:-1] sections_split[wrapper_name] = section_name - jobs_by_section[wrapper_name] = dict() - jobs_by_section[wrapper_name][section_name] = list() + jobs_by_section[wrapper_name] = list() - jobs_by_section["SIMPLE"] = collections.defaultdict(list) - remaining_jobs = copy.copy(jobs_list) + jobs_by_section["SIMPLE"] = [] for wrapper_name,section_name in sections_split.items(): - for job in jobs_list: + for job in jobs_list[:]: if job.section.upper() in section_name.split("&"): - jobs_by_section[wrapper_name][section_name].append(job) - try: - remaining_jobs.remove(job) - except ValueError: - pass - for job in remaining_jobs: - jobs_by_section["SIMPLE"][job.section].append(job) + jobs_by_section[wrapper_name].append(job) + jobs_list.remove(job) + # jobs not in wrapper + for job in (job for job in jobs_list): + jobs_by_section["SIMPLE"].append(job) + for wrappers in list(jobs_by_section.keys()): + if len(jobs_by_section[wrappers]) == 0: + del jobs_by_section[wrappers] return jobs_by_section diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index ebdbf3d7c4e5d005d3159f619f0592ce76cfd789..86e790791018773792c71f983cee5bec512d721c 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -112,9 +112,6 @@ class JobPackageBase(object): Log.warning("On submission script has some empty variables") else: Log.result("Script {0} OK", job.name) - lock.acquire() - job.update_parameters(configuration, parameters) - lock.release() # looking for directives on jobs self._custom_directives = self._custom_directives | set(job.custom_directives) @threaded @@ -399,12 +396,12 @@ class JobPackageThread(JobPackageBase): # temporal hetjob code , to be upgraded in the future if configuration is not None: self.inner_retrials = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, - {}).get("RETRIALS", - configuration.get_retrials()) + {}).get("RETRIALS",self.jobs[0].retrials) if self.inner_retrials == 0: self.inner_retrials = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, - {}).get("INNER_RETRIALS", - configuration.get_retrials()) + {}).get("INNER_RETRIALS",self.jobs[0].retrials) + for job in self.jobs: + job.retrials = self.inner_retrials self.export = configuration.get_wrapper_export(configuration.experiment_data["WRAPPERS"][self.current_wrapper_section]) if self.export.lower() != "none" and len(self.export) > 0: for job in self.jobs: @@ -746,13 +743,14 @@ class JobPackageVertical(JobPackageThread): return timedelta(**time_params),format_ def _common_script_content(self): if self.jobs[0].wrapper_type == "vertical": - #wallclock = datetime.datetime.strptime(self._wallclock, '%H:%M') wallclock,format_ = self.parse_time() + original_wallclock_to_seconds = wallclock.days * 86400.0 + wallclock.seconds + if format_ == "hour": total = wallclock.days * 24 + wallclock.seconds / 60 / 60 else: total = wallclock.days * 24 + wallclock.seconds / 60 - total = total * 1.15 + if format_ == "hour": hour = int(total ) minute = int((total - int(total)) * 60.0) @@ -766,14 +764,11 @@ class JobPackageVertical(JobPackageThread): wallclock_seconds = wallclock_delta.days * 24 * 60 * 60 + wallclock_delta.seconds wallclock_by_level = wallclock_seconds/(self.jobs[-1].level+1) if self.extensible_wallclock > 0: - original_wallclock_to_seconds = wallclock.days * 86400.0 + wallclock.seconds wallclock_seconds = int(original_wallclock_to_seconds + wallclock_by_level * self.extensible_wallclock) wallclock_delta = datetime.timedelta(hours=0, minutes=0, seconds=wallclock_seconds) - total = wallclock.days * 24 + wallclock.seconds / 60 / 60 + total = wallclock_delta.days * 24 + wallclock_delta.seconds / 60 / 60 hh = int(total) mm = int((total - int(total)) * 60.0) - ss = int(((total - int(total)) * 60 - - int((total - int(total)) * 60.0)) * 60.0) if hh < 10: hh_str='0'+str(hh) else: diff --git a/autosubmit/job/job_utils.py b/autosubmit/job/job_utils.py index 9782122738093e02d00be3c1df3aedd8f3840247..c02a92952778361cbb50b599e9c568316914fafb 100644 --- a/autosubmit/job/job_utils.py +++ b/autosubmit/job/job_utils.py @@ -17,33 +17,31 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import networkx import os - -from networkx.algorithms.dag import is_directed_acyclic_graph -from networkx import DiGraph -from networkx import dfs_edges -from networkx import NetworkXError from autosubmit.job.job_package_persistence import JobPackagePersistence from autosubmitconfigparser.config.basicconfig import BasicConfig from typing import Dict def transitive_reduction(graph): - try: - return networkx.algorithms.dag.transitive_reduction(graph) - except Exception as exp: - if not is_directed_acyclic_graph(graph): - raise NetworkXError( - "Transitive reduction only uniquely defined on directed acyclic graphs.") - reduced_graph = DiGraph() - reduced_graph.add_nodes_from(graph.nodes()) - for u in graph: - u_edges = set(graph[u]) - for v in graph[u]: - u_edges -= {y for x, y in dfs_edges(graph, v)} - reduced_graph.add_edges_from((u, v) for v in u_edges) - return reduced_graph + """ + + Returns transitive reduction of a directed graph + + The transitive reduction of G = (V,E) is a graph G- = (V,E-) such that + for all v,w in V there is an edge (v,w) in E- if and only if (v,w) is + in E and there is no path from v to w in G with length greater than 1. + + :param graph: A directed acyclic graph (DAG) + :type graph: NetworkX DiGraph + :return: The transitive reduction of G + """ + for u in graph: + graph.nodes[u]["job"].parents = set() + graph.nodes[u]["job"].children = set() + for u in graph: + graph.nodes[u]["job"].add_children([graph.nodes[v]["job"] for v in graph[u]]) + return graph def get_job_package_code(expid, job_name): # type: (str, str) -> int diff --git a/autosubmit/monitor/diagram.py b/autosubmit/monitor/diagram.py index d2408f954b4d8317768ae8df0965e4407a1f45dc..661c757cb3e17eedb7a2ad79ab92dcee639c820c 100644 --- a/autosubmit/monitor/diagram.py +++ b/autosubmit/monitor/diagram.py @@ -90,7 +90,6 @@ def create_bar_diagram(experiment_id, jobs_list, general_stats, output_file, per # Plotting total_plots_count = normal_plots_count + failed_jobs_plots_count # num_plots = norma - # ind = np.arrange(int(MAX_JOBS_PER_PLOT)) width = 0.16 # Creating stats figure + sanity check plot = True diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index f1de4888578fce3b3c09d5ba6eddc6c8af4ebb3f..e1b9bb3b256434becb3868f449471303d69b6779 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -270,11 +270,6 @@ class Monitor: else: return None, None - - - - - def _add_children(self, job, exp, node_job, groups, hide_groups): if job in self.nodes_plotted: return diff --git a/autosubmit/platforms/ecplatform.py b/autosubmit/platforms/ecplatform.py index 3c4110f006fa3ff842e0146593e496db6e98de87..fb880e694e26ec671b61235b2afde0b7e99b11b4 100644 --- a/autosubmit/platforms/ecplatform.py +++ b/autosubmit/platforms/ecplatform.py @@ -95,8 +95,6 @@ class EcPlatform(ParamikoPlatform): self._checkjob_cmd = "ecaccess-job-list " self._checkhost_cmd = "ecaccess-certificate-list" self._checkvalidcert_cmd = "ecaccess-gateway-connected" - self._submit_cmd = ("ecaccess-job-submit -distant -queueName " + self.ec_queue + " " + self.host + ":" + - self.remote_log_dir + "/") self._submit_command_name = "ecaccess-job-submit" self.put_cmd = "ecaccess-file-put" self.get_cmd = "ecaccess-file-get" @@ -115,7 +113,8 @@ class EcPlatform(ParamikoPlatform): def get_mkdir_cmd(self): return self.mkdir_cmd - def set_submit_cmd(self,ec_queue="hpc"): + + def set_submit_cmd(self,ec_queue): self._submit_cmd = ("ecaccess-job-submit -distant -queueName " + ec_queue + " " + self.host + ":" + self.remote_log_dir + "/") @@ -223,22 +222,22 @@ class EcPlatform(ParamikoPlatform): return False def send_command(self, command, ignore_log=False, x11 = False): - lang = locale.getlocale()[1] - if lang is None: - lang = locale.getdefaultlocale()[1] - if lang is None: - lang = 'UTF-8' - try: - output = subprocess.check_output(command, shell=True).decode(lang) - except subprocess.CalledProcessError as e: - if command.find("ecaccess-job-submit") != -1: - raise AutosubmitError("bad parameters. Error submitting job.") - if not ignore_log: - raise AutosubmitError('Could not execute command {0} on {1}'.format(e.cmd, self.host),7500,str(e)) - return False - self._ssh_output = output + lang = locale.getlocale()[1] or locale.getdefaultlocale()[1] or 'UTF-8' + err_message = 'command not executed' + for _ in range(3): + try: + self._ssh_output = subprocess.check_output(command, shell=True).decode(lang) + except Exception as e: + err_message = str(e) + sleep(1) + else: + break + else: # if break was not called in the for, all attemps failed! + raise AutosubmitError(f'Could not execute command {command} on {self.host}', 7500, str(err_message)) return True + + def send_file(self, filename, check=True): self.check_remote_log_dir() self.delete_file(filename) diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index 58582bc0b9e71963111afe547c5ef8ced1975ea9..a759f6d5472c9f79f1e77545f7130ae4d14de0dc 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -513,6 +513,7 @@ class ParamikoPlatform(Platform): x11 = job.x11 cmd = self.get_submit_cmd(script_name, job, hold=hold, export=export) + Log.debug(f"Submitting job with the command: {cmd}") if cmd is None: return None if self.send_command(cmd,x11=x11): diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index ce8c9b358cc3ee1091f33061ca58901ba3f73c1a..c25777b94f496369e511837e9666079d5fe6e7e1 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -183,8 +183,8 @@ class ParamikoSubmitter(Submitter): # Retrieve more configurations settings and save them in the object remote_platform.max_wallclock = platform_data[section].get('MAX_WALLCLOCK',"2:00") remote_platform.max_processors = platform_data[section].get('MAX_PROCESSORS',asconf.get_max_processors()) - remote_platform.max_waiting_jobs = platform_data[section].get('MAX_WAITING_JOBS',asconf.get_max_waiting_jobs()) - remote_platform.total_jobs = platform_data[section].get('TOTAL_JOBS',asconf.get_total_jobs()) + remote_platform.max_waiting_jobs = platform_data[section].get('MAX_WAITING_JOBS',platform_data[section].get('MAXWAITINGJOBS',asconf.get_max_waiting_jobs())) + remote_platform.total_jobs = platform_data[section].get('TOTAL_JOBS',platform_data[section].get('TOTALJOBS',asconf.get_total_jobs())) remote_platform.hyperthreading = str(platform_data[section].get('HYPERTHREADING',False)).lower() remote_platform.project = platform_data[section].get('PROJECT',"") remote_platform.budget = platform_data[section].get('BUDGET', "") diff --git a/autosubmit/platforms/wrappers/wrapper_factory.py b/autosubmit/platforms/wrappers/wrapper_factory.py index a70d8adc89f5c218f7e1c3e442fe01453b336a0c..31c553973d4b744fda56263ed78796b3c168697c 100644 --- a/autosubmit/platforms/wrappers/wrapper_factory.py +++ b/autosubmit/platforms/wrappers/wrapper_factory.py @@ -33,8 +33,8 @@ class WrapperFactory(object): def get_wrapper(self, wrapper_builder, **kwargs): wrapper_data = kwargs['wrapper_data'] wrapper_data.wallclock = kwargs['wallclock'] - #todo here hetjobs - if wrapper_data.het["HETSIZE"] <= 1: + # This was crashing in horizontal, non related to this issue + if wrapper_data.het.get("HETSIZE",0) <= 1: kwargs['allocated_nodes'] = self.allocated_nodes() kwargs['dependency'] = self.dependency(kwargs['dependency']) kwargs['partition'] = self.partition(wrapper_data.partition) diff --git a/autosubmit/statistics/jobs_stat.py b/autosubmit/statistics/jobs_stat.py index b2d1de97b02f51c41555046304d32743d4c106be..8eec5ec65aea75b5187779d0820dbb523b4b0048 100644 --- a/autosubmit/statistics/jobs_stat.py +++ b/autosubmit/statistics/jobs_stat.py @@ -1,27 +1,53 @@ #!/bin/env/python from datetime import datetime, timedelta from .utils import timedelta2hours +from log.log import Log +import math class JobStat(object): - def __init__(self, name, processors, wallclock, section, date, member, chunk): - # type: (str, int, float, str, str, str, str) -> None - self._name = name - self._processors = processors - self._wallclock = wallclock - self.submit_time = None # type: datetime - self.start_time = None # type: datetime - self.finish_time = None # type: datetime - self.completed_queue_time = timedelta() - self.completed_run_time = timedelta() - self.failed_queue_time = timedelta() - self.failed_run_time = timedelta() - self.retrial_count = 0 - self.completed_retrial_count = 0 - self.failed_retrial_count = 0 - self.section = section - self.date = date - self.member = member - self.chunk = chunk + def __init__(self, name, processors, wallclock, section, date, member, chunk, processors_per_node, tasks, nodes, exclusive ): + # type: (str, int, float, str, str, str, str, str, str , str, str) -> None + self._name = name + self._processors = self._calculate_processing_elements(nodes, processors, tasks, processors_per_node, exclusive) + self._wallclock = wallclock + self.submit_time = None # type: datetime + self.start_time = None # type: datetime + self.finish_time = None # type: datetime + self.completed_queue_time = timedelta() + self.completed_run_time = timedelta() + self.failed_queue_time = timedelta() + self.failed_run_time = timedelta() + self.retrial_count = 0 + self.completed_retrial_count = 0 + self.failed_retrial_count = 0 + self.section = section + self.date = date + self.member = member + self.chunk = chunk + + def _estimate_requested_nodes(self,nodes,processors,tasks,processors_per_node) -> int: + if str(nodes).isdigit(): + return int(nodes) + elif str(tasks).isdigit(): + return math.ceil(int(processors) / int(tasks)) + elif str(processors_per_node).isdigit() and int(processors) > int(processors_per_node): + return math.ceil(int(processors) / int(processors_per_node)) + else: + return 1 + + def _calculate_processing_elements(self,nodes,processors,tasks,processors_per_node,exclusive) -> int: + if str(processors_per_node).isdigit(): + if str(nodes).isdigit(): + return int(nodes) * int(processors_per_node) + else: + estimated_nodes = self._estimate_requested_nodes(nodes,processors,tasks,processors_per_node) + if not exclusive and estimated_nodes <= 1 and int(processors) <= int(processors_per_node): + return int(processors) + else: + return estimated_nodes * int(processors_per_node) + elif (str(tasks).isdigit() or str(nodes).isdigit()): + Log.warning(f'Missing PROCESSORS_PER_NODE. Should be set if TASKS or NODES are defined. The PROCESSORS will used instead.') + return int(processors) def inc_retrial_count(self): self.retrial_count += 1 @@ -51,7 +77,7 @@ class JobStat(object): @property def expected_cpu_consumption(self): return self._wallclock * self._processors - + @property def name(self): return self._name diff --git a/autosubmit/statistics/statistics.py b/autosubmit/statistics/statistics.py index 9f759065761fa54ad889cf6214b9cb3c03e89371..0f4037793bc3c4c6fea3d2fa95324951240ab6d2 100644 --- a/autosubmit/statistics/statistics.py +++ b/autosubmit/statistics/statistics.py @@ -47,9 +47,8 @@ class Statistics(object): for index, job in enumerate(self._jobs): retrials = job.get_last_retrials() for retrial in retrials: - print(retrial) job_stat = self._name_to_jobstat_dict.setdefault(job.name, JobStat(job.name, parse_number_processors( - job.processors), job.total_wallclock, job.section, job.date, job.member, job.chunk)) + job.processors), job.total_wallclock, job.section, job.date, job.member, job.chunk, job.processors_per_node, job.tasks, job.nodes, job.exclusive )) job_stat.inc_retrial_count() if Job.is_a_completed_retrial(retrial): job_stat.inc_completed_retrial_count() diff --git a/bin/autosubmit b/bin/autosubmit index d3775fbb1d0179539f8e9de0e400f237fbb7f6b5..4280dc71ef5677d3c4b2b70d2679ed389af56618 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -34,9 +34,11 @@ from autosubmit.autosubmit import Autosubmit # noinspection PyProtectedMember def main(): try: - Autosubmit.parse_args() + return_value = Autosubmit.parse_args() if os.path.exists(os.path.join(Log.file_path, "autosubmit.lock")): os.remove(os.path.join(Log.file_path, "autosubmit.lock")) + if type(return_value) is int: + os._exit(return_value) os._exit(0) except AutosubmitError as e: if os.path.exists(os.path.join(Log.file_path, "autosubmit.lock")): @@ -67,7 +69,7 @@ def main(): else: exception_stream = StringIO() traceback.print_exc(file=exception_stream) - Log.critical("{1}{0}\nUnhandled error: If you see this message, please report it in Autosubmit's GitLab project".format(str(e),exception_stream.getvalue()), 7000) + raise AutosubmitCritical("Unhandled error: If you see this message, please report it in Autosubmit's GitLab project", 7000, str(e)) os._exit(1) diff --git a/docs/source/troubleshooting/changelog.rst b/docs/source/troubleshooting/changelog.rst index 34adb74db5bcbfbc92493e6ed476e1d723159b67..bf5c26f0fd3c7a888e42270d3de6f1a3dbdffe6a 100644 --- a/docs/source/troubleshooting/changelog.rst +++ b/docs/source/troubleshooting/changelog.rst @@ -299,41 +299,102 @@ In order to generate the following jobs: .. code-block:: yaml - POST_20: - FILE: POST.sh - RUNNING: chunk - WALLCLOCK: '00:05' - PROCESSORS: 20 - THREADS: 1 - DEPENDENCIES: SIM_20 POST_20-1 - POST_40: - FILE: POST.sh - RUNNING: chunk - WALLCLOCK: '00:05' - PROCESSORS: 40 - THREADS: 1 - DEPENDENCIES: SIM_40 POST_40-1 - POST_80: - FILE: POST.sh - RUNNING: chunk - WALLCLOCK: '00:05' - PROCESSORS: 80 - THREADS: 1 - DEPENDENCIES: SIM_80 POST_80-1 + experiment: + DATELIST: 19600101 + MEMBERS: "00" + CHUNKSIZEUNIT: day + CHUNKSIZE: '1' + NUMCHUNKS: '2' + CALENDAR: standard + JOBS: + POST_20: + + DEPENDENCIES: + POST_20: + SIM_20: + FILE: POST.sh + PROCESSORS: '20' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + POST_40: + + DEPENDENCIES: + POST_40: + SIM_40: + FILE: POST.sh + PROCESSORS: '40' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + POST_80: + + DEPENDENCIES: + POST_80: + SIM_80: + FILE: POST.sh + PROCESSORS: '80' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + SIM_20: + + DEPENDENCIES: + SIM_20-1: + FILE: POST.sh + PROCESSORS: '20' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + SIM_40: + + DEPENDENCIES: + SIM_40-1: + FILE: POST.sh + PROCESSORS: '40' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + SIM_80: + + DEPENDENCIES: + SIM_80-1: + FILE: POST.sh + PROCESSORS: '80' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 One can use now the following configuration: .. code-block:: yaml - POST: + experiment: + DATELIST: 19600101 + MEMBERS: "00" + CHUNKSIZEUNIT: day + CHUNKSIZE: '1' + NUMCHUNKS: '2' + CALENDAR: standard + JOBS: + SIM: FOR: NAME: [ 20,40,80 ] PROCESSORS: [ 20,40,80 ] THREADS: [ 1,1,1 ] - DEPENDENCIES: [ SIM_20 POST_20-1,SIM_40 POST_40-1,SIM_80 POST_80-1 ] + DEPENDENCIES: [ SIM_20-1,SIM_40-1,SIM_80-1 ] FILE: POST.sh RUNNING: chunk WALLCLOCK: '00:05' + POST: + FOR: + NAME: [ 20,40,80 ] + PROCESSORS: [ 20,40,80 ] + THREADS: [ 1,1,1 ] + DEPENDENCIES: [ SIM_20 POST_20,SIM_40 POST_40,SIM_80 POST_80 ] + FILE: POST.sh + RUNNING: chunk + WALLCLOCK: '00:05' .. warning:: Only the parameters that changes must be included inside the `FOR` key. @@ -598,11 +659,11 @@ Example 2: Crossdate wrappers using the the new dependencies COMPILE_DA: DA: DATES_FROM: - "20120201": - CHUNKS_FROM: - 1: - DATES_TO: "20120101" - CHUNKS_TO: "1" + "20120201": + CHUNKS_FROM: + 1: + DATES_TO: "20120101" + CHUNKS_TO: "1" RUNNING: chunk SYNCHRONIZE: member DELAY: '0' diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index 5b09b6905c75344b9798e64cda376ae20e3652ff..360d7a95995c95da296b8806c66a892d8385d9ce 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -180,7 +180,9 @@ To add a new hetjob, open the /cxxx/conf/jobs_cxxx.yml fi This will create a new job named "new_hetjob" with two components that will be executed once. +* EXTENDED_HEADER_PATH: specify the path relative to the project folder where the extension to the autosubmit's header is +* EXTENDED_TAILER_PATH: specify the path relative to the project folder where the extension to the autosubmit's tailer is How to configure email notifications ------------------------------------ diff --git a/docs/source/userguide/defining_workflows/fig/for.png b/docs/source/userguide/defining_workflows/fig/for.png new file mode 100644 index 0000000000000000000000000000000000000000..f7d5b4bb79eca2b0caf55dc3d03f56623ed167e3 Binary files /dev/null and b/docs/source/userguide/defining_workflows/fig/for.png differ diff --git a/docs/source/userguide/defining_workflows/index.rst b/docs/source/userguide/defining_workflows/index.rst index 257178ee73bedfff9480f23271172a5d05832058..57e12ff87afa86d4ed1410af4cc4fee5e6876dc6 100644 --- a/docs/source/userguide/defining_workflows/index.rst +++ b/docs/source/userguide/defining_workflows/index.rst @@ -733,40 +733,108 @@ To generate the following jobs: .. code-block:: yaml - POST_20: - FILE: POST.sh - RUNNING: chunk - WALLCLOCK: '00:05' - PROCESSORS: 20 - THREADS: 1 - DEPENDENCIES: SIM_20 POST_20-1 - POST_40: - FILE: POST.sh - RUNNING: chunk - WALLCLOCK: '00:05' - PROCESSORS: 40 - THREADS: 1 - DEPENDENCIES: SIM_40 POST_40-1 - POST_80: - FILE: POST.sh - RUNNING: chunk - WALLCLOCK: '00:05' - PROCESSORS: 80 - THREADS: 1 - DEPENDENCIES: SIM_80 POST_80-1 + experiment: + DATELIST: 19600101 + MEMBERS: "00" + CHUNKSIZEUNIT: day + CHUNKSIZE: '1' + NUMCHUNKS: '2' + CALENDAR: standard + JOBS: + POST_20: + + DEPENDENCIES: + POST_20: + SIM_20: + FILE: POST.sh + PROCESSORS: '20' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + POST_40: + + DEPENDENCIES: + POST_40: + SIM_40: + FILE: POST.sh + PROCESSORS: '40' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + POST_80: + + DEPENDENCIES: + POST_80: + SIM_80: + FILE: POST.sh + PROCESSORS: '80' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + SIM_20: + + DEPENDENCIES: + SIM_20-1: + FILE: POST.sh + PROCESSORS: '20' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + SIM_40: + + DEPENDENCIES: + SIM_40-1: + FILE: POST.sh + PROCESSORS: '40' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 + SIM_80: + + DEPENDENCIES: + SIM_80-1: + FILE: POST.sh + PROCESSORS: '80' + RUNNING: chunk + THREADS: '1' + WALLCLOCK: 00:05 -One can now use the following configuration: +One can use now the following configuration: .. code-block:: yaml - POST: + experiment: + DATELIST: 19600101 + MEMBERS: "00" + CHUNKSIZEUNIT: day + CHUNKSIZE: '1' + NUMCHUNKS: '2' + CALENDAR: standard + JOBS: + SIM: FOR: NAME: [ 20,40,80 ] PROCESSORS: [ 20,40,80 ] THREADS: [ 1,1,1 ] - DEPENDENCIES: [ SIM_20 POST_20-1,SIM_40 POST_40-1,SIM_80 POST_80-1 ] + DEPENDENCIES: [ SIM_20-1,SIM_40-1,SIM_80-1 ] FILE: POST.sh RUNNING: chunk WALLCLOCK: '00:05' + POST: + FOR: + NAME: [ 20,40,80 ] + PROCESSORS: [ 20,40,80 ] + THREADS: [ 1,1,1 ] + DEPENDENCIES: [ SIM_20 POST_20,SIM_40 POST_40,SIM_80 POST_80 ] + FILE: POST.sh + RUNNING: chunk + WALLCLOCK: '00:05' + + +.. warning:: The mutable parameters must be inside the `FOR` key. -.. warning:: The mutable parameters must be inside the `FOR` key. \ No newline at end of file +.. figure:: fig/for.png + :name: for + :width: 100% + :align: center + :alt: for \ No newline at end of file diff --git a/docs/source/userguide/run/index.rst b/docs/source/userguide/run/index.rst index a3b7984ad41aa10b9a3d816b74b71b2d14263a9e..e61e62f23d8e8bc0df190f2d35df4ded8925f155 100644 --- a/docs/source/userguide/run/index.rst +++ b/docs/source/userguide/run/index.rst @@ -12,7 +12,9 @@ Launch Autosubmit with the command: ssh-add ~/.ssh/id_rsa autosubmit run EXPID -*EXPID* is the experiment identifier. +In the previous command output ``EXPID`` is the experiment identifier. The command +exits with ``0`` when the workflow finishes with no failed jobs, and with ``1`` +otherwise. Options: :: @@ -40,7 +42,7 @@ Example: ssh-add ~/.ssh/id_rsa autosubmit run cxxx -.. important:: If the autosubmit version is set on autosubmit.yml it must match the actual autosubmit version +.. important:: If the autosubmit version is set on ``autosubmit.yml`` it must match the actual autosubmit version .. hint:: It is recommended to launch it in background and with ``nohup`` (continue running although the user who launched the process logs out). .. code-block:: bash diff --git a/docs/source/userguide/wrappers/index.rst b/docs/source/userguide/wrappers/index.rst index 168e5afa8c7093ec5da4d3359946a9ce156c04a1..fc678eee6b95d9b861b34ebdf2c429e0c860e2e7 100644 --- a/docs/source/userguide/wrappers/index.rst +++ b/docs/source/userguide/wrappers/index.rst @@ -23,9 +23,9 @@ To configure a new wrapper, the user has to define a `WRAPPERS` section in any c .. code-block:: YAML - WRAPPERS: - WRAPPER_0: - TYPE: "horizontal" + WRAPPERS: + WRAPPER_0: + TYPE: "horizontal" By default, Autosubmit will try to bundle jobs of the same type. The user can alter this behavior by setting the `JOBS_IN_WRAPPER` parameter directive in the wrapper section. @@ -47,6 +47,51 @@ When using multiple wrappers or 2-dim wrappers is essential to define the `JOBS_ TYPE: "horizontal-vertical" JOBS_IN_WRAPPER: "SIM5 SIM6" + experiment: + DATELIST: 20220101 + MEMBERS: "fc0 fc1" + CHUNKSIZEUNIT: day + CHUNKSIZE: '1' + NUMCHUNKS: '4' + CALENDAR: standard + JOBS: + SIM: + FILE: sim.sh + RUNNING: chunk + QUEUE: debug + DEPENDENCIES: SIM-1 + WALLCLOCK: 00:15 + SIM2: + FILE: sim.sh + RUNNING: chunk + QUEUE: debug + DEPENDENCIES: SIM2-1 + WALLCLOCK: 00:15 + SIM3: + FILE: sim.sh + RUNNING: chunk + QUEUE: debug + DEPENDENCIES: SIM3-1 + WALLCLOCK: 00:15 + SIM4: + FILE: sim.sh + RUNNING: chunk + QUEUE: debug + DEPENDENCIES: SIM4-1 + WALLCLOCK: 00:15 + SIM5: + FILE: sim.sh + RUNNING: chunk + QUEUE: debug + DEPENDENCIES: SIM5-1 + WALLCLOCK: 00:15 + SIM6: + FILE: sim.sh + RUNNING: chunk + QUEUE: debug + DEPENDENCIES: SIM6-1 + WALLCLOCK: 00:15 + .. figure:: fig/wrapper_all.png :name: wrapper all :align: center @@ -391,9 +436,9 @@ Considering the following configuration: DATES_FROM: "20120201": CHUNKS_FROM: - 1: - DATES_TO: "20120101" - CHUNKS_TO: "1" + 1: + DATES_TO: "20120101" + CHUNKS_TO: "1" RUNNING: chunk SYNCHRONIZE: member DELAY: '0' diff --git a/requeriments.txt b/requeriments.txt index d357f39dd55bda022a103d910c76efa2192375a6..ae0b28c5cd0ab49855b732c2930f4dfb44d65cb3 100644 --- a/requeriments.txt +++ b/requeriments.txt @@ -1,6 +1,7 @@ +zipp>=3.1.0 setuptools>=60.8.2 cython -autosubmitconfigparser==1.0.50 +autosubmitconfigparser==1.0.56 paramiko>=2.9.2 bcrypt>=3.2 PyNaCl>=1.5.0 diff --git a/setup.py b/setup.py index 7ad4b3409d3262b0c2cfac859c4a15c0e8ac2bc4..164dae7c7de87f1878c3ffd2eea1c4948abfc27f 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=['ruamel.yaml==0.17.21','cython','autosubmitconfigparser','bcrypt>=3.2','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2,<=2.7.0','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','pythondialog','pytest','nose','coverage','PyNaCl>=1.5.0','Pygments','psutil','rocrate==0.*'], + install_requires=['zipp>=3.1.0','ruamel.yaml==0.17.21','cython','autosubmitconfigparser','bcrypt>=3.2','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2,<=2.7.0','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','pythondialog','pytest','nose','coverage','PyNaCl>=1.5.0','Pygments','psutil','rocrate==0.*'], classifiers=[ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.9", diff --git a/test/regression/4.0_multi_testb.txt b/test/regression/4.0_multi_testb.txt new file mode 100644 index 0000000000000000000000000000000000000000..57606c291b827a10a2d36fb36170c5ec21b9858d --- /dev/null +++ b/test/regression/4.0_multi_testb.txt @@ -0,0 +1,1014 @@ +a01f +## String representation of Job List [8] ## +a01f_SYNC_TO_REMOTE ~ [1 child] +| a01f_REMOTE_SETUP ~ [2 children] +| | a01f_19910101_SIM ~ [1 child] +| | | a01f_19910101_GRAPH ~ [1 child] +| | | | a01f_SYNC_FROM_REMOTE ~ [1 child] +| | | | | a01f_CLEAN +| | a01f_19930101_SIM ~ [1 child] +| | | a01f_19930101_GRAPH ~ [1 child] +a015 +## String representation of Job List [14] ## +a015_LOCAL_SETUP ~ [1 child] +| a015_SYNCHRONIZE ~ [1 child] +| | a015_REMOTE_SETUP ~ [1 child] +| | | a015_19900101_fc0_INI ~ [1 child] +| | | | a015_19900101_fc0_1_SIM ~ [1 child] +| | | | | a015_19900101_fc0_2_SIM ~ [1 child] +| | | | | | a015_19900101_fc0_3_SIM ~ [1 child] +| | | | | | | a015_19900101_fc0_4_SIM ~ [1 child] +| | | | | | | | a015_19900101_fc0_5_SIM ~ [1 child] +| | | | | | | | | a015_19900101_fc0_6_SIM ~ [1 child] +| | | | | | | | | | a015_19900101_fc0_7_SIM ~ [1 child] +| | | | | | | | | | | a015_19900101_fc0_8_SIM ~ [1 child] +| | | | | | | | | | | | a015_19900101_fc0_9_SIM ~ [1 child] +| | | | | | | | | | | | | a015_19900101_fc0_10_SIM +a00e +## String representation of Job List [9] ## +a00e_20000101_fc0_1_SIM ~ [2 children] +| a00e_20000101_fc0_1_1_DN ~ [4 children] +| | a00e_20000101_fc0_1_1_OPA_2TMAX ~ [2 children] +| | | a00e_20000101_fc0_1_1_URBAN +| | | a00e_20000101_fc0_1_2_URBAN +| | a00e_20000101_fc0_1_1_OPA_2TMIN +| | a00e_20000101_fc0_1_2_OPA_2TMAX ~ [2 children] +| | a00e_20000101_fc0_1_2_OPA_2TMIN +| a00e_20000101_fc0_1_2_DN ~ [4 children] +a00t +## String representation of Job List [8] ## +a00t_LOCAL_SETUP ~ [1 child] +| a00t_REMOTE_SETUP ~ [1 child] +| | a00t_20000101_fc0_INI ~ [1 child] +| | | a00t_20000101_fc0_1_SIM ~ [1 child] +| | | | a00t_20000101_fc0_2_SIM ~ [1 child] +| | | | | a00t_POST ~ [1 child] +| | | | | | a00t_CLEAN ~ [1 child] +| | | | | | | a00t_20000101_fc0_TRANSFER +a01n +## String representation of Job List [8] ## +a01n_LOCAL_SETUP ~ [1 child] +| a01n_REMOTE_SETUP ~ [1 child] +| | a01n_20000101_fc0_INI ~ [1 child] +| | | a01n_20000101_fc0_1_SIM ~ [1 child] +| | | | a01n_20000101_fc0_2_SIM ~ [1 child] +| | | | | a01n_POST ~ [1 child] +| | | | | | a01n_CLEAN ~ [1 child] +| | | | | | | a01n_20000101_fc0_TRANSFER +a01v +## String representation of Job List [8] ## +a01v_LOCAL_SETUP ~ [1 child] +| a01v_REMOTE_SETUP ~ [1 child] +| | a01v_20000101_fc0_INI ~ [1 child] +| | | a01v_20000101_fc0_1_SIM ~ [1 child] +| | | | a01v_20000101_fc0_2_SIM ~ [1 child] +| | | | | a01v_POST ~ [1 child] +| | | | | | a01v_CLEAN ~ [1 child] +| | | | | | | a01v_20000101_fc0_TRANSFER +t006 +## String representation of Job List [5] ## +t006_LOCAL_SETUP ~ [1 child] +| t006_SYNCHRONIZE ~ [1 child] +| | t006_REMOTE_SETUP ~ [1 child] +| | | t006_19900101_default_INI ~ [1 child] +| | | | t006_19900101_default_1_SIM +a019 +## String representation of Job List [1] ## +a019_20210811_StrongScaling_PARAVER +a01a +## String representation of Job List [10] ## +a01a_20000101_fc0_1_HETJOB ~ [1 child] +| a01a_20000101_fc0_2_HETJOB ~ [1 child] +| | a01a_20000101_fc0_3_HETJOB ~ [1 child] +| | | a01a_20000101_fc0_4_HETJOB ~ [1 child] +| | | | a01a_20000101_fc0_5_HETJOB ~ [1 child] +| | | | | a01a_20000101_fc0_6_HETJOB ~ [1 child] +| | | | | | a01a_20000101_fc0_7_HETJOB ~ [1 child] +| | | | | | | a01a_20000101_fc0_8_HETJOB ~ [1 child] +| | | | | | | | a01a_20000101_fc0_9_HETJOB ~ [1 child] +| | | | | | | | | a01a_20000101_fc0_10_HETJOB +t001 +## String representation of Job List [2] ## +t001_20000101_fc0_1_HETJOB ~ [1 child] +| t001_20000101_fc0_2_HETJOB +a00i +## String representation of Job List [16] ## +a00i_1_OPA_TEMP +a00i_2_OPA_TEMP +a00i_3_OPA_TEMP +a00i_4_OPA_TEMP +a00i_1_OPA_WIND +a00i_2_OPA_WIND +a00i_3_OPA_WIND +a00i_4_OPA_WIND +a00i_1_OPA_SST +a00i_2_OPA_SST +a00i_3_OPA_SST +a00i_4_OPA_SST +a00i_1_OPA_ETC +a00i_2_OPA_ETC +a00i_3_OPA_ETC +a00i_4_OPA_ETC +a00w +## String representation of Job List [15] ## +a00w_LOCAL_SETUP ~ [1 child] +| a00w_SYNCHRONIZE ~ [1 child] +| | a00w_REMOTE_SETUP ~ [1 child] +| | | a00w_20211201_fc0_1_DN ~ [3 children] +| | | | a00w_20211201_fc0_1_1_OPA ~ [3 children] +| | | | | a00w_20211201_fc0_1_APP +| | | | | a00w_20211201_fc0_2_1_OPA ~ [3 children] +| | | | | | a00w_20211201_fc0_2_APP +| | | | | | a00w_20211201_fc0_3_1_OPA ~ [1 child] +| | | | | | | a00w_20211201_fc0_3_APP +| | | | | | a00w_20211201_fc0_3_2_OPA ~ [1 child] +| | | | | a00w_20211201_fc0_2_2_OPA ~ [3 children] +| | | | a00w_20211201_fc0_1_2_OPA ~ [3 children] +| | | | a00w_20211201_fc0_2_DN ~ [3 children] +| | | | | a00w_20211201_fc0_3_DN ~ [2 children] +a01r +## String representation of Job List [4] ## +a01r_REMOTE_SETUP ~ [1 child] +| a01r_20200128_fc0_1_DN ~ [1 child] +| | a01r_20200128_fc0_2_DN ~ [1 child] +| | | a01r_20200128_fc0_3_DN +a00g +## String representation of Job List [4] ## +a00g_19900101_fc0_1_SIM ~ [1 child] +| a00g_19900101_fc0_2_SIM ~ [1 child] +| | a00g_19900101_fc0_3_SIM ~ [1 child] +| | | a00g_19900101_fc0_4_SIM +a022 +## String representation of Job List [15] ## +a022_LOCAL_SETUP ~ [1 child] +| a022_SYNCHRONIZE ~ [1 child] +| | a022_REMOTE_SETUP ~ [1 child] +| | | a022_20200128_fc0_1_DN ~ [3 children] +| | | | a022_20200128_fc0_1_1_OPA ~ [2 children] +| | | | | a022_20200128_fc0_2_1_OPA ~ [2 children] +| | | | | | a022_20200128_fc0_3_1_OPA ~ [2 children] +| | | | | | | a022_20200128_fc0_4_1_OPA +| | | | | | | a022_20200128_fc0_4_2_OPA +| | | | | | a022_20200128_fc0_3_2_OPA ~ [2 children] +| | | | | a022_20200128_fc0_2_2_OPA ~ [2 children] +| | | | a022_20200128_fc0_1_2_OPA ~ [2 children] +| | | | a022_20200128_fc0_2_DN ~ [3 children] +| | | | | a022_20200128_fc0_3_DN ~ [3 children] +| | | | | | a022_20200128_fc0_4_DN ~ [2 children] +a004 +## String representation of Job List [6] ## +a004_LOCAL_SETUP ~ [1 child] +| a004_SYNCHRONIZE ~ [1 child] +| | a004_REMOTE_SETUP ~ [1 child] +| | | a004_20200120_fc0_1_DN ~ [1 child] +| | | | a004_20200120_fc0_1_OPA ~ [1 child] +| | | | | a004_20200120_fc0_1_APP +a00a +## String representation of Job List [1] ## +a00a_COPY_NAMELIST +a00b +## String representation of Job List [1] ## +a009_COPY_NAMELIST +a014 +## String representation of Job List [67] ## +a014_CLEAN ~ [1 child] +| a014_REMOTE_SETUP ~ [4 children] +| | a014_20210811_CompilationEfficiency_REMOTE_INIDATA ~ [1 child] +| | | a014_20210811_CompilationEfficiency_REMOTE_COMPILATION ~ [1 child] +| | | | a014_20210811_CompilationEfficiency_PREPARE_TESTS ~ [5 children] +| | | | | a014_20210811_CompilationEfficiency_FUNCTIONS_DIR ~ [4 children] +| | | | | | a014_20210811_CompilationEfficiency_TRACE_190 ~ [1 child] +| | | | | | | a014_20210811_CompilationEfficiency_REPORT_SETUP ~ [1 child] +| | | | | | | | a014_20210811_CompilationEfficiency_TRACE_CUT ~ [2 children] +| | | | | | | | | a014_20210811_CompilationEfficiency_DIMEMAS ~ [1 child] +| | | | | | | | | | a014_20210811_CompilationEfficiency_CONFIGURATION_JSON ~ [1 child] +| | | | | | | | | | | a014_20210811_CompilationEfficiency_PARAVER ~ [1 child] +| | | | | | | | | | | | a014_20210811_CompilationEfficiency_ADD_SECTION ~ [1 child] +| | | | | | | | | | | | | a014_REPORT +| | | | | | | | | a014_20210811_CompilationEfficiency_PARAMEDIR ~ [1 child] +| | | | | | a014_20210811_CompilationEfficiency_TRACE_192 ~ [1 child] +| | | | | | a014_20210811_CompilationEfficiency_TRACE_48 ~ [1 child] +| | | | | | a014_20210811_CompilationEfficiency_TRACE_96 ~ [1 child] +| | | | | a014_20210811_CompilationEfficiency_SCALABILITY_190 ~ [1 child] +| | | | | a014_20210811_CompilationEfficiency_SCALABILITY_192 ~ [1 child] +| | | | | a014_20210811_CompilationEfficiency_SCALABILITY_48 ~ [1 child] +| | | | | a014_20210811_CompilationEfficiency_SCALABILITY_96 ~ [1 child] +| | a014_20210811_HardwareBenchmarks_REMOTE_INIDATA ~ [1 child] +| | | a014_20210811_HardwareBenchmarks_REMOTE_COMPILATION ~ [1 child] +| | | | a014_20210811_HardwareBenchmarks_PREPARE_TESTS ~ [10 children] +| | | | | a014_20210811_HardwareBenchmarks_OSU_RUN_192 ~ [1 child] +| | | | | | a014_20210811_HardwareBenchmarks_REPORT_SETUP ~ [1 child] +| | | | | | | a014_20210811_HardwareBenchmarks_HARDWARE_BENCH_IMAGES ~ [1 child] +| | | | | | | | a014_20210811_HardwareBenchmarks_ADD_SECTION ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_OSU_RUN_48 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_OSU_RUN_96 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_STREAM_1 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_STREAM_16 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_STREAM_2 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_STREAM_32 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_STREAM_4 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_STREAM_48 ~ [1 child] +| | | | | a014_20210811_HardwareBenchmarks_STREAM_8 ~ [1 child] +| | a014_20210811_StrongScaling_REMOTE_INIDATA ~ [1 child] +| | | a014_20210811_StrongScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a014_20210811_StrongScaling_PREPARE_TESTS ~ [2 children] +| | | | | a014_20210811_StrongScaling_FUNCTIONS_DIR ~ [1 child] +| | | | | | a014_20210811_StrongScaling_TRACE_190 ~ [1 child] +| | | | | | | a014_20210811_StrongScaling_REPORT_SETUP ~ [1 child] +| | | | | | | | a014_20210811_StrongScaling_TRACE_CUT ~ [3 children] +| | | | | | | | | a014_20210811_StrongScaling_DIMEMAS ~ [1 child] +| | | | | | | | | | a014_20210811_StrongScaling_CONFIGURATION_JSON ~ [1 child] +| | | | | | | | | | | a014_20210811_StrongScaling_PARAVER ~ [1 child] +| | | | | | | | | | | | a014_20210811_StrongScaling_ADD_SECTION ~ [1 child] +| | | | | | | | | a014_20210811_StrongScaling_MODELFACTORS ~ [1 child] +| | | | | | | | | a014_20210811_StrongScaling_PARAMEDIR ~ [1 child] +| | | | | a014_20210811_StrongScaling_SCALABILITY_190 ~ [1 child] +| | a014_20210811_WeakScaling_REMOTE_INIDATA ~ [1 child] +| | | a014_20210811_WeakScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a014_20210811_WeakScaling_PREPARE_TESTS ~ [3 children] +| | | | | a014_20210811_WeakScaling_FUNCTIONS_DIR ~ [2 children] +| | | | | | a014_20210811_WeakScaling_TRACE_384 ~ [1 child] +| | | | | | | a014_20210811_WeakScaling_REPORT_SETUP ~ [1 child] +| | | | | | | | a014_20210811_WeakScaling_TRACE_CUT ~ [2 children] +| | | | | | | | | a014_20210811_WeakScaling_DIMEMAS ~ [1 child] +| | | | | | | | | | a014_20210811_WeakScaling_CONFIGURATION_JSON ~ [1 child] +| | | | | | | | | | | a014_20210811_WeakScaling_PARAVER ~ [1 child] +| | | | | | | | | | | | a014_20210811_WeakScaling_ADD_SECTION ~ [1 child] +| | | | | | | | | a014_20210811_WeakScaling_PARAMEDIR ~ [1 child] +| | | | | | a014_20210811_WeakScaling_TRACE_756 ~ [1 child] +| | | | | a014_20210811_WeakScaling_SCALABILITY_384 ~ [1 child] +| | | | | a014_20210811_WeakScaling_SCALABILITY_756 ~ [1 child] +a00d +## String representation of Job List [8] ## +a00d_LOCAL_SETUP ~ [1 child] +| a00d_REMOTE_SETUP ~ [1 child] +| | a00d_20220401_fc0_INI ~ [1 child] +| | | a00d_20220401_fc0_1_SIM ~ [1 child] +| | | | a00d_20220401_fc0_2_SIM ~ [1 child] +| | | | | a00d_POST ~ [1 child] +| | | | | | a00d_CLEAN ~ [1 child] +| | | | | | | a00d_20220401_fc0_TRANSFER +a012 +## String representation of Job List [1] ## +a012_TEST_X11 +a018 +## String representation of Job List [8] ## +a018_LOCAL_SETUP ~ [1 child] +| a018_SYNCHRONIZE ~ [1 child] +| | a018_REMOTE_SETUP ~ [1 child] +| | | a018_19900101_fc0_INI ~ [1 child] +| | | | a018_19900101_fc0_1_SIM ~ [1 child] +| | | | | a018_19900101_fc0_2_SIM ~ [1 child] +| | | | | | a018_19900101_fc0_3_SIM ~ [1 child] +| | | | | | | a018_19900101_fc0_4_SIM +a00u +## String representation of Job List [25] ## +a00u_LOCAL_SETUP ~ [8 children] +| a00u_20120101_000_1_LOCAL_SEND_INITIAL +| a00u_20120101_000_2_LOCAL_SEND_INITIAL +| a00u_20120101_001_1_LOCAL_SEND_INITIAL +| a00u_20120101_001_2_LOCAL_SEND_INITIAL +| a00u_20120101_002_1_LOCAL_SEND_INITIAL +| a00u_20120101_002_2_LOCAL_SEND_INITIAL +| a00u_LOCAL_SEND_SOURCE ~ [1 child] +| | a00u_REMOTE_COMPILE ~ [3 children] +| | | a00u_20120101_000_PREPROCFIX ~ [2 children] +| | | | a00u_20120101_000_1_PREPROCVAR ~ [1 child] +| | | | | a00u_20120101_000_1_CLEAN +| | | | a00u_20120101_000_2_PREPROCVAR ~ [1 child] +| | | | | a00u_20120101_000_2_CLEAN +| | | a00u_20120101_001_PREPROCFIX ~ [2 children] +| | | | a00u_20120101_001_1_PREPROCVAR ~ [1 child] +| | | | | a00u_20120101_001_1_CLEAN +| | | | a00u_20120101_001_2_PREPROCVAR ~ [1 child] +| | | | | a00u_20120101_001_2_CLEAN +| | | a00u_20120101_002_PREPROCFIX ~ [2 children] +| | | | a00u_20120101_002_1_PREPROCVAR ~ [1 child] +| | | | | a00u_20120101_002_1_CLEAN +| | | | a00u_20120101_002_2_PREPROCVAR ~ [1 child] +| | | | | a00u_20120101_002_2_CLEAN +| a00u_LOCAL_SEND_STATIC ~ [3 children] +a017 +## String representation of Job List [17] ## +a017_LOCAL_SETUP ~ [2 children] +| a017_REMOTE_SETUP ~ [2 children] +| | a017_20000101_fc0_1_DN ~ [3 children] +| | | a017_20000101_fc0_1_1_OPA ~ [3 children] +| | | | a017_20000101_fc0_1_APP +| | | | a017_20000101_fc0_2_1_OPA ~ [1 child] +| | | | | a017_20000101_fc0_2_APP +| | | | a017_20000101_fc0_2_2_OPA ~ [1 child] +| | | a017_20000101_fc0_1_2_OPA ~ [3 children] +| | | a017_20000101_fc0_2_DN ~ [2 children] +| | a017_20000101_fc0_INI ~ [1 child] +| | | a017_20000101_fc0_1_SIM ~ [1 child] +| | | | a017_20000101_fc0_2_SIM ~ [1 child] +| | | | | a017_POST ~ [1 child] +| | | | | | a017_CLEAN ~ [1 child] +| | | | | | | a017_20000101_fc0_TRANSFER +| a017_SYNCHRONIZE +a013 +## String representation of Job List [1] ## +a013_WAIT +t005 +## String representation of Job List [0] ## +a00p +## String representation of Job List [4] ## +a00p_2020100100_m1_1_DUMMY +a00p_2020100100_m1_2_DUMMY +a00p_2020100100_m1_3_DUMMY +a00p_2020100100_m1_4_DUMMY +a007 +## String representation of Job List [8] ## +a007_LOCAL_SETUP ~ [1 child] +| a007_REMOTE_SETUP ~ [1 child] +| | a007_20220401_fc0_INI ~ [1 child] +| | | a007_20220401_fc0_1_SIM ~ [1 child] +| | | | a007_20220401_fc0_2_SIM ~ [1 child] +| | | | | a007_POST ~ [1 child] +| | | | | | a007_CLEAN ~ [1 child] +| | | | | | | a007_20220401_fc0_TRANSFER +a001 +## String representation of Job List [40] ## +a001_CLEAN ~ [1 child] +| a001_REMOTE_SETUP ~ [4 children] +| | a001_20210811_CompilationEfficiency_REMOTE_INIDATA ~ [1 child] +| | | a001_20210811_CompilationEfficiency_REMOTE_COMPILATION ~ [1 child] +| | | | a001_20210811_CompilationEfficiency_PREPARE_TESTS +| | a001_20210811_HardwareBenchmarks_REMOTE_INIDATA ~ [1 child] +| | | a001_20210811_HardwareBenchmarks_REMOTE_COMPILATION ~ [1 child] +| | | | a001_20210811_HardwareBenchmarks_PREPARE_TESTS ~ [10 children] +| | | | | a001_20210811_HardwareBenchmarks_OSU_RUN_192 +| | | | | a001_20210811_HardwareBenchmarks_OSU_RUN_48 +| | | | | a001_20210811_HardwareBenchmarks_OSU_RUN_96 +| | | | | a001_20210811_HardwareBenchmarks_STREAM_1 +| | | | | a001_20210811_HardwareBenchmarks_STREAM_16 +| | | | | a001_20210811_HardwareBenchmarks_STREAM_2 +| | | | | a001_20210811_HardwareBenchmarks_STREAM_32 +| | | | | a001_20210811_HardwareBenchmarks_STREAM_4 +| | | | | a001_20210811_HardwareBenchmarks_STREAM_48 +| | | | | a001_20210811_HardwareBenchmarks_STREAM_8 +| | a001_20210811_StrongScaling_REMOTE_INIDATA ~ [1 child] +| | | a001_20210811_StrongScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a001_20210811_StrongScaling_PREPARE_TESTS ~ [1 child] +| | | | | a001_20210811_StrongScaling_SCALABILITY_192 ~ [1 child] +| | | | | | a001_20210811_StrongScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a001_20210811_StrongScaling_TRACE_192 +| | | | | | | a001_20210811_StrongScaling_TRACE_48 +| | | | | | | a001_20210811_StrongScaling_TRACE_96 +| | a001_20210811_WeakScaling_REMOTE_INIDATA ~ [1 child] +| | | a001_20210811_WeakScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a001_20210811_WeakScaling_PREPARE_TESTS ~ [2 children] +| | | | | a001_20210811_WeakScaling_SCALABILITY_1024 +| | | | | a001_20210811_WeakScaling_SCALABILITY_768 +a001_20210811_WeakScaling_SCALABILITY_48 ~ [1 child] +| a001_20210811_WeakScaling_FUNCTIONS_DIR ~ [3 children] +| | a001_20210811_WeakScaling_TRACE_192 +| | a001_20210811_WeakScaling_TRACE_48 +| | a001_20210811_WeakScaling_TRACE_96 +a001_20210811_StrongScaling_SCALABILITY_48 ~ [1 child] +a001_20210811_WeakScaling_SCALABILITY_96 ~ [1 child] +a001_20210811_StrongScaling_SCALABILITY_96 ~ [1 child] +a001_20210811_WeakScaling_SCALABILITY_192 ~ [1 child] +a01w +## String representation of Job List [8] ## +a01w_LOCAL_SETUP ~ [1 child] +| a01w_REMOTE_SETUP ~ [1 child] +| | a01w_20000101_fc0_INI ~ [1 child] +| | | a01w_20000101_fc0_1_SIM ~ [1 child] +| | | | a01w_20000101_fc0_2_SIM ~ [1 child] +| | | | | a01w_POST ~ [1 child] +| | | | | | a01w_CLEAN ~ [1 child] +| | | | | | | a01w_20000101_fc0_TRANSFER +t004 +## String representation of Job List [3] ## +t004_20000101_fc0_1_JOBA ~ [1 child] +| t004_20000101_fc0_2_JOBA ~ [1 child] +| | t004_20000101_fc0_3_JOBA +a01m +## String representation of Job List [6] ## +a01m_LOCAL_SETUP ~ [1 child] +| a01m_SYNCHRONIZE ~ [1 child] +| | a01m_REMOTE_SETUP ~ [1 child] +| | | a01m_19500101_default_INI ~ [1 child] +| | | | a01m_19500101_default_1_SIM ~ [1 child] +| | | | | a01m_19500101_default_2_SIM +t003 +## String representation of Job List [2] ## +t003_20000101_fc0_1_JOBA ~ [1 child] +| t003_20000101_fc0_2_JOBA +a01x +## String representation of Job List [8] ## +a01x_LOCAL_SETUP ~ [1 child] +| a01x_REMOTE_SETUP ~ [1 child] +| | a01x_20000101_fc0_INI ~ [1 child] +| | | a01x_20000101_fc0_1_SIM ~ [1 child] +| | | | a01x_20000101_fc0_2_SIM ~ [1 child] +| | | | | a01x_POST ~ [1 child] +| | | | | | a01x_CLEAN ~ [1 child] +| | | | | | | a01x_20000101_fc0_TRANSFER +a01b +## String representation of Job List [1] ## +a01b_REMOTE_SETUP +a009 +## String representation of Job List [1] ## +a009_COPY_NAMELIST +a01k +## String representation of Job List [6] ## +a01k_LOCAL_SETUP ~ [1 child] +| a01k_REMOTE_SETUP ~ [1 child] +| | a01k_SYNC ~ [1 child] +| | | a01k_SIM ~ [1 child] +| | | | a01k_GRAPH ~ [1 child] +| | | | | a01k_COPY_GRAPH +a00m +## String representation of Job List [45] ## +a00m_CLEAN ~ [1 child] +| a00m_REMOTE_SETUP ~ [4 children] +| | a00m_20210811_CompilationEfficiency_REMOTE_INIDATA ~ [1 child] +| | | a00m_20210811_CompilationEfficiency_REMOTE_COMPILATION ~ [1 child] +| | | | a00m_20210811_CompilationEfficiency_PREPARE_TESTS ~ [3 children] +| | | | | a00m_20210811_CompilationEfficiency_SCALABILITY_192 ~ [1 child] +| | | | | | a00m_20210811_CompilationEfficiency_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00m_20210811_CompilationEfficiency_TRACE_192 +| | | | | | | a00m_20210811_CompilationEfficiency_TRACE_48 +| | | | | | | a00m_20210811_CompilationEfficiency_TRACE_96 +| | | | | a00m_20210811_CompilationEfficiency_SCALABILITY_48 ~ [1 child] +| | | | | a00m_20210811_CompilationEfficiency_SCALABILITY_96 ~ [1 child] +| | a00m_20210811_HardwareBenchmarks_REMOTE_INIDATA ~ [1 child] +| | | a00m_20210811_HardwareBenchmarks_REMOTE_COMPILATION ~ [1 child] +| | | | a00m_20210811_HardwareBenchmarks_PREPARE_TESTS ~ [10 children] +| | | | | a00m_20210811_HardwareBenchmarks_OSU_RUN_192 +| | | | | a00m_20210811_HardwareBenchmarks_OSU_RUN_48 +| | | | | a00m_20210811_HardwareBenchmarks_OSU_RUN_96 +| | | | | a00m_20210811_HardwareBenchmarks_STREAM_1 +| | | | | a00m_20210811_HardwareBenchmarks_STREAM_16 +| | | | | a00m_20210811_HardwareBenchmarks_STREAM_2 +| | | | | a00m_20210811_HardwareBenchmarks_STREAM_32 +| | | | | a00m_20210811_HardwareBenchmarks_STREAM_4 +| | | | | a00m_20210811_HardwareBenchmarks_STREAM_48 +| | | | | a00m_20210811_HardwareBenchmarks_STREAM_8 +| | a00m_20210811_StrongScaling_REMOTE_INIDATA ~ [1 child] +| | | a00m_20210811_StrongScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a00m_20210811_StrongScaling_PREPARE_TESTS ~ [3 children] +| | | | | a00m_20210811_StrongScaling_SCALABILITY_192 ~ [1 child] +| | | | | | a00m_20210811_StrongScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00m_20210811_StrongScaling_TRACE_192 +| | | | | | | a00m_20210811_StrongScaling_TRACE_48 +| | | | | | | a00m_20210811_StrongScaling_TRACE_96 +| | | | | a00m_20210811_StrongScaling_SCALABILITY_48 ~ [1 child] +| | | | | a00m_20210811_StrongScaling_SCALABILITY_96 ~ [1 child] +| | a00m_20210811_WeakScaling_REMOTE_INIDATA ~ [1 child] +| | | a00m_20210811_WeakScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a00m_20210811_WeakScaling_PREPARE_TESTS ~ [3 children] +| | | | | a00m_20210811_WeakScaling_SCALABILITY_192 ~ [1 child] +| | | | | | a00m_20210811_WeakScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00m_20210811_WeakScaling_TRACE_192 +| | | | | | | a00m_20210811_WeakScaling_TRACE_48 +| | | | | | | a00m_20210811_WeakScaling_TRACE_96 +| | | | | a00m_20210811_WeakScaling_SCALABILITY_48 ~ [1 child] +| | | | | a00m_20210811_WeakScaling_SCALABILITY_96 ~ [1 child] +a01g +## String representation of Job List [8] ## +a01g_LOCAL_SETUP ~ [1 child] +| a01g_REMOTE_SETUP ~ [1 child] +| | a01g_20000101_fc0_INI ~ [1 child] +| | | a01g_20000101_fc0_1_SIM ~ [1 child] +| | | | a01g_20000101_fc0_2_SIM ~ [1 child] +| | | | | a01g_POST ~ [1 child] +| | | | | | a01g_CLEAN ~ [1 child] +| | | | | | | a01g_20000101_fc0_TRANSFER +a011 +## String representation of Job List [7] ## +a011_LOCAL_SETUP ~ [1 child] +| a011_SYNCHRONIZE ~ [1 child] +| | a011_REMOTE_SETUP ~ [1 child] +| | | a011_20000101_fc0_INI ~ [1 child] +| | | | a011_20000101_fc0_1_SIM ~ [1 child] +| | | | | a011_20000101_fc0_1_GSV ~ [1 child] +| | | | | | a011_20000101_fc0_1_APPLICATION +a010 +## String representation of Job List [60] ## +a010_1_DN ~ [1 child] +| a010_1_OPA_VENTICUATRO +a010_2_DN ~ [2 children] +| a010_2_OPA_DOCE +| a010_2_OPA_VENTICUATRO +a010_3_DN ~ [1 child] +| a010_3_OPA_VENTICUATRO +a010_4_DN ~ [2 children] +| a010_4_OPA_DOCE +| a010_4_OPA_VENTICUATRO +a010_5_DN ~ [1 child] +| a010_5_OPA_VENTICUATRO +a010_6_DN ~ [2 children] +| a010_6_OPA_DOCE +| a010_6_OPA_VENTICUATRO +a010_7_DN ~ [1 child] +| a010_7_OPA_VENTICUATRO +a010_8_DN ~ [2 children] +| a010_8_OPA_DOCE +| a010_8_OPA_VENTICUATRO +a010_9_DN ~ [1 child] +| a010_9_OPA_VENTICUATRO +a010_10_DN ~ [2 children] +| a010_10_OPA_DOCE +| a010_10_OPA_VENTICUATRO +a010_11_DN ~ [1 child] +| a010_11_OPA_VENTICUATRO +a010_12_DN ~ [2 children] +| a010_12_OPA_DOCE +| a010_12_OPA_VENTICUATRO +a010_13_DN ~ [1 child] +| a010_13_OPA_VENTICUATRO +a010_14_DN ~ [2 children] +| a010_14_OPA_DOCE +| a010_14_OPA_VENTICUATRO +a010_15_DN ~ [1 child] +| a010_15_OPA_VENTICUATRO +a010_16_DN ~ [2 children] +| a010_16_OPA_DOCE +| a010_16_OPA_VENTICUATRO +a010_17_DN ~ [1 child] +| a010_17_OPA_VENTICUATRO +a010_18_DN ~ [2 children] +| a010_18_OPA_DOCE +| a010_18_OPA_VENTICUATRO +a010_19_DN ~ [1 child] +| a010_19_OPA_VENTICUATRO +a010_20_DN ~ [2 children] +| a010_20_OPA_DOCE +| a010_20_OPA_VENTICUATRO +a010_21_DN ~ [1 child] +| a010_21_OPA_VENTICUATRO +a010_22_DN ~ [2 children] +| a010_22_OPA_DOCE +| a010_22_OPA_VENTICUATRO +a010_23_DN ~ [1 child] +| a010_23_OPA_VENTICUATRO +a010_24_DN ~ [2 children] +| a010_24_OPA_DOCE +| a010_24_OPA_VENTICUATRO +a01o +## String representation of Job List [8] ## +a01o_LOCAL_SETUP ~ [1 child] +| a01o_SYNCHRONIZE ~ [1 child] +| | a01o_REMOTE_SETUP ~ [1 child] +| | | a01o_20220401_fc0_INI ~ [2 children] +| | | | a01o_20220401_fc0_1_SIM ~ [1 child] +| | | | | a01o_POST ~ [1 child] +| | | | | | a01o_CLEAN +| | | | a01o_20220401_fc0_2_SIM ~ [1 child] +a01q +## String representation of Job List [1] ## +a01q_COPY_NAMELIST +a01p +## String representation of Job List [1] ## +a009_COPY_NAMELIST +a00c +## String representation of Job List [13] ## +a00c_LOCAL_SETUP ~ [1 child] +| a00c_REMOTE_SETUP ~ [1 child] +| | a00c_20000101_fc0_INI ~ [1 child] +| | | a00c_20000101_fc0_1_SIM ~ [2 children] +| | | | a00c_20000101_fc0_1_DATA_NOTIFY ~ [1 child] +| | | | | a00c_20000101_fc0_1_OPA ~ [2 children] +| | | | | | a00c_20000101_fc0_1_APPLICATION +| | | | | | a00c_20000101_fc0_1_AQUA_DIAGNOSTIC +| | | | a00c_20000101_fc0_2_SIM ~ [1 child] +| | | | | a00c_20000101_fc0_2_DATA_NOTIFY ~ [1 child] +| | | | | | a00c_20000101_fc0_2_OPA ~ [2 children] +| | | | | | | a00c_20000101_fc0_2_APPLICATION +| | | | | | | a00c_20000101_fc0_2_AQUA_DIAGNOSTIC +a01s +## String representation of Job List [10] ## +a01s_LOCAL_SETUP ~ [1 child] +| a01s_SYNCHRONIZE ~ [1 child] +| | a01s_REMOTE_SETUP ~ [1 child] +| | | a01s_20220401_fc0_INI ~ [4 children] +| | | | a01s_20220401_fc0_1_SIM ~ [1 child] +| | | | | a01s_POST ~ [1 child] +| | | | | | a01s_CLEAN +| | | | a01s_20220401_fc0_2_SIM ~ [1 child] +| | | | a01s_20220401_fc0_3_SIM ~ [1 child] +| | | | a01s_20220401_fc0_4_SIM ~ [1 child] +a01j +## String representation of Job List [8] ## +a01j_LOCAL_SETUP ~ [1 child] +| a01j_REMOTE_SETUP ~ [1 child] +| | a01j_20000101_fc0_INI ~ [1 child] +| | | a01j_20000101_fc0_1_SIM ~ [1 child] +| | | | a01j_20000101_fc0_2_SIM ~ [1 child] +| | | | | a01j_POST ~ [1 child] +| | | | | | a01j_CLEAN ~ [1 child] +| | | | | | | a01j_20000101_fc0_TRANSFER +a00n +## String representation of Job List [162] ## +a00n_CLEAN ~ [1 child] +| a00n_REMOTE_SETUP ~ [8 children] +| | a00n_20210811_CompilationEfficiency_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210811_CompilationEfficiency_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210811_CompilationEfficiency_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210811_CompilationEfficiency_OSU_RUN_192 +| | | | | a00n_20210811_CompilationEfficiency_OSU_RUN_48 +| | | | | a00n_20210811_CompilationEfficiency_OSU_RUN_96 +| | | | | a00n_20210811_CompilationEfficiency_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210811_CompilationEfficiency_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210811_CompilationEfficiency_TRACE_192 +| | | | | | | a00n_20210811_CompilationEfficiency_TRACE_48 +| | | | | | | a00n_20210811_CompilationEfficiency_TRACE_96 +| | | | | a00n_20210811_CompilationEfficiency_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210811_CompilationEfficiency_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210811_CompilationEfficiency_STREAM_1 +| | | | | a00n_20210811_CompilationEfficiency_STREAM_16 +| | | | | a00n_20210811_CompilationEfficiency_STREAM_2 +| | | | | a00n_20210811_CompilationEfficiency_STREAM_32 +| | | | | a00n_20210811_CompilationEfficiency_STREAM_4 +| | | | | a00n_20210811_CompilationEfficiency_STREAM_48 +| | | | | a00n_20210811_CompilationEfficiency_STREAM_8 +| | a00n_20210811_HardwareBenchmarks_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210811_HardwareBenchmarks_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210811_HardwareBenchmarks_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210811_HardwareBenchmarks_OSU_RUN_192 +| | | | | a00n_20210811_HardwareBenchmarks_OSU_RUN_48 +| | | | | a00n_20210811_HardwareBenchmarks_OSU_RUN_96 +| | | | | a00n_20210811_HardwareBenchmarks_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210811_HardwareBenchmarks_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210811_HardwareBenchmarks_TRACE_192 +| | | | | | | a00n_20210811_HardwareBenchmarks_TRACE_48 +| | | | | | | a00n_20210811_HardwareBenchmarks_TRACE_96 +| | | | | a00n_20210811_HardwareBenchmarks_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210811_HardwareBenchmarks_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210811_HardwareBenchmarks_STREAM_1 +| | | | | a00n_20210811_HardwareBenchmarks_STREAM_16 +| | | | | a00n_20210811_HardwareBenchmarks_STREAM_2 +| | | | | a00n_20210811_HardwareBenchmarks_STREAM_32 +| | | | | a00n_20210811_HardwareBenchmarks_STREAM_4 +| | | | | a00n_20210811_HardwareBenchmarks_STREAM_48 +| | | | | a00n_20210811_HardwareBenchmarks_STREAM_8 +| | a00n_20210811_StrongScaling_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210811_StrongScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210811_StrongScaling_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210811_StrongScaling_OSU_RUN_192 +| | | | | a00n_20210811_StrongScaling_OSU_RUN_48 +| | | | | a00n_20210811_StrongScaling_OSU_RUN_96 +| | | | | a00n_20210811_StrongScaling_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210811_StrongScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210811_StrongScaling_TRACE_192 +| | | | | | | a00n_20210811_StrongScaling_TRACE_48 +| | | | | | | a00n_20210811_StrongScaling_TRACE_96 +| | | | | a00n_20210811_StrongScaling_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210811_StrongScaling_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210811_StrongScaling_STREAM_1 +| | | | | a00n_20210811_StrongScaling_STREAM_16 +| | | | | a00n_20210811_StrongScaling_STREAM_2 +| | | | | a00n_20210811_StrongScaling_STREAM_32 +| | | | | a00n_20210811_StrongScaling_STREAM_4 +| | | | | a00n_20210811_StrongScaling_STREAM_48 +| | | | | a00n_20210811_StrongScaling_STREAM_8 +| | a00n_20210811_WeakScaling_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210811_WeakScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210811_WeakScaling_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210811_WeakScaling_OSU_RUN_192 +| | | | | a00n_20210811_WeakScaling_OSU_RUN_48 +| | | | | a00n_20210811_WeakScaling_OSU_RUN_96 +| | | | | a00n_20210811_WeakScaling_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210811_WeakScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210811_WeakScaling_TRACE_192 +| | | | | | | a00n_20210811_WeakScaling_TRACE_48 +| | | | | | | a00n_20210811_WeakScaling_TRACE_96 +| | | | | a00n_20210811_WeakScaling_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210811_WeakScaling_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210811_WeakScaling_STREAM_1 +| | | | | a00n_20210811_WeakScaling_STREAM_16 +| | | | | a00n_20210811_WeakScaling_STREAM_2 +| | | | | a00n_20210811_WeakScaling_STREAM_32 +| | | | | a00n_20210811_WeakScaling_STREAM_4 +| | | | | a00n_20210811_WeakScaling_STREAM_48 +| | | | | a00n_20210811_WeakScaling_STREAM_8 +| | a00n_20210812_CompilationEfficiency_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210812_CompilationEfficiency_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210812_CompilationEfficiency_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210812_CompilationEfficiency_OSU_RUN_192 +| | | | | a00n_20210812_CompilationEfficiency_OSU_RUN_48 +| | | | | a00n_20210812_CompilationEfficiency_OSU_RUN_96 +| | | | | a00n_20210812_CompilationEfficiency_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210812_CompilationEfficiency_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210812_CompilationEfficiency_TRACE_192 +| | | | | | | a00n_20210812_CompilationEfficiency_TRACE_48 +| | | | | | | a00n_20210812_CompilationEfficiency_TRACE_96 +| | | | | a00n_20210812_CompilationEfficiency_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210812_CompilationEfficiency_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210812_CompilationEfficiency_STREAM_1 +| | | | | a00n_20210812_CompilationEfficiency_STREAM_16 +| | | | | a00n_20210812_CompilationEfficiency_STREAM_2 +| | | | | a00n_20210812_CompilationEfficiency_STREAM_32 +| | | | | a00n_20210812_CompilationEfficiency_STREAM_4 +| | | | | a00n_20210812_CompilationEfficiency_STREAM_48 +| | | | | a00n_20210812_CompilationEfficiency_STREAM_8 +| | a00n_20210812_HardwareBenchmarks_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210812_HardwareBenchmarks_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210812_HardwareBenchmarks_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210812_HardwareBenchmarks_OSU_RUN_192 +| | | | | a00n_20210812_HardwareBenchmarks_OSU_RUN_48 +| | | | | a00n_20210812_HardwareBenchmarks_OSU_RUN_96 +| | | | | a00n_20210812_HardwareBenchmarks_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210812_HardwareBenchmarks_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210812_HardwareBenchmarks_TRACE_192 +| | | | | | | a00n_20210812_HardwareBenchmarks_TRACE_48 +| | | | | | | a00n_20210812_HardwareBenchmarks_TRACE_96 +| | | | | a00n_20210812_HardwareBenchmarks_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210812_HardwareBenchmarks_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210812_HardwareBenchmarks_STREAM_1 +| | | | | a00n_20210812_HardwareBenchmarks_STREAM_16 +| | | | | a00n_20210812_HardwareBenchmarks_STREAM_2 +| | | | | a00n_20210812_HardwareBenchmarks_STREAM_32 +| | | | | a00n_20210812_HardwareBenchmarks_STREAM_4 +| | | | | a00n_20210812_HardwareBenchmarks_STREAM_48 +| | | | | a00n_20210812_HardwareBenchmarks_STREAM_8 +| | a00n_20210812_StrongScaling_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210812_StrongScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210812_StrongScaling_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210812_StrongScaling_OSU_RUN_192 +| | | | | a00n_20210812_StrongScaling_OSU_RUN_48 +| | | | | a00n_20210812_StrongScaling_OSU_RUN_96 +| | | | | a00n_20210812_StrongScaling_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210812_StrongScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210812_StrongScaling_TRACE_192 +| | | | | | | a00n_20210812_StrongScaling_TRACE_48 +| | | | | | | a00n_20210812_StrongScaling_TRACE_96 +| | | | | a00n_20210812_StrongScaling_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210812_StrongScaling_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210812_StrongScaling_STREAM_1 +| | | | | a00n_20210812_StrongScaling_STREAM_16 +| | | | | a00n_20210812_StrongScaling_STREAM_2 +| | | | | a00n_20210812_StrongScaling_STREAM_32 +| | | | | a00n_20210812_StrongScaling_STREAM_4 +| | | | | a00n_20210812_StrongScaling_STREAM_48 +| | | | | a00n_20210812_StrongScaling_STREAM_8 +| | a00n_20210812_WeakScaling_REMOTE_INIDATA ~ [1 child] +| | | a00n_20210812_WeakScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a00n_20210812_WeakScaling_PREPARE_TESTS ~ [13 children] +| | | | | a00n_20210812_WeakScaling_OSU_RUN_192 +| | | | | a00n_20210812_WeakScaling_OSU_RUN_48 +| | | | | a00n_20210812_WeakScaling_OSU_RUN_96 +| | | | | a00n_20210812_WeakScaling_SCALABILITY_192 ~ [1 child] +| | | | | | a00n_20210812_WeakScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | | a00n_20210812_WeakScaling_TRACE_192 +| | | | | | | a00n_20210812_WeakScaling_TRACE_48 +| | | | | | | a00n_20210812_WeakScaling_TRACE_96 +| | | | | a00n_20210812_WeakScaling_SCALABILITY_48 ~ [1 child] +| | | | | a00n_20210812_WeakScaling_SCALABILITY_96 ~ [1 child] +| | | | | a00n_20210812_WeakScaling_STREAM_1 +| | | | | a00n_20210812_WeakScaling_STREAM_16 +| | | | | a00n_20210812_WeakScaling_STREAM_2 +| | | | | a00n_20210812_WeakScaling_STREAM_32 +| | | | | a00n_20210812_WeakScaling_STREAM_4 +| | | | | a00n_20210812_WeakScaling_STREAM_48 +| | | | | a00n_20210812_WeakScaling_STREAM_8 +a01l +## String representation of Job List [20] ## +a01l_20000101_fc0_1_1_DN ~ [2 children] +| a01l_20000101_fc0_1_1_OPA_2TMAX ~ [1 child] +| | a01l_20000101_fc0_1_1_URBAN +| a01l_20000101_fc0_1_1_OPA_2TMIN ~ [2 children] +| | a01l_20000101_fc0_1_1_MHM +a01l_20000101_fc0_1_2_DN ~ [2 children] +| a01l_20000101_fc0_1_2_OPA_2TMAX ~ [1 child] +| | a01l_20000101_fc0_1_2_URBAN +| a01l_20000101_fc0_1_2_OPA_2TMIN ~ [2 children] +| | a01l_20000101_fc0_1_2_MHM +a01l_20000101_fc0_2_1_DN ~ [2 children] +| a01l_20000101_fc0_2_1_OPA_2TMAX ~ [1 child] +| | a01l_20000101_fc0_2_1_URBAN +| a01l_20000101_fc0_2_1_OPA_2TMIN ~ [2 children] +| | a01l_20000101_fc0_2_1_MHM +a01l_20000101_fc0_2_2_DN ~ [2 children] +| a01l_20000101_fc0_2_2_OPA_2TMAX ~ [1 child] +| | a01l_20000101_fc0_2_2_URBAN +| a01l_20000101_fc0_2_2_OPA_2TMIN ~ [2 children] +| | a01l_20000101_fc0_2_2_MHM +a016 +## String representation of Job List [5] ## +a68z_20210811_CompilationEfficiency_PARAVER ~ [1 child] +| a68z_ADD_SECTION +a68z_20210811_HardwareBenchmarks_PARAVER +a68z_20210811_StrongScaling_PARAVER +a68z_20210811_WeakScaling_PARAVER +a00k +## String representation of Job List [1] ## +a00k_HELLO_WORLD +a00v +## String representation of Job List [9] ## +a00v_LOCAL_SETUP ~ [1 child] +| a00v_SYNCHRONIZE ~ [1 child] +| | a00v_REMOTE_SETUP ~ [1 child] +| | | a00v_20200120_fc0_INI ~ [1 child] +| | | | a00v_20200120_fc0_1_SIM ~ [1 child] +| | | | | a00v_20200120_fc0_2_SIM ~ [1 child] +| | | | | | a00v_20200120_fc0_3_SIM ~ [1 child] +| | | | | | | a00v_20200120_fc0_4_SIM ~ [1 child] +| | | | | | | | a00v_20200120_fc0_5_SIM +a01z +## String representation of Job List [8] ## +a01z_LOCAL_SETUP ~ [1 child] +| a01z_REMOTE_SETUP ~ [1 child] +| | a01z_20000101_fc0_INI ~ [1 child] +| | | a01z_20000101_fc0_1_SIM ~ [1 child] +| | | | a01z_20000101_fc0_2_SIM ~ [1 child] +| | | | | a01z_POST ~ [1 child] +| | | | | | a01z_CLEAN ~ [1 child] +| | | | | | | a01z_20000101_fc0_TRANSFER +a00l +## String representation of Job List [4] ## +a00l_20000101_fc0_1_SIM +a00l_20000101_fc0_2_SIM +a00l_20000101_fc0_3_SIM +a00l_20000101_fc0_4_SIM +a01u +## String representation of Job List [86] ## +a01u_CLEAN ~ [1 child] +| a01u_REMOTE_SETUP ~ [4 children] +| | a01u_20210811_CompilationEfficiency_REMOTE_INIDATA ~ [1 child] +| | | a01u_20210811_CompilationEfficiency_REMOTE_COMPILATION ~ [1 child] +| | | | a01u_20210811_CompilationEfficiency_PREPARE_TESTS ~ [9 children] +| | | | | a01u_20210811_CompilationEfficiency_FUNCTIONS_DIR ~ [8 children] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_O0_48 ~ [1 child] +| | | | | | | a01u_20210811_CompilationEfficiency_REPORT_SETUP ~ [1 child] +| | | | | | | | a01u_20210811_CompilationEfficiency_TRACE_CUT ~ [2 children] +| | | | | | | | | a01u_20210811_CompilationEfficiency_DIMEMAS_TRACES ~ [2 children] +| | | | | | | | | | a01u_20210811_CompilationEfficiency_DIMEMAS_IMAGES ~ [1 child] +| | | | | | | | | | | a01u_ADD_SECTION ~ [1 child] +| | | | | | | | | | | | a01u_REPORT +| | | | | | | | | | a01u_20210811_CompilationEfficiency_PARADIM ~ [1 child] +| | | | | | | | | | | a01u_20210811_CompilationEfficiency_CONFIGURATION_JSON ~ [1 child] +| | | | | | | | | | | | a01u_20210811_CompilationEfficiency_PARAVER ~ [1 child] +| | | | | | | | | a01u_20210811_CompilationEfficiency_PARAMEDIR ~ [1 child] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_O0_96 ~ [1 child] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_O1_48 ~ [1 child] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_O1_96 ~ [1 child] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_O3_48 ~ [1 child] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_O3_96 ~ [1 child] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_XHOST_48 ~ [1 child] +| | | | | | a01u_20210811_CompilationEfficiency_TRACE_XHOST_96 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_O0_48 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_O0_96 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_O1_48 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_O1_96 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_O3_48 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_O3_96 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_XHOST_48 ~ [1 child] +| | | | | a01u_20210811_CompilationEfficiency_SCALABILITY_XHOST_96 ~ [1 child] +| | a01u_20210811_HardwareBenchmarks_REMOTE_INIDATA ~ [1 child] +| | | a01u_20210811_HardwareBenchmarks_REMOTE_COMPILATION ~ [1 child] +| | | | a01u_20210811_HardwareBenchmarks_PREPARE_TESTS ~ [10 children] +| | | | | a01u_20210811_HardwareBenchmarks_OSU_RUN_192 ~ [1 child] +| | | | | | a01u_20210811_HardwareBenchmarks_REPORT_SETUP ~ [1 child] +| | | | | | | a01u_20210811_HardwareBenchmarks_HARDWARE_BENCH_IMAGES ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_OSU_RUN_48 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_OSU_RUN_96 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_STREAM_1 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_STREAM_16 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_STREAM_2 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_STREAM_32 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_STREAM_4 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_STREAM_48 ~ [1 child] +| | | | | a01u_20210811_HardwareBenchmarks_STREAM_8 ~ [1 child] +| | a01u_20210811_StrongScaling_REMOTE_INIDATA ~ [1 child] +| | | a01u_20210811_StrongScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a01u_20210811_StrongScaling_PREPARE_TESTS ~ [4 children] +| | | | | a01u_20210811_StrongScaling_FUNCTIONS_DIR ~ [3 children] +| | | | | | a01u_20210811_StrongScaling_TRACE_BENCH_189 ~ [1 child] +| | | | | | | a01u_20210811_StrongScaling_REPORT_SETUP ~ [1 child] +| | | | | | | | a01u_20210811_StrongScaling_TRACE_CUT ~ [3 children] +| | | | | | | | | a01u_20210811_StrongScaling_DIMEMAS_TRACES ~ [2 children] +| | | | | | | | | | a01u_20210811_StrongScaling_DIMEMAS_IMAGES ~ [1 child] +| | | | | | | | | | a01u_20210811_StrongScaling_PARADIM ~ [1 child] +| | | | | | | | | | | a01u_20210811_StrongScaling_CONFIGURATION_JSON ~ [1 child] +| | | | | | | | | | | | a01u_20210811_StrongScaling_PARAVER ~ [1 child] +| | | | | | | | | a01u_20210811_StrongScaling_MODELFACTORS ~ [1 child] +| | | | | | | | | a01u_20210811_StrongScaling_PARAMEDIR ~ [1 child] +| | | | | | a01u_20210811_StrongScaling_TRACE_BENCH_48 ~ [1 child] +| | | | | | a01u_20210811_StrongScaling_TRACE_BENCH_96 ~ [1 child] +| | | | | a01u_20210811_StrongScaling_SCALABILITY_BENCH_189 ~ [1 child] +| | | | | a01u_20210811_StrongScaling_SCALABILITY_BENCH_48 ~ [1 child] +| | | | | a01u_20210811_StrongScaling_SCALABILITY_BENCH_96 ~ [1 child] +| | a01u_20210811_WeakScaling_REMOTE_INIDATA ~ [1 child] +| | | a01u_20210811_WeakScaling_REMOTE_COMPILATION ~ [1 child] +| | | | a01u_20210811_WeakScaling_PREPARE_TESTS ~ [5 children] +| | | | | a01u_20210811_WeakScaling_FUNCTIONS_DIR ~ [4 children] +| | | | | | a01u_20210811_WeakScaling_TRACE_ORCA025_576 ~ [1 child] +| | | | | | | a01u_20210811_WeakScaling_REPORT_SETUP ~ [1 child] +| | | | | | | | a01u_20210811_WeakScaling_TRACE_CUT ~ [2 children] +| | | | | | | | | a01u_20210811_WeakScaling_DIMEMAS_TRACES ~ [2 children] +| | | | | | | | | | a01u_20210811_WeakScaling_DIMEMAS_IMAGES ~ [1 child] +| | | | | | | | | | a01u_20210811_WeakScaling_PARADIM ~ [1 child] +| | | | | | | | | | | a01u_20210811_WeakScaling_CONFIGURATION_JSON ~ [1 child] +| | | | | | | | | | | | a01u_20210811_WeakScaling_PARAVER ~ [1 child] +| | | | | | | | | a01u_20210811_WeakScaling_PARAMEDIR ~ [1 child] +| | | | | | a01u_20210811_WeakScaling_TRACE_ORCA025_740 ~ [1 child] +| | | | | | a01u_20210811_WeakScaling_TRACE_ORCA1_189 ~ [1 child] +| | | | | | a01u_20210811_WeakScaling_TRACE_ORCA1_48 ~ [1 child] +| | | | | a01u_20210811_WeakScaling_SCALABILITY_ORCA025_576 ~ [1 child] +| | | | | a01u_20210811_WeakScaling_SCALABILITY_ORCA025_740 ~ [1 child] +| | | | | a01u_20210811_WeakScaling_SCALABILITY_ORCA1_189 ~ [1 child] +| | | | | a01u_20210811_WeakScaling_SCALABILITY_ORCA1_48 ~ [1 child] +a005 +## String representation of Job List [56] ## +a005_20200120_fc0_1_1_DN ~ [2 children] +| a005_20200120_fc0_1_1_OPA ~ [1 child] +| | a005_20200120_fc0_1_APP ~ [1 child] +| | | a005_20200120_fc0_2_APP ~ [1 child] +| | | | a005_20200120_fc0_3_APP ~ [1 child] +| | | | | a005_20200120_fc0_4_APP +| a005_20200120_fc0_1_2_OPA ~ [1 child] +a005_20200120_fc0_1_2_DN ~ [2 children] +| a005_20200120_fc0_1_3_OPA ~ [1 child] +| a005_20200120_fc0_1_4_OPA ~ [1 child] +a005_20200120_fc0_2_1_DN ~ [2 children] +| a005_20200120_fc0_2_1_OPA ~ [1 child] +| a005_20200120_fc0_2_2_OPA ~ [1 child] +a005_20200120_fc0_2_2_DN ~ [2 children] +| a005_20200120_fc0_2_3_OPA ~ [1 child] +| a005_20200120_fc0_2_4_OPA ~ [1 child] +a005_20200120_fc0_3_1_DN ~ [2 children] +| a005_20200120_fc0_3_1_OPA ~ [1 child] +| a005_20200120_fc0_3_2_OPA ~ [1 child] +a005_20200120_fc0_3_2_DN ~ [2 children] +| a005_20200120_fc0_3_3_OPA ~ [1 child] +| a005_20200120_fc0_3_4_OPA ~ [1 child] +a005_20200120_fc0_4_1_DN ~ [2 children] +| a005_20200120_fc0_4_1_OPA ~ [1 child] +| a005_20200120_fc0_4_2_OPA ~ [1 child] +a005_20200120_fc0_4_2_DN ~ [2 children] +| a005_20200120_fc0_4_3_OPA ~ [1 child] +| a005_20200120_fc0_4_4_OPA ~ [1 child] +a005_20200120_fc1_1_1_DN ~ [2 children] +| a005_20200120_fc1_1_1_OPA ~ [1 child] +| | a005_20200120_fc1_1_APP ~ [1 child] +| | | a005_20200120_fc1_2_APP ~ [1 child] +| | | | a005_20200120_fc1_3_APP ~ [1 child] +| | | | | a005_20200120_fc1_4_APP +| a005_20200120_fc1_1_2_OPA ~ [1 child] +a005_20200120_fc1_1_2_DN ~ [2 children] +| a005_20200120_fc1_1_3_OPA ~ [1 child] +| a005_20200120_fc1_1_4_OPA ~ [1 child] +a005_20200120_fc1_2_1_DN ~ [2 children] +| a005_20200120_fc1_2_1_OPA ~ [1 child] +| a005_20200120_fc1_2_2_OPA ~ [1 child] +a005_20200120_fc1_2_2_DN ~ [2 children] +| a005_20200120_fc1_2_3_OPA ~ [1 child] +| a005_20200120_fc1_2_4_OPA ~ [1 child] +a005_20200120_fc1_3_1_DN ~ [2 children] +| a005_20200120_fc1_3_1_OPA ~ [1 child] +| a005_20200120_fc1_3_2_OPA ~ [1 child] +a005_20200120_fc1_3_2_DN ~ [2 children] +| a005_20200120_fc1_3_3_OPA ~ [1 child] +| a005_20200120_fc1_3_4_OPA ~ [1 child] +a005_20200120_fc1_4_1_DN ~ [2 children] +| a005_20200120_fc1_4_1_OPA ~ [1 child] +| a005_20200120_fc1_4_2_OPA ~ [1 child] +a005_20200120_fc1_4_2_DN ~ [2 children] +| a005_20200120_fc1_4_3_OPA ~ [1 child] +| a005_20200120_fc1_4_4_OPA ~ [1 child] +a00x +## String representation of Job List [8] ## +a00x_LOCAL_SETUP ~ [1 child] +| a00x_REMOTE_SETUP ~ [1 child] +| | a00x_20000101_fc0_INI ~ [1 child] +| | | a00x_20000101_fc0_1_SIM ~ [1 child] +| | | | a00x_20000101_fc0_2_SIM ~ [1 child] +| | | | | a00x_POST ~ [1 child] +| | | | | | a00x_CLEAN ~ [1 child] +| | | | | | | a00x_20000101_fc0_TRANSFER +a02a +## String representation of Job List [29] ## +a02a_LOCAL_SETUP ~ [4 children] +| a02a_20120101_1_LOCAL_SEND_INITIAL_DA ~ [1 child] +| | a02a_20120101_2_LOCAL_SEND_INITIAL_DA ~ [1 child] +| | | a02a_20120101_3_LOCAL_SEND_INITIAL_DA ~ [1 child] +| | | | a02a_20120101_1_DA ~ [3 children] +| | | | | a02a_20120101_000_2_SIM ~ [1 child] +| | | | | | a02a_20120101_2_DA ~ [2 children] +| | | | | | | a02a_20120101_000_3_SIM ~ [1 child] +| | | | | | | | a02a_20120101_3_DA +| | | | | | | a02a_20120101_001_3_SIM ~ [1 child] +| | | | | a02a_20120101_001_2_SIM ~ [1 child] +| | | | | a02a_20120201_1_DA ~ [2 children] +| | | | | | a02a_20120201_000_2_SIM ~ [1 child] +| | | | | | | a02a_20120201_2_DA ~ [2 children] +| | | | | | | | a02a_20120201_000_3_SIM ~ [1 child] +| | | | | | | | | a02a_20120201_3_DA +| | | | | | | | a02a_20120201_001_3_SIM ~ [1 child] +| | | | | | a02a_20120201_001_2_SIM ~ [1 child] +| a02a_20120201_1_LOCAL_SEND_INITIAL_DA ~ [1 child] +| | a02a_20120201_2_LOCAL_SEND_INITIAL_DA ~ [1 child] +| | | a02a_20120201_3_LOCAL_SEND_INITIAL_DA ~ [1 child] +| a02a_LOCAL_SEND_SOURCE ~ [2 children] +| | a02a_COMPILE_DA ~ [1 child] +| | a02a_REMOTE_COMPILE ~ [4 children] +| | | a02a_20120101_000_1_SIM ~ [1 child] +| | | a02a_20120101_001_1_SIM ~ [1 child] +| | | a02a_20120201_000_1_SIM ~ [1 child] +| | | a02a_20120201_001_1_SIM ~ [1 child] +| a02a_LOCAL_SEND_STATIC ~ [4 children] diff --git a/test/regression/local_asparser_test.py b/test/regression/local_asparser_test.py index b3f77a066cdd5886d0318df0b5c675ca36347eed..7eebd0c2ca6c125ab85c935abacfc048f493e19a 100644 --- a/test/regression/local_asparser_test.py +++ b/test/regression/local_asparser_test.py @@ -90,6 +90,7 @@ CONFIG.AUTOSUBMIT_VERSION=4.0.0b break print(sucess) print(error) + print("Testing EXPID a009: Config in a external file") perform_test("a009") print("Testing EXPID a00a: Config in the minimal file") diff --git a/test/regression/local_asparser_test_4.1.py b/test/regression/local_asparser_test_4.1.py new file mode 100644 index 0000000000000000000000000000000000000000..93edaba45bf41d2d1ac4e3729c97311523bfbb62 --- /dev/null +++ b/test/regression/local_asparser_test_4.1.py @@ -0,0 +1,95 @@ +""" +This test checks that the autosubmit report command works as expected. +It is a regression test, so it is not run by default. +It only run within my home desktop computer. It is not run in the CI. Eventually it will be included TODO +Just to be sure that the autosubmitconfigparser work as expected if there are changes. +""" + +import subprocess +import os +from pathlib import Path +BIN_PATH = '../../bin' + + +def check_cmd(command, path=BIN_PATH): + try: + output = subprocess.check_output(os.path.join(path, command), shell=True, stderr=subprocess.STDOUT) + error = False + except subprocess.CalledProcessError as e: + output = e.output + error = True + return output, error + +def report_test(expid): + output = check_cmd("autosubmit report {0} -all -v".format(expid)) + return output +def perform_test(expid): + + output,error = report_test(expid) + if error: + print("ERR: autosubmit report command failed") + print(output.decode("UTF-8")) + exit(0) + report_file = output.decode("UTF-8").split("list of all parameters has been written on ")[1] + report_file = report_file.split(".txt")[0] + ".txt" + list_of_parameters_to_find = """ +DEFAULT.CUSTOM_CONFIG.PRE +DEFAULT.CUSTOM_CONFIG.POST +DIRECTORIES.INDIR +DIRECTORIES.OUTDIR +DIRECTORIES.TESTDIR +TESTKEY +TESTKEY-TWO +TESTKEY-LEVANTE +PLATFORMS.LEVANTE-LOGIN.USER +PLATFORMS.LEVANTE-LOGIN.PROJECT +PLATFORMS.LEVANTE.USER +PLATFORMS.LEVANTE.PROJECT +DIRECTORIES.TEST_FILE +PROJECT.PROJECT_TYPE +PROJECT.PROJECT_DESTINATION +TOLOAD +TOLOAD2 +CONFIG.AUTOSUBMIT_VERSION + """.split("\n") + expected_output =""" +DIRECTORIES.INDIR=my-updated-indir +DIRECTORIES.OUTDIR=from_main +DIRECTORIES.TEST_FILE=from_main +DIRECTORIES.TESTDIR=another-dir +TESTKEY=abcd +TESTKEY-TWO=HPCARCH is levante +TESTKEY-LEVANTE=L-abcd +PLATFORMS.LEVANTE-LOGIN.USER=b382351 +PLATFORMS.LEVANTE-LOGIN.PROJECT=bb1153 +PLATFORMS.LEVANTE.USER=b382351 +PLATFORMS.LEVANTE.PROJECT=bb1153 +PROJECT.PROJECT_TYPE=none +PROJECT.PROJECT_DESTINATION=auto-icon +TOLOAD=from_testfile2 +TOLOAD2=from_version +CONFIG.AUTOSUBMIT_VERSION=4.1.0b + """.split("\n") + if Path(report_file).exists(): + print("OK: report file exists") + else: + print("ERR: report file does not exist") + exit(0) + sucess="" + error="" + for line in Path(report_file).read_text().split("\n"): + if line.split("=")[0] in list_of_parameters_to_find[1:-1]: + if line in expected_output: + sucess +="OK: " + line + "\n" + else: + for error_line in expected_output: + if line.split("=")[0] in error_line: + error += "ERR: " + line + " EXPECTED: " + error_line + "\n" + break + print(sucess) + print(error) + +print("Testing EXPID a01p copy of a009: Config in a external file") +perform_test("a01p") +print("Testing EXPID a01q copy of a00a: Config in the minimal file") +perform_test("a01q") \ No newline at end of file diff --git a/test/regression/local_check_details.py b/test/regression/local_check_details.py new file mode 100644 index 0000000000000000000000000000000000000000..ad757806369bea9702c6ac27cce5bdb24303d5a6 --- /dev/null +++ b/test/regression/local_check_details.py @@ -0,0 +1,71 @@ +""" +This test took the now ordered by name -d option of autosubmit create and checks that the workflow of 4.1 and 4.0 match. +Works under local_computer TODO introduce in CI +""" + +# Check: a014, a016 + + +import os +import subprocess +BIN_PATH = '../../bin' +ACTIVE_DOCS = True # Use autosubmit_docs database +VERSION = 4.1 # 4.0 or 4.1 + +if ACTIVE_DOCS: + EXPERIMENTS_PATH = '/home/dbeltran/autosubmit_docs' + FILE_NAME = f"{VERSION}_docs_test.txt" + BANNED_TESTS = [] +else: + EXPERIMENTS_PATH = '/home/dbeltran/new_autosubmit' + FILE_NAME = f"{VERSION}_multi_test.txt" + BANNED_TESTS = ["a02j","t002","a006","a00s","a029","a00z","a02l","a026","a012","a018","a02f","t000","a02d","a02i","a025","a02e","a02h","a02b","a023","a02k","a02c"] + +def check_cmd(command, path=BIN_PATH): + try: + output = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT) + error = False + except subprocess.CalledProcessError as e: + output = e.output + error = True + return output, error + +def run_test(expid): + if VERSION == 4.0: + output = check_cmd(f"../../bin/autosubmit create {expid} -np -v -d -cw;") + else: + output = check_cmd(f"../../bin/autosubmit create {expid} -np -v -d -cw -f;") + return output +def perform_test(expids): + to_exclude = [] + for expid in expids: + try: + output,error = run_test(expid) + # output to str + output = output.decode("UTF-8") + output = output.split("Job list created successfully")[1] + output = expid + output + # put it in a single file + with open(f"{FILE_NAME}", "a") as myfile: + myfile.write(output) + except: + to_exclude.append(expid) + # print to_exclude in format ["a001","a002"] + print(to_exclude) + + +open(f"{FILE_NAME}", "w").close() + +# list all experiments under ~/new_autosubmit. +# except the excluded ones, which are not run +expids = [] +#excluded = ['a026', 'a01y', 'a00j', 'a020', 'a01t', 'a00q', 'a00f', 'a01h', 'a00o', 'a01c', 'a00z', 't008', 'a00y', 'a00r', 't009', 'a000', 'a01e', 'a01i', 'a002', 'a008', 'a010', 'a003', 't007', 'a01d', 'autosubmit.db', 'a021', 'a00h', 'as_times.db', 'a04d', 'a02v'] +excluded = [] + +for experiment in os.listdir(f"{EXPERIMENTS_PATH}"): + if ( experiment.startswith("a") or experiment.startswith("t") ) and len(experiment) == 4: + if experiment not in BANNED_TESTS: + expids.append(experiment) +# Force +# expids = ["a001"] +perform_test(expids) \ No newline at end of file diff --git a/test/regression/local_check_details_wrapper.py b/test/regression/local_check_details_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..7165889eaf64e76c4e8b1b686237f9dbaffbacb8 --- /dev/null +++ b/test/regression/local_check_details_wrapper.py @@ -0,0 +1,54 @@ +""" +This test took the now ordered by name -d option of autosubmit create and checks that the workflow of 4.1 and 4.0 match. +Works under local_computer TODO introduce in CI +""" + +import os +import subprocess +BIN_PATH = '../../bin' +VERSION = 4.1 + +def check_cmd(command, path=BIN_PATH): + try: + output = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT) + error = False + except subprocess.CalledProcessError as e: + output = e.output + error = True + return output, error + +def run_test(expid): + #check_cmd(f"rm -r /home/dbeltran/new_autosubmit/{expid}/tmp/LOG_{expid}/*") + output = check_cmd(f"../../bin/autosubmit create {expid} -np -v -d -cw;") + return output +def perform_test(expids): + to_exclude = [] + + for expid in expids: + try: + output,error = run_test(expid) + # output to str + output = output.decode("UTF-8") + output = output.split("Job list created successfully")[1] + output = expid + output + # put it in a single file + with open(f"{VERSION}_multi_test.txt", "a") as myfile: + myfile.write(output) + except: + raise Exception(f"Error in {expid}") + + # print to_exclude in format ["a001","a002"] + print(to_exclude) + + +open(f"{VERSION}_multi_test.txt", "w").close() + +# list all experiments under ~/new_autosubmit. +# except the excluded ones, which are not run +expids = [] +excluded = ['a01y', 'a00j', 'a020', 'a01t', 'a00q', 'a00f', 'a01h', 'a00o', 'a01c', 'a00z', 't008', 'a00y', 'a00r', 't009', 'a000', 'a01e', 'a01i', 'a002', 'a008', 'a010', 'a003', 't007', 'a01d', 'autosubmit.db', 'a021', 'a00h', 'as_times.db', 'a04d', 'a02v'] +for experiment in os.listdir("/home/dbeltran/new_autosubmit"): + if experiment.startswith("a") or experiment.startswith("t") and len(experiment) == 4: + if experiment not in excluded: + expids.append(experiment) +perform_test(expids) \ No newline at end of file diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index e787f4e5133bf8e4ef2866830da6117f6d5aef7d..21938bec043d2418cd0e031dcf56d3bfbddaad66 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock + import copy import inspect import mock @@ -6,6 +8,7 @@ import unittest from copy import deepcopy from datetime import datetime +from autosubmit.job.job_dict import DicJobs from autosubmit.job.job import Job from autosubmit.job.job_common import Status from autosubmit.job.job_list import JobList @@ -16,6 +19,7 @@ from autosubmitconfigparser.config.yamlparser import YAMLParserFactory class FakeBasicConfig: def __init__(self): pass + def props(self): pr = {} for name in dir(self): @@ -23,6 +27,7 @@ class FakeBasicConfig: if not name.startswith('__') and not inspect.ismethod(value) and not inspect.isfunction(value): pr[name] = value return pr + DB_DIR = '/dummy/db/dir' DB_FILE = '/dummy/db/file' DB_PATH = '/dummy/db/path' @@ -32,6 +37,7 @@ class FakeBasicConfig: DEFAULT_PLATFORMS_CONF = '' DEFAULT_JOBS_CONF = '' + class TestJobList(unittest.TestCase): def setUp(self): self.experiment_id = 'random-id' @@ -42,8 +48,9 @@ class TestJobList(unittest.TestCase): self.as_conf.experiment_data["PLATFORMS"] = dict() self.temp_directory = tempfile.mkdtemp() self.JobList = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), - JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) - self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] + JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) + self.date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", + "20020208", "20020209", "20020210"] self.member_list = ["fc1", "fc2", "fc3", "fc4", "fc5", "fc6", "fc7", "fc8", "fc9", "fc10"] self.chunk_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] self.split_list = [1, 2, 3, 4, 5] @@ -52,97 +59,97 @@ class TestJobList(unittest.TestCase): self.JobList._chunk_list = self.chunk_list self.JobList._split_list = self.split_list - # Define common test case inputs here self.relationships_dates = { - "DATES_FROM": { - "20020201": { - "MEMBERS_FROM": { - "fc2": { - "DATES_TO": "[20020201:20020202]*,20020203", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all" - } - }, - "SPLITS_FROM": { - "ALL": { - "SPLITS_TO": "1" - } + "DATES_FROM": { + "20020201": { + "MEMBERS_FROM": { + "fc2": { + "DATES_TO": "[20020201:20020202]*,20020203", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all" + } + }, + "SPLITS_FROM": { + "ALL": { + "SPLITS_TO": "1" } } } } + } self.relationships_dates_optional = deepcopy(self.relationships_dates) - self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { "fc2?": { "DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "all", "SPLITS_TO": "5" } } - self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = { "ALL": { "SPLITS_TO": "1?" } } + self.relationships_dates_optional["DATES_FROM"]["20020201"]["MEMBERS_FROM"] = { + "fc2?": {"DATES_TO": "20020201", "MEMBERS_TO": "fc2", "CHUNKS_TO": "all", "SPLITS_TO": "5"}} + self.relationships_dates_optional["DATES_FROM"]["20020201"]["SPLITS_FROM"] = {"ALL": {"SPLITS_TO": "1?"}} self.relationships_members = { - "MEMBERS_FROM": { - "fc2": { - "SPLITS_FROM": { - "ALL": { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "MEMBERS_FROM": { + "fc2": { + "SPLITS_FROM": { + "ALL": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" } } } } + } self.relationships_chunks = { - "CHUNKS_FROM": { - "1": { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "CHUNKS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" } } + } self.relationships_chunks2 = { - "CHUNKS_FROM": { - "1": { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - }, - "2": { - "SPLITS_FROM": { - "5": { - "SPLITS_TO": "2" - } + "CHUNKS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + }, + "2": { + "SPLITS_FROM": { + "5": { + "SPLITS_TO": "2" } } } } + } self.relationships_splits = { - "SPLITS_FROM": { - "1": { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "SPLITS_FROM": { + "1": { + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" } } + } self.relationships_general = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } self.relationships_general_1_to_1 = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1*,2*,3*,4*,5*" - } + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1*,2*,3*,4*,5*" + } # Create a mock Job object - self.mock_job = mock.MagicMock(spec=Job) + self.mock_job = Mock(wraps=Job) # Set the attributes on the mock object self.mock_job.name = "Job1" @@ -196,16 +203,16 @@ class TestJobList(unittest.TestCase): def test_parse_filters_to_check(self): """Test the _parse_filters_to_check function""" - result = self.JobList._parse_filters_to_check("20020201,20020202,20020203",self.date_list) - expected_output = ["20020201","20020202","20020203"] + result = self.JobList._parse_filters_to_check("20020201,20020202,20020203", self.date_list) + expected_output = ["20020201", "20020202", "20020203"] self.assertEqual(result, expected_output) - result = self.JobList._parse_filters_to_check("20020201,[20020203:20020205]",self.date_list) - expected_output = ["20020201","20020203","20020204","20020205"] + result = self.JobList._parse_filters_to_check("20020201,[20020203:20020205]", self.date_list) + expected_output = ["20020201", "20020203", "20020204", "20020205"] self.assertEqual(result, expected_output) - result = self.JobList._parse_filters_to_check("[20020201:20020203],[20020205:20020207]",self.date_list) - expected_output = ["20020201","20020202","20020203","20020205","20020206","20020207"] + result = self.JobList._parse_filters_to_check("[20020201:20020203],[20020205:20020207]", self.date_list) + expected_output = ["20020201", "20020202", "20020203", "20020205", "20020206", "20020207"] self.assertEqual(result, expected_output) - result = self.JobList._parse_filters_to_check("20020201",self.date_list) + result = self.JobList._parse_filters_to_check("20020201", self.date_list) expected_output = ["20020201"] self.assertEqual(result, expected_output) @@ -215,44 +222,43 @@ class TestJobList(unittest.TestCase): # a range: [0:], [:N], [0:N], [:-1], [0:N:M] ... # a value: N # a range with step: [0::M], [::2], [0::3], [::3] ... - result = self.JobList._parse_filter_to_check("20020201",self.date_list) + result = self.JobList._parse_filter_to_check("20020201", self.date_list) expected_output = ["20020201"] self.assertEqual(result, expected_output) - result = self.JobList._parse_filter_to_check("[20020201:20020203]",self.date_list) - expected_output = ["20020201","20020202","20020203"] + result = self.JobList._parse_filter_to_check("[20020201:20020203]", self.date_list) + expected_output = ["20020201", "20020202", "20020203"] self.assertEqual(result, expected_output) - result = self.JobList._parse_filter_to_check("[20020201:20020203:2]",self.date_list) - expected_output = ["20020201","20020203"] + result = self.JobList._parse_filter_to_check("[20020201:20020203:2]", self.date_list) + expected_output = ["20020201", "20020203"] self.assertEqual(result, expected_output) - result = self.JobList._parse_filter_to_check("[20020202:]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020202:]", self.date_list) expected_output = self.date_list[1:] self.assertEqual(result, expected_output) - result = self.JobList._parse_filter_to_check("[:20020203]",self.date_list) + result = self.JobList._parse_filter_to_check("[:20020203]", self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) - result = self.JobList._parse_filter_to_check("[::2]",self.date_list) + result = self.JobList._parse_filter_to_check("[::2]", self.date_list) expected_output = self.date_list[::2] self.assertEqual(result, expected_output) - result = self.JobList._parse_filter_to_check("[20020203::]",self.date_list) + result = self.JobList._parse_filter_to_check("[20020203::]", self.date_list) expected_output = self.date_list[2:] self.assertEqual(result, expected_output) - result = self.JobList._parse_filter_to_check("[:20020203:]",self.date_list) + result = self.JobList._parse_filter_to_check("[:20020203:]", self.date_list) expected_output = self.date_list[:3] self.assertEqual(result, expected_output) # test with a member N:N - result = self.JobList._parse_filter_to_check("[fc2:fc3]",self.member_list) - expected_output = ["fc2","fc3"] + result = self.JobList._parse_filter_to_check("[fc2:fc3]", self.member_list) + expected_output = ["fc2", "fc3"] self.assertEqual(result, expected_output) # test with a chunk - result = self.JobList._parse_filter_to_check("[1:2]",self.chunk_list,level_to_check="CHUNKS_FROM") - expected_output = [1,2] + result = self.JobList._parse_filter_to_check("[1:2]", self.chunk_list, level_to_check="CHUNKS_FROM") + expected_output = [1, 2] self.assertEqual(result, expected_output) # test with a split - result = self.JobList._parse_filter_to_check("[1:2]",self.split_list,level_to_check="SPLITS_FROM") - expected_output = [1,2] + result = self.JobList._parse_filter_to_check("[1:2]", self.split_list, level_to_check="SPLITS_FROM") + expected_output = [1, 2] self.assertEqual(result, expected_output) - def test_check_dates(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") @@ -261,18 +267,17 @@ class TestJobList(unittest.TestCase): self.mock_job.split = 1 result = self.JobList._check_dates(self.relationships_dates, self.mock_job) expected_output = { - "DATES_TO": "20020201*,20020202*,20020203", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "DATES_TO": "20020201*,20020202*,20020203", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } self.assertEqual(result, expected_output) # failure self.mock_job.date = datetime.strptime("20020301", "%Y%m%d") result = self.JobList._check_dates(self.relationships_dates, self.mock_job) self.assertEqual(result, {}) - def test_check_members(self): # Call the function to get the result self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") @@ -280,11 +285,11 @@ class TestJobList(unittest.TestCase): result = self.JobList._check_members(self.relationships_members, self.mock_job) expected_output = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } self.assertEqual(result, expected_output) self.mock_job.member = "fc3" result = self.JobList._check_members(self.relationships_members, self.mock_job) @@ -294,18 +299,17 @@ class TestJobList(unittest.TestCase): result = self.JobList._check_members(self.relationships_members, self.mock_job) self.assertEqual(result, {}) - def test_check_splits(self): # Call the function to get the result self.mock_job.split = 1 result = self.JobList._check_splits(self.relationships_splits, self.mock_job) expected_output = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } self.assertEqual(result, expected_output) self.mock_job.split = 2 result = self.JobList._check_splits(self.relationships_splits, self.mock_job) @@ -321,11 +325,11 @@ class TestJobList(unittest.TestCase): self.mock_job.chunk = 1 result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) expected_output = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } self.assertEqual(result, expected_output) self.mock_job.chunk = 2 result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) @@ -335,9 +339,6 @@ class TestJobList(unittest.TestCase): result = self.JobList._check_chunks(self.relationships_chunks, self.mock_job) self.assertEqual(result, {}) - - - def test_check_general(self): # Call the function to get the result @@ -345,246 +346,94 @@ class TestJobList(unittest.TestCase): self.mock_job.member = "fc2" self.mock_job.chunk = 1 self.mock_job.split = 1 - result = self.JobList._filter_current_job(self.mock_job,self.relationships_general) + result = self.JobList._filter_current_job(self.mock_job, self.relationships_general) expected_output = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } + "DATES_TO": "20020201", + "MEMBERS_TO": "fc2", + "CHUNKS_TO": "all", + "SPLITS_TO": "1" + } self.assertEqual(result, expected_output) - - def test_valid_parent(self): - - # Call the function to get the result - date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] - member_list = ["fc1", "fc2", "fc3"] - chunk_list = [1, 2, 3] - self.mock_job.splits = 10 - is_a_natural_relation = False - # Filter_to values - filter_ = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } - # PArent job values - self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") - self.mock_job.member = "fc2" - self.mock_job.chunk = 1 - self.mock_job.split = 1 - child = copy.deepcopy(self.mock_job) - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - # it returns a tuple, the first element is the result, the second is the optional flag - self.assertEqual(result, True) - filter_ = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1?" - } - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - filter_ = { - "DATES_TO": "20020201", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1?" - } - self.mock_job.split = 2 - - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - filter_ = { - "DATES_TO": "[20020201:20020205]", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - filter_ = { - "DATES_TO": "[20020201:20020205]", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "all", - "SPLITS_TO": "1" - } - self.mock_job.date = datetime.strptime("20020206", "%Y%m%d") - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - filter_ = { - "DATES_TO": "[20020201:20020205]", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "[2:4]", - "SPLITS_TO": "[1:5]" - } - self.mock_job.date = datetime.strptime("20020201", "%Y%m%d") - self.mock_job.chunk = 2 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - - - def test_valid_parent_1_to_1(self): - child = copy.deepcopy(self.mock_job) - child.splits = 6 - - date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] - member_list = ["fc1", "fc2", "fc3"] - chunk_list = [1, 2, 3] - is_a_natural_relation = False - - # Test 1_to_1 - filter_ = { - "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "1,2,3,4,5,6", - "SPLITS_TO": "1*,2*,3*,4*,5*,6" - } - self.mock_job.splits = 6 - self.mock_job.split = 1 - self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") - self.mock_job.chunk = 5 - child.split = 1 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - child.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - - def test_valid_parent_1_to_n(self): - self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") - self.mock_job.chunk = 5 - child = copy.deepcopy(self.mock_job) - child.splits = 4 - self.mock_job.splits = 2 - - date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] - member_list = ["fc1", "fc2", "fc3"] - chunk_list = [1, 2, 3] - is_a_natural_relation = False - - # Test 1_to_N - filter_ = { - "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "1,2,3,4,5,6", - "SPLITS_TO": "1*\\2,2*\\2" - } - child.split = 1 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - child.split = 2 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - child.split = 3 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - child.split = 4 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - - child.split = 1 - self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - child.split = 2 - self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - child.split = 3 - self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - child.split = 4 - self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - - def test_valid_parent_n_to_1(self): - self.mock_job.date = datetime.strptime("20020204", "%Y%m%d") - self.mock_job.chunk = 5 - child = copy.deepcopy(self.mock_job) - child.splits = 2 - self.mock_job.splits = 4 - - date_list = ["20020201", "20020202", "20020203", "20020204", "20020205", "20020206", "20020207", "20020208", "20020209", "20020210"] - member_list = ["fc1", "fc2", "fc3"] - chunk_list = [1, 2, 3] - is_a_natural_relation = False - - # Test N_to_1 - filter_ = { - "DATES_TO": "[20020201:20020202],20020203,20020204,20020205", - "MEMBERS_TO": "fc2", - "CHUNKS_TO": "1,2,3,4,5,6", - "SPLITS_TO": "1*\\2,2*\\2,3*\\2,4*\\2" - } - child.split = 1 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - child.split = 1 - self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - child.split = 1 - self.mock_job.split = 3 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - child.split = 1 - self.mock_job.split = 4 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - - child.split = 2 - self.mock_job.split = 1 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - child.split = 2 - self.mock_job.split = 2 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, False) - child.split = 2 - self.mock_job.split = 3 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - child.split = 2 - self.mock_job.split = 4 - result = self.JobList._valid_parent(self.mock_job, member_list, date_list, chunk_list, is_a_natural_relation, filter_,child) - self.assertEqual(result, True) - def test_check_relationship(self): - relationships = {'MEMBERS_FROM': {'TestMember, TestMember2,TestMember3 ': {'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}}} + relationships = {'MEMBERS_FROM': { + 'TestMember, TestMember2,TestMember3 ': {'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, + 'MEMBERS_TO': 'None', 'STATUS': None}}} level_to_check = "MEMBERS_FROM" value_to_check = "TestMember" result = self.JobList._check_relationship(relationships, level_to_check, value_to_check) - expected_output = [{'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] + expected_output = [ + {'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] self.assertEqual(result, expected_output) value_to_check = "TestMember2" result = self.JobList._check_relationship(relationships, level_to_check, value_to_check) - expected_output = [{'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] + expected_output = [ + {'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] self.assertEqual(result, expected_output) value_to_check = "TestMember3" result = self.JobList._check_relationship(relationships, level_to_check, value_to_check) - expected_output = [{'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] + expected_output = [ + {'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] self.assertEqual(result, expected_output) value_to_check = "TestMember " result = self.JobList._check_relationship(relationships, level_to_check, value_to_check) - expected_output = [{'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] + expected_output = [ + {'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] self.assertEqual(result, expected_output) value_to_check = " TestMember" - result = self.JobList._check_relationship(relationships,level_to_check,value_to_check ) - expected_output = [{'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] + result = self.JobList._check_relationship(relationships, level_to_check, value_to_check) + expected_output = [ + {'CHUNKS_TO': 'None', 'DATES_TO': 'None', 'FROM_STEP': None, 'MEMBERS_TO': 'None', 'STATUS': None}] self.assertEqual(result, expected_output) + def test_add_special_conditions(self): + # Method from job_list + job = Job("child", 1, Status.READY, 1) + job.section = "child_one" + job.date = datetime.strptime("20200128", "%Y%m%d") + job.member = "fc0" + job.chunk = 1 + job.split = 1 + job.splits = 1 + job.max_checkpoint_step = 0 + special_conditions = {"STATUS": "RUNNING", "FROM_STEP": "2"} + only_marked_status = False + filters_to_apply = {"DATES_TO": "all", "MEMBERS_TO": "all", "CHUNKS_TO": "all", "SPLITS_TO": "all"} + parent = Job("parent", 1, Status.READY, 1) + parent.section = "parent_one" + parent.date = datetime.strptime("20200128", "%Y%m%d") + parent.member = "fc0" + parent.chunk = 1 + parent.split = 1 + parent.splits = 1 + parent.max_checkpoint_step = 0 + job.status = Status.READY + job_list = Mock(wraps=self.JobList) + job_list._job_list = [job, parent] + job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent) + # self.JobList.jobs_edges + # job.edges = self.JobList.jobs_edges[job.name] + # assert + self.assertEqual(job.max_checkpoint_step, 2) + value = job.edge_info.get("RUNNING", "").get("parent", ()) + self.assertEqual((value[0].name, value[1]), (parent.name, "2")) + self.assertEqual(len(job.edge_info.get("RUNNING", "")), 1) + + self.assertEqual(str(job_list.jobs_edges.get("RUNNING", ())), str({job})) + only_marked_status = False + parent2 = Job("parent2", 1, Status.READY, 1) + parent2.section = "parent_two" + parent2.date = datetime.strptime("20200128", "%Y%m%d") + parent2.member = "fc0" + parent2.chunk = 1 + + job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent2) + value = job.edge_info.get("RUNNING", "").get("parent2", ()) + self.assertEqual(len(job.edge_info.get("RUNNING", "")), 2) + self.assertEqual((value[0].name, value[1]), (parent2.name, "2")) + self.assertEqual(str(job_list.jobs_edges.get("RUNNING", ())), str({job})) + job_list.add_special_conditions(job, special_conditions, filters_to_apply, parent2) + self.assertEqual(len(job.edge_info.get("RUNNING", "")), 2) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/test_dic_jobs.py b/test/unit/test_dic_jobs.py index fd8b459d72c2a9c168e62f2112b8d75ebec75b7d..bf5360070ab9694ad261e7b847a87d701a460439 100644 --- a/test/unit/test_dic_jobs.py +++ b/test/unit/test_dic_jobs.py @@ -1,3 +1,5 @@ +from bscearth.utils.date import date2str + from datetime import datetime from unittest import TestCase @@ -5,19 +7,25 @@ from mock import Mock import math import shutil import tempfile + +from autosubmit.job.job import Job from autosubmitconfigparser.config.yamlparser import YAMLParserFactory from autosubmit.job.job_common import Status from autosubmit.job.job_common import Type from autosubmit.job.job_dict import DicJobs from autosubmit.job.job_list import JobList from autosubmit.job.job_list_persistence import JobListPersistenceDb +from unittest.mock import patch class TestDicJobs(TestCase): def setUp(self): self.experiment_id = 'random-id' self.as_conf = Mock() + self.as_conf.experiment_data = dict() + self.as_conf.experiment_data["DEFAULT"] = {} + self.as_conf.experiment_data["DEFAULT"]["EXPID"] = self.experiment_id self.as_conf.experiment_data["JOBS"] = dict() self.as_conf.jobs_data = self.as_conf.experiment_data["JOBS"] self.as_conf.experiment_data["PLATFORMS"] = dict() @@ -32,14 +40,17 @@ class TestDicJobs(TestCase): self.chunk_list = list(range(1, self.num_chunks + 1)) self.date_format = 'H' self.default_retrials = 999 - self.dictionary = DicJobs(self.job_list,self.date_list, self.member_list, self.chunk_list, - self.date_format, self.default_retrials,self.as_conf.jobs_data,self.as_conf) + self.dictionary = DicJobs(self.date_list, self.member_list, self.chunk_list, self.date_format, default_retrials=self.default_retrials,as_conf=self.as_conf) + self.dictionary.changes = {} def tearDown(self) -> None: shutil.rmtree(self.temp_directory) - - def test_read_section_running_once_create_jobs_once(self): + @patch('autosubmit.job.job_dict.date2str') + def test_read_section_running_once_create_jobs_once(self, mock_date2str): # arrange + mock_date2str.side_effect = lambda x, y: str(x) + self.dictionary.compare_section = Mock() + section = 'fake-section' priority = 999 frequency = 123 @@ -62,18 +73,22 @@ class TestDicJobs(TestCase): self.dictionary._create_jobs_startdate = Mock() self.dictionary._create_jobs_member = Mock() self.dictionary._create_jobs_chunk = Mock() + self.dictionary.compare_section = Mock() # act self.dictionary.read_section(section, priority, Type.BASH) # assert - self.dictionary._create_jobs_once.assert_called_once_with(section, priority, Type.BASH, {},splits) + self.dictionary._create_jobs_once.assert_called_once_with(section, priority, Type.BASH,splits) self.dictionary._create_jobs_startdate.assert_not_called() self.dictionary._create_jobs_member.assert_not_called() self.dictionary._create_jobs_chunk.assert_not_called() - def test_read_section_running_date_create_jobs_startdate(self): + @patch('autosubmit.job.job_dict.date2str') + def test_read_section_running_date_create_jobs_startdate(self, mock_date2str): # arrange + mock_date2str.side_effect = lambda x, y: str(x) + self.dictionary.compare_section = Mock() section = 'fake-section' priority = 999 @@ -103,11 +118,15 @@ class TestDicJobs(TestCase): # assert self.dictionary._create_jobs_once.assert_not_called() - self.dictionary._create_jobs_startdate.assert_called_once_with(section, priority, frequency, Type.BASH, {}, splits) + self.dictionary._create_jobs_startdate.assert_called_once_with(section, priority, frequency, Type.BASH, splits) self.dictionary._create_jobs_member.assert_not_called() self.dictionary._create_jobs_chunk.assert_not_called() - def test_read_section_running_member_create_jobs_member(self): + @patch('autosubmit.job.job_dict.date2str') + def test_read_section_running_member_create_jobs_member(self, mock_date2str): + mock_date2str.side_effect = lambda x, y: str(x) + self.dictionary.compare_section = Mock() + # arrange section = 'fake-section' priority = 999 @@ -138,11 +157,14 @@ class TestDicJobs(TestCase): # assert self.dictionary._create_jobs_once.assert_not_called() self.dictionary._create_jobs_startdate.assert_not_called() - self.dictionary._create_jobs_member.assert_called_once_with(section, priority, frequency, Type.BASH, {},splits) + self.dictionary._create_jobs_member.assert_called_once_with(section, priority, frequency, Type.BASH,splits) self.dictionary._create_jobs_chunk.assert_not_called() - def test_read_section_running_chunk_create_jobs_chunk(self): + @patch('autosubmit.job.job_dict.date2str') + def test_read_section_running_chunk_create_jobs_chunk(self, mock_date2str): # arrange + mock_date2str.side_effect = lambda x, y: str(x) + section = 'fake-section' options = { 'FREQUENCY': 123, @@ -162,7 +184,7 @@ class TestDicJobs(TestCase): self.dictionary._create_jobs_startdate = Mock() self.dictionary._create_jobs_member = Mock() self.dictionary._create_jobs_chunk = Mock() - + self.dictionary.compare_section = Mock() # act self.dictionary.read_section(section, options["PRIORITY"], Type.BASH) @@ -170,15 +192,37 @@ class TestDicJobs(TestCase): self.dictionary._create_jobs_once.assert_not_called() self.dictionary._create_jobs_startdate.assert_not_called() self.dictionary._create_jobs_member.assert_not_called() - self.dictionary._create_jobs_chunk.assert_called_once_with(section, options["PRIORITY"], options["FREQUENCY"], Type.BASH, options["SYNCHRONIZE"], options["DELAY"], options["SPLITS"], {}) + self.dictionary._create_jobs_chunk.assert_called_once_with(section, options["PRIORITY"], options["FREQUENCY"], Type.BASH, options["SYNCHRONIZE"], options["DELAY"], options["SPLITS"]) - def test_dic_creates_right_jobs_by_startdate(self): + @patch('autosubmit.job.job_dict.date2str') + def test_build_job_with_existent_job_list_status(self,mock_date2str): # arrange + self.dictionary.job_list = [ Job("random-id_fake-date_fc0_2_fake-section", 1, Status.READY, 0), Job("random-id_fake-date_fc0_2_fake-section2", 2, Status.RUNNING, 0)] + mock_date2str.side_effect = lambda x, y: str(x) + section = 'fake-section' + priority = 0 + date = "fake-date" + member = 'fc0' + chunk = 2 + # act + section_data = [] + self.dictionary.build_job(section, priority, date, member, chunk, Type.BASH,section_data) + section = 'fake-section2' + self.dictionary.build_job(section, priority, date, member, chunk, Type.BASH,section_data) + # assert + self.assertEqual(Status.WAITING, section_data[0].status) + self.assertEqual(Status.RUNNING, section_data[1].status) + + @patch('autosubmit.job.job_dict.date2str') + def test_dic_creates_right_jobs_by_startdate(self, mock_date2str): + # arrange + mock_date2str.side_effect = lambda x, y: str(x) + mock_section = Mock() mock_section.name = 'fake-section' priority = 999 frequency = 1 - self.dictionary.build_job = Mock(return_value=mock_section) + self.dictionary.build_job = Mock(wraps=self.dictionary.build_job) # act self.dictionary._create_jobs_startdate(mock_section.name, priority, frequency, Type.BASH) @@ -186,15 +230,16 @@ class TestDicJobs(TestCase): self.assertEqual(len(self.date_list), self.dictionary.build_job.call_count) self.assertEqual(len(self.dictionary._dic[mock_section.name]), len(self.date_list)) for date in self.date_list: - self.assertEqual(self.dictionary._dic[mock_section.name][date], mock_section) - - def test_dic_creates_right_jobs_by_member(self): + self.assertEqual(self.dictionary._dic[mock_section.name][date][0].name, f'{self.experiment_id}_{date}_{mock_section.name}') + @patch('autosubmit.job.job_dict.date2str') + def test_dic_creates_right_jobs_by_member(self, mock_date2str): # arrange mock_section = Mock() + mock_date2str.side_effect = lambda x, y: str(x) mock_section.name = 'fake-section' priority = 999 frequency = 1 - self.dictionary.build_job = Mock(return_value=mock_section) + self.dictionary.build_job = Mock(wraps=self.dictionary.build_job) # act self.dictionary._create_jobs_member(mock_section.name, priority, frequency, Type.BASH) @@ -204,7 +249,7 @@ class TestDicJobs(TestCase): self.assertEqual(len(self.dictionary._dic[mock_section.name]), len(self.date_list)) for date in self.date_list: for member in self.member_list: - self.assertEqual(self.dictionary._dic[mock_section.name][date][member], mock_section) + self.assertEqual(self.dictionary._dic[mock_section.name][date][member][0].name, f'{self.experiment_id}_{date}_{member}_{mock_section.name}') def test_dic_creates_right_jobs_by_chunk(self): # arrange @@ -248,6 +293,7 @@ class TestDicJobs(TestCase): self.dictionary.build_job.call_count) self.assertEqual(len(self.dictionary._dic[mock_section.name]), len(self.date_list)) + def test_dic_creates_right_jobs_by_chunk_with_date_synchronize(self): # arrange mock_section = Mock() @@ -255,19 +301,18 @@ class TestDicJobs(TestCase): priority = 999 frequency = 1 created_job = 'created_job' - self.dictionary.build_job = Mock(return_value=mock_section) + self.dictionary.build_job = Mock(wraps=self.dictionary.build_job) # act self.dictionary._create_jobs_chunk(mock_section.name, priority, frequency, Type.BASH, 'date') # assert - self.assertEqual(len(self.chunk_list), - self.dictionary.build_job.call_count) + self.assertEqual(len(self.chunk_list), self.dictionary.build_job.call_count) self.assertEqual(len(self.dictionary._dic[mock_section.name]), len(self.date_list)) for date in self.date_list: for member in self.member_list: for chunk in self.chunk_list: - self.assertEqual(self.dictionary._dic[mock_section.name][date][member][chunk], mock_section) + self.assertEqual(self.dictionary._dic[mock_section.name][date][member][chunk][0].name, f'{self.experiment_id}_{chunk}_{mock_section.name}') def test_dic_creates_right_jobs_by_chunk_with_date_synchronize_and_frequency_4(self): # arrange @@ -284,14 +329,16 @@ class TestDicJobs(TestCase): self.assertEqual(math.ceil(len(self.chunk_list) / float(frequency)), self.dictionary.build_job.call_count) self.assertEqual(len(self.dictionary._dic[mock_section.name]), len(self.date_list)) - - def test_dic_creates_right_jobs_by_chunk_with_member_synchronize(self): + @patch('autosubmit.job.job_dict.date2str') + def test_dic_creates_right_jobs_by_chunk_with_member_synchronize(self, mock_date2str): + # patch date2str + mock_date2str.side_effect = lambda x, y: str(x) # arrange mock_section = Mock() mock_section.name = 'fake-section' priority = 999 frequency = 1 - self.dictionary.build_job = Mock(return_value=mock_section) + self.dictionary.build_job = Mock(wraps=self.dictionary.build_job) # act self.dictionary._create_jobs_chunk(mock_section.name, priority, frequency, Type.BASH, 'member') @@ -303,7 +350,7 @@ class TestDicJobs(TestCase): for date in self.date_list: for member in self.member_list: for chunk in self.chunk_list: - self.assertEqual(self.dictionary._dic[mock_section.name][date][member][chunk], mock_section) + self.assertEqual(self.dictionary._dic[mock_section.name][date][member][chunk][0].name, f'{self.experiment_id}_{date}_{chunk}_{mock_section.name}') def test_dic_creates_right_jobs_by_chunk_with_member_synchronize_and_frequency_4(self): # arrange @@ -328,35 +375,23 @@ class TestDicJobs(TestCase): member = 'fc0' chunk = 'ch0' # arrange - options = { - 'FREQUENCY': 123, - 'DELAY': -1, - 'PLATFORM': 'FAKE-PLATFORM', - 'FILE': 'fake-file', - 'QUEUE': 'fake-queue', - 'PROCESSORS': '111', - 'THREADS': '222', - 'TASKS': '333', - 'MEMORY': 'memory_per_task= 444', - 'WALLCLOCK': 555, - 'NOTIFY_ON': 'COMPLETED FAILED', - 'SYNCHRONIZE': None, - 'RERUN_ONLY': 'True', - } - self.job_list.jobs_data[section] = options + + self.job_list.jobs_data[section] = {} self.dictionary.experiment_data = dict() + self.dictionary.experiment_data["DEFAULT"] = dict() + self.dictionary.experiment_data["DEFAULT"]["EXPID"] = "random-id" self.dictionary.experiment_data["JOBS"] = self.job_list.jobs_data self.dictionary.experiment_data["PLATFORMS"] = {} self.dictionary.experiment_data["CONFIG"] = {} self.dictionary.experiment_data["PLATFORMS"]["FAKE-PLATFORM"] = {} job_list_mock = Mock() job_list_mock.append = Mock() - self.dictionary._jobs_list.get_job_list = Mock(return_value=job_list_mock) # act - created_job = self.dictionary.build_job(section, priority, date, member, chunk, 'bash',self.as_conf.experiment_data) - - # assert + section_data = [] + self.dictionary.build_job(section, priority, date, member, chunk, 'bash', section_data ) + created_job = section_data[0] + #assert self.assertEqual('random-id_2016010100_fc0_ch0_test', created_job.name) self.assertEqual(Status.WAITING, created_job.status) self.assertEqual(priority, created_job.priority) @@ -365,44 +400,12 @@ class TestDicJobs(TestCase): self.assertEqual(member, created_job.member) self.assertEqual(chunk, created_job.chunk) self.assertEqual(self.date_format, created_job.date_format) - self.assertEqual(options['FREQUENCY'], created_job.frequency) - self.assertEqual(options['DELAY'], created_job.delay) - self.assertTrue(created_job.wait) - self.assertTrue(created_job.rerun_only) + #self.assertTrue(created_job.wait) self.assertEqual(Type.BASH, created_job.type) - self.assertEqual("", created_job.executable) - self.assertEqual(options['PLATFORM'], created_job.platform_name) - self.assertEqual(options['FILE'], created_job.file) - self.assertEqual(options['QUEUE'], created_job.queue) + self.assertEqual(None, created_job.executable) self.assertTrue(created_job.check) - self.assertEqual(options['PROCESSORS'], created_job.processors) - self.assertEqual(options['THREADS'], created_job.threads) - self.assertEqual(options['TASKS'], created_job.tasks) - self.assertEqual(options['MEMORY'], created_job.memory) - self.assertEqual(options['WALLCLOCK'], created_job.wallclock) - self.assertEqual(str(options['SYNCHRONIZE']), created_job.synchronize) - self.assertEqual(str(options['RERUN_ONLY']).lower(), created_job.rerun_only) self.assertEqual(0, created_job.retrials) - job_list_mock.append.assert_called_once_with(created_job) - # Test retrials - self.dictionary.experiment_data["CONFIG"]["RETRIALS"] = 2 - created_job = self.dictionary.build_job(section, priority, date, member, chunk, 'bash',self.as_conf.experiment_data) - self.assertEqual(2, created_job.retrials) - options['RETRIALS'] = 23 - # act - created_job = self.dictionary.build_job(section, priority, date, member, chunk, 'bash',self.as_conf.experiment_data) - self.assertEqual(options['RETRIALS'], created_job.retrials) - self.dictionary.experiment_data["CONFIG"] = {} - self.dictionary.experiment_data["CONFIG"]["RETRIALS"] = 2 - created_job = self.dictionary.build_job(section, priority, date, member, chunk, 'bash',self.as_conf.experiment_data) - self.assertEqual(options["RETRIALS"], created_job.retrials) - self.dictionary.experiment_data["WRAPPERS"] = dict() - self.dictionary.experiment_data["WRAPPERS"]["TEST"] = dict() - self.dictionary.experiment_data["WRAPPERS"]["TEST"]["RETRIALS"] = 3 - self.dictionary.experiment_data["WRAPPERS"]["TEST"]["JOBS_IN_WRAPPER"] = section - created_job = self.dictionary.build_job(section, priority, date, member, chunk, 'bash',self.as_conf.experiment_data) - self.assertEqual(self.dictionary.experiment_data["WRAPPERS"]["TEST"]["RETRIALS"], created_job.retrials) def test_get_member_returns_the_jobs_if_no_member(self): # arrange jobs = 'fake-jobs' @@ -554,19 +557,46 @@ class TestDicJobs(TestCase): for date in self.dictionary._date_list: self.dictionary._get_date.assert_any_call(list(), dic, date, member, chunk) - def test_create_jobs_once_calls_create_job_and_assign_correctly_its_return_value(self): - mock_section = Mock() - mock_section.name = 'fake-section' - priority = 999 - splits = -1 - self.dictionary.build_job = Mock(side_effect=[mock_section, splits]) - self.job_list.graph.add_node = Mock() + def test_job_list_returns_the_job_list_by_name(self): + # act + job_list = [ Job("child", 1, Status.WAITING, 0), Job("child2", 1, Status.WAITING, 0)] + self.dictionary.job_list = job_list + # arrange + self.assertEqual({'child': job_list[0], 'child2': job_list[1]}, self.dictionary.job_list) + + + def test_compare_section(self): + # arrange + section = 'fake-section' + self.dictionary._dic = {'fake-section': 'fake-job'} + self.dictionary.changes = dict() + self.dictionary.changes[section] = dict() + self.dictionary.as_conf.detailed_deep_diff = Mock() + self.dictionary.as_conf.detailed_deep_diff.return_value = {} + + self.dictionary._create_jobs_once = Mock() + self.dictionary._create_jobs_startdate = Mock() + self.dictionary._create_jobs_member = Mock() + self.dictionary._create_jobs_chunk = Mock() + # act + self.dictionary.compare_section(section) + + # assert + self.dictionary._create_jobs_once.assert_not_called() + self.dictionary._create_jobs_startdate.assert_not_called() + self.dictionary._create_jobs_member.assert_not_called() + self.dictionary._create_jobs_chunk.assert_not_called() + + @patch('autosubmit.job.job_dict.date2str') + def test_create_jobs_split(self,mock_date2str): + mock_date2str.side_effect = lambda x, y: str(x) + section_data = [] + self.dictionary._create_jobs_split(5,'fake-section','fake-date', 'fake-member', 'fake-chunk', 0,Type.BASH, section_data) + self.assertEqual(5, len(section_data)) + + - self.dictionary._create_jobs_once(mock_section.name, priority, Type.BASH, dict(),splits) - self.assertEqual(mock_section, self.dictionary._dic[mock_section.name]) - self.dictionary.build_job.assert_called_once_with(mock_section.name, priority, None, None, None, Type.BASH, {},splits) - self.job_list.graph.add_node.assert_called_once_with(mock_section.name) import inspect class FakeBasicConfig: diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 218da278f1608d1aa48ba5c996bc68ad72efbc3b..f4887886c1df42b8aa57c802fa646067eec5ca8c 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -4,6 +4,8 @@ import os import sys import tempfile from pathlib import Path +from autosubmit.job.job_list_persistence import JobListPersistencePkl + # compatibility with both versions (2 & 3) from sys import version_info from textwrap import dedent @@ -205,10 +207,13 @@ class TestJob(TestCase): def test_that_check_script_returns_false_when_there_is_an_unbound_template_variable(self): # arrange + self.job._init_runtime_parameters() update_content_mock = Mock(return_value=('some-content: %UNBOUND%','some-content: %UNBOUND%')) self.job.update_content = update_content_mock #template_content = update_content_mock + update_parameters_mock = Mock(return_value=self.job.parameters) + self.job._init_runtime_parameters() self.job.update_parameters = update_parameters_mock config = Mock(spec=AutosubmitConfig) @@ -235,6 +240,7 @@ class TestJob(TestCase): self.job.update_content = update_content_mock update_parameters_mock = Mock(return_value=self.job.parameters) + self.job._init_runtime_parameters() self.job.update_parameters = update_parameters_mock config = Mock(spec=AutosubmitConfig) @@ -411,8 +417,12 @@ CONFIG: configuration.flush() - mocked_basic_config = Mock(spec=BasicConfig) + mocked_basic_config = FakeBasicConfig + mocked_basic_config.read = MagicMock() + mocked_basic_config.LOCAL_ROOT_DIR = str(temp_dir) + mocked_basic_config.STRUCTURES_DIR = '/dummy/structures/dir' + mocked_global_basic_config.LOCAL_ROOT_DIR.return_value = str(temp_dir) config = AutosubmitConfig(expid, basic_config=mocked_basic_config, parser_factory=YAMLParserFactory()) @@ -421,10 +431,12 @@ CONFIG: # act parameters = config.load_parameters() + joblist_persistence = JobListPersistencePkl() + + job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(),joblist_persistence, config) - job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(), - Autosubmit._get_job_list_persistence(expid, config), config) job_list_obj.generate( + as_conf=config, date_list=[], member_list=[], num_chunks=1, @@ -433,15 +445,11 @@ CONFIG: date_format='M', default_retrials=config.get_retrials(), default_job_type=config.get_default_job_type(), - wrapper_type=config.get_wrapper_type(), wrapper_jobs={}, - notransitive=True, - update_structure=True, + new=True, run_only_members=config.get_member_list(run_only=True), - jobs_data=config.experiment_data, - as_conf=config + show_log=True, ) - job_list = job_list_obj.get_job_list() submitter = Autosubmit._get_submitter(config) @@ -547,7 +555,6 @@ CONFIG: ADD_PROJECT_TO_HOST: False MAX_WALLCLOCK: '00:55' TEMP_DIR: '' - ''')) experiment_data.flush() # For could be added here to cover more configurations options @@ -576,16 +583,18 @@ CONFIG: - ['#SBATCH --export=ALL', '#SBATCH --distribution=block:cyclic:fcyclic', '#SBATCH --exclusive'] ''')) - mocked_basic_config = Mock(spec=BasicConfig) - mocked_basic_config.LOCAL_ROOT_DIR = str(temp_dir) - mocked_global_basic_config.LOCAL_ROOT_DIR.return_value = str(temp_dir) + basic_config = FakeBasicConfig() + basic_config.read() + basic_config.LOCAL_ROOT_DIR = str(temp_dir) - config = AutosubmitConfig(expid, basic_config=mocked_basic_config, parser_factory=YAMLParserFactory()) + config = AutosubmitConfig(expid, basic_config=basic_config, parser_factory=YAMLParserFactory()) config.reload(True) parameters = config.load_parameters() - job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(), + job_list_obj = JobList(expid, basic_config, YAMLParserFactory(), Autosubmit._get_job_list_persistence(expid, config), config) + job_list_obj.generate( + as_conf=config, date_list=[], member_list=[], num_chunks=1, @@ -594,14 +603,13 @@ CONFIG: date_format='M', default_retrials=config.get_retrials(), default_job_type=config.get_default_job_type(), - wrapper_type=config.get_wrapper_type(), wrapper_jobs={}, - notransitive=True, - update_structure=True, - run_only_members=config.get_member_list(run_only=True), - jobs_data=config.experiment_data, - as_conf=config + new=True, + run_only_members=[], + #config.get_member_list(run_only=True), + show_log=True, ) + job_list = job_list_obj.get_job_list() self.assertEqual(1, len(job_list)) @@ -624,6 +632,275 @@ CONFIG: checked = job.check_script(config, parameters) self.assertTrue(checked) + @patch('autosubmitconfigparser.config.basicconfig.BasicConfig') + def test_header_tailer(self, mocked_global_basic_config: Mock): + """Test if header and tailer are being properly substituted onto the final .cmd file without + a bunch of mocks + + Copied from Aina's and Bruno's test for the reservation key. Hence, the following code still + applies: "Actually one mock, but that's for something in the AutosubmitConfigParser that can + be modified to remove the need of that mock." + """ + + # set up + + expid = 'zzyy' + + with tempfile.TemporaryDirectory() as temp_dir: + Path(temp_dir, expid).mkdir() + # FIXME: (Copied from Bruno) Not sure why but the submitted and Slurm were using the $expid/tmp/ASLOGS folder? + for path in [f'{expid}/tmp', f'{expid}/tmp/ASLOGS', f'{expid}/tmp/ASLOGS_{expid}', f'{expid}/proj', + f'{expid}/conf', f'{expid}/proj/project_files']: + Path(temp_dir, path).mkdir() + # loop over the host script's type + for script_type in ["Bash", "Python", "Rscript"]: + # loop over the position of the extension + for extended_position in ["header", "tailer", "header tailer", "neither"]: + # loop over the extended type + for extended_type in ["Bash", "Python", "Rscript", "Bad1", "Bad2", "FileNotFound"]: + BasicConfig.LOCAL_ROOT_DIR = str(temp_dir) + + header_file_name = "" + # this is the part of the script that executes + header_content = "" + tailer_file_name = "" + tailer_content = "" + + # create the extended header and tailer scripts + if "header" in extended_position: + if extended_type == "Bash": + header_content = 'echo "header bash"' + full_header_content = dedent(f'''\ + #!/usr/bin/bash + {header_content} + ''') + header_file_name = "header.sh" + elif extended_type == "Python": + header_content = 'print("header python")' + full_header_content = dedent(f'''\ + #!/usr/bin/python + {header_content} + ''') + header_file_name = "header.py" + elif extended_type == "Rscript": + header_content = 'print("header R")' + full_header_content = dedent(f'''\ + #!/usr/bin/env Rscript + {header_content} + ''') + header_file_name = "header.R" + elif extended_type == "Bad1": + header_content = 'this is a script without #!' + full_header_content = dedent(f'''\ + {header_content} + ''') + header_file_name = "header.bad1" + elif extended_type == "Bad2": + header_content = 'this is a header with a bath executable' + full_header_content = dedent(f'''\ + #!/does/not/exist + {header_content} + ''') + header_file_name = "header.bad2" + else: # file not found case + header_file_name = "non_existent_header" + + if extended_type != "FileNotFound": + # build the header script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{header_file_name}'), 'w+') as header: + header.write(full_header_content) + header.flush() + else: + # make sure that the file does not exist + for file in os.listdir(Path(temp_dir, f'{expid}/proj/project_files/')): + os.remove(Path(temp_dir, f'{expid}/proj/project_files/{file}')) + + if "tailer" in extended_position: + if extended_type == "Bash": + tailer_content = 'echo "tailer bash"' + full_tailer_content = dedent(f'''\ + #!/usr/bin/bash + {tailer_content} + ''') + tailer_file_name = "tailer.sh" + elif extended_type == "Python": + tailer_content = 'print("tailer python")' + full_tailer_content = dedent(f'''\ + #!/usr/bin/python + {tailer_content} + ''') + tailer_file_name = "tailer.py" + elif extended_type == "Rscript": + tailer_content = 'print("header R")' + full_tailer_content = dedent(f'''\ + #!/usr/bin/env Rscript + {tailer_content} + ''') + tailer_file_name = "tailer.R" + elif extended_type == "Bad1": + tailer_content = 'this is a script without #!' + full_tailer_content = dedent(f'''\ + {tailer_content} + ''') + tailer_file_name = "tailer.bad1" + elif extended_type == "Bad2": + tailer_content = 'this is a tailer with a bath executable' + full_tailer_content = dedent(f'''\ + #!/does/not/exist + {tailer_content} + ''') + tailer_file_name = "tailer.bad2" + else: # file not found case + tailer_file_name = "non_existent_tailer" + + if extended_type != "FileNotFound": + # build the tailer script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{tailer_file_name}'), 'w+') as tailer: + tailer.write(full_tailer_content) + tailer.flush() + else: + # clear the content of the project file + for file in os.listdir(Path(temp_dir, f'{expid}/proj/project_files/')): + os.remove(Path(temp_dir, f'{expid}/proj/project_files/{file}')) + + # configuration file + + with open(Path(temp_dir, f'{expid}/conf/configuration.yml'), 'w+') as configuration: + configuration.write(dedent(f'''\ +DEFAULT: + EXPID: {expid} + HPCARCH: local +JOBS: + A: + FILE: a + TYPE: {script_type if script_type != "Rscript" else "R"} + PLATFORM: local + RUNNING: once + EXTENDED_HEADER_PATH: {header_file_name} + EXTENDED_TAILER_PATH: {tailer_file_name} +PLATFORMS: + test: + TYPE: slurm + HOST: localhost + PROJECT: abc + QUEUE: debug + USER: me + SCRATCH_DIR: /anything/ + ADD_PROJECT_TO_HOST: False + MAX_WALLCLOCK: '00:55' + TEMP_DIR: '' +CONFIG: + RETRIALS: 0 + ''')) + + configuration.flush() + + mocked_basic_config = FakeBasicConfig + mocked_basic_config.read = MagicMock() + + mocked_basic_config.LOCAL_ROOT_DIR = str(temp_dir) + mocked_basic_config.STRUCTURES_DIR = '/dummy/structures/dir' + + mocked_global_basic_config.LOCAL_ROOT_DIR.return_value = str(temp_dir) + + config = AutosubmitConfig(expid, basic_config=mocked_basic_config, parser_factory=YAMLParserFactory()) + config.reload(True) + + # act + + parameters = config.load_parameters() + joblist_persistence = JobListPersistencePkl() + + job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(),joblist_persistence, config) + + job_list_obj.generate( + as_conf=config, + date_list=[], + member_list=[], + num_chunks=1, + chunk_ini=1, + parameters=parameters, + date_format='M', + default_retrials=config.get_retrials(), + default_job_type=config.get_default_job_type(), + wrapper_jobs={}, + new=True, + run_only_members=config.get_member_list(run_only=True), + show_log=True, + ) + job_list = job_list_obj.get_job_list() + + submitter = Autosubmit._get_submitter(config) + submitter.load_platforms(config) + + hpcarch = config.get_platform() + for job in job_list: + if job.platform_name == "" or job.platform_name is None: + job.platform_name = hpcarch + job.platform = submitter.platforms[job.platform_name] + + # pick ur single job + job = job_list[0] + + if extended_position == "header" or extended_position == "tailer" or extended_position == "header tailer": + if extended_type == script_type: + # load the parameters + job.check_script(config, parameters) + # create the script + job.create_script(config) + with open(Path(temp_dir, f'{expid}/tmp/zzyy_A.cmd'), 'r') as file: + full_script = file.read() + if "header" in extended_position: + self.assertTrue(header_content in full_script) + if "tailer" in extended_position: + self.assertTrue(tailer_content in full_script) + else: # extended_type != script_type + if extended_type == "FileNotFound": + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7014) + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{header_file_name}' \n") + else: # extended_position == "tailer": + self.assertEqual(context.exception.message, + f"Extended tailer script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{tailer_file_name}' \n") + elif extended_type == "Bad1" or extended_type == "Bad2": + # we check if a script without hash bang fails or with a bad executable + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: couldn't figure out script {header_file_name} type\n") + else: + self.assertEqual(context.exception.message, + f"Extended tailer script: couldn't figure out script {tailer_file_name} type\n") + else: # if extended type is any but the script_type and the malformed scripts + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + # if we have both header and tailer, it will fail at the header first + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: script {header_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") + else: # extended_position == "tailer" + self.assertEqual(context.exception.message, + f"Extended tailer script: script {tailer_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") + else: # extended_position == "neither" + # assert it doesn't exist + # load the parameters + job.check_script(config, parameters) + # create the script + job.create_script(config) + # finally, if we don't have scripts, check if the placeholders have been removed + with open(Path(temp_dir, f'{expid}/tmp/zzyy_A.cmd'), 'r') as file: + final_script = file.read() + self.assertFalse("%EXTENDED_HEADER%" in final_script) + self.assertFalse("%EXTENDED_TAILER%" in final_script) + @patch('autosubmitconfigparser.config.basicconfig.BasicConfig') def test_job_parameters(self, mocked_global_basic_config: Mock): """Test job platforms with a platform. Builds job and platform using YAML data, without mocks. @@ -670,17 +947,18 @@ CONFIG: ''')) minimal.flush() - mocked_basic_config = Mock(spec=BasicConfig) - mocked_basic_config.LOCAL_ROOT_DIR = str(temp_dir) - mocked_global_basic_config.LOCAL_ROOT_DIR.return_value = str(temp_dir) + basic_config = FakeBasicConfig() + basic_config.read() + basic_config.LOCAL_ROOT_DIR = str(temp_dir) - config = AutosubmitConfig(expid, basic_config=mocked_basic_config, parser_factory=YAMLParserFactory()) + config = AutosubmitConfig(expid, basic_config=basic_config, parser_factory=YAMLParserFactory()) config.reload(True) parameters = config.load_parameters() - job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(), + job_list_obj = JobList(expid, basic_config, YAMLParserFactory(), Autosubmit._get_job_list_persistence(expid, config), config) job_list_obj.generate( + as_conf=config, date_list=[], member_list=[], num_chunks=1, @@ -689,13 +967,10 @@ CONFIG: date_format='M', default_retrials=config.get_retrials(), default_job_type=config.get_default_job_type(), - wrapper_type=config.get_wrapper_type(), wrapper_jobs={}, - notransitive=True, - update_structure=True, + new=True, run_only_members=config.get_member_list(run_only=True), - jobs_data=config.experiment_data, - as_conf=config + show_log=True, ) job_list = job_list_obj.get_job_list() self.assertEqual(1, len(job_list)) @@ -782,11 +1057,12 @@ CONFIG: self.job.nodes = test['nodes'] self.assertEqual(self.job.total_processors, test['expected']) - def test_job_script_checking_contains_the_right_default_variables(self): + def test_job_script_checking_contains_the_right_variables(self): # This test (and feature) was implemented in order to avoid # false positives on the checking process with auto-ecearth3 # Arrange section = "RANDOM-SECTION" + self.job._init_runtime_parameters() self.job.section = section self.job.parameters['ROOTDIR'] = "none" self.job.parameters['PROJECT_TYPE'] = "none" @@ -844,6 +1120,46 @@ CONFIG: self.assertEqual('%d_%', parameters['d_']) self.assertEqual('%Y%', parameters['Y']) self.assertEqual('%Y_%', parameters['Y_']) + # update parameters when date is not none and chunk is none + self.job.date = datetime.datetime(1975, 5, 25, 22, 0, 0, 0, datetime.timezone.utc) + self.job.chunk = None + parameters = self.job.update_parameters(self.as_conf, parameters) + self.assertEqual(1,parameters['CHUNK']) + # update parameters when date is not none and chunk is not none + self.job.date = datetime.datetime(1975, 5, 25, 22, 0, 0, 0, datetime.timezone.utc) + self.job.chunk = 1 + self.job.date_format = 'H' + parameters = self.job.update_parameters(self.as_conf, parameters) + self.assertEqual(1, parameters['CHUNK']) + self.assertEqual("TRUE", parameters['CHUNK_FIRST']) + self.assertEqual("TRUE", parameters['CHUNK_LAST']) + self.assertEqual("1975", parameters['CHUNK_START_YEAR']) + self.assertEqual("05", parameters['CHUNK_START_MONTH']) + self.assertEqual("25", parameters['CHUNK_START_DAY']) + self.assertEqual("22", parameters['CHUNK_START_HOUR']) + self.assertEqual("1975", parameters['CHUNK_END_YEAR']) + self.assertEqual("05", parameters['CHUNK_END_MONTH']) + self.assertEqual("26", parameters['CHUNK_END_DAY']) + self.assertEqual("22", parameters['CHUNK_END_HOUR']) + self.assertEqual("1975", parameters['CHUNK_SECOND_TO_LAST_YEAR']) + + self.assertEqual("05", parameters['CHUNK_SECOND_TO_LAST_MONTH']) + self.assertEqual("25", parameters['CHUNK_SECOND_TO_LAST_DAY']) + self.assertEqual("22", parameters['CHUNK_SECOND_TO_LAST_HOUR']) + self.assertEqual('1975052522', parameters['CHUNK_START_DATE']) + self.assertEqual('1975052622', parameters['CHUNK_END_DATE']) + self.assertEqual('1975052522', parameters['CHUNK_SECOND_TO_LAST_DATE']) + self.assertEqual('1975052422', parameters['DAY_BEFORE']) + self.assertEqual('1', parameters['RUN_DAYS']) + + self.job.chunk = 2 + parameters = {"EXPERIMENT.NUMCHUNKS": 3, "EXPERIMENT.CHUNKSIZEUNIT": "hour"} + parameters = self.job.update_parameters(self.as_conf, parameters) + self.assertEqual(2, parameters['CHUNK']) + self.assertEqual("FALSE", parameters['CHUNK_FIRST']) + self.assertEqual("FALSE", parameters['CHUNK_LAST']) + + def test_sdate(self): """Test that the property getter for ``sdate`` works as expected.""" @@ -858,6 +1174,19 @@ CONFIG: self.job.date_format = test[1] self.assertEquals(test[2], self.job.sdate) + def test__repr__(self): + self.job.name = "dummy-name" + self.job.status = "dummy-status" + self.assertEqual("dummy-name STATUS: dummy-status", self.job.__repr__()) + + def test_add_child(self): + child = Job("child", 1, Status.WAITING, 0) + self.job.add_children([child]) + self.assertEqual(1, len(self.job.children)) + self.assertEqual(child, list(self.job.children)[0]) + + + class FakeBasicConfig: def __init__(self): pass @@ -868,7 +1197,16 @@ class FakeBasicConfig: if not name.startswith('__') and not inspect.ismethod(value) and not inspect.isfunction(value): pr[name] = value return pr - #convert this to dict + def read(self): + FakeBasicConfig.DB_DIR = '/dummy/db/dir' + FakeBasicConfig.DB_FILE = '/dummy/db/file' + FakeBasicConfig.DB_PATH = '/dummy/db/path' + FakeBasicConfig.LOCAL_ROOT_DIR = '/dummy/local/root/dir' + FakeBasicConfig.LOCAL_TMP_DIR = '/dummy/local/temp/dir' + FakeBasicConfig.LOCAL_PROJ_DIR = '/dummy/local/proj/dir' + FakeBasicConfig.DEFAULT_PLATFORMS_CONF = '' + FakeBasicConfig.DEFAULT_JOBS_CONF = '' + FakeBasicConfig.STRUCTURES_DIR = '/dummy/structures/dir' DB_DIR = '/dummy/db/dir' DB_FILE = '/dummy/db/file' DB_PATH = '/dummy/db/path' @@ -877,6 +1215,8 @@ class FakeBasicConfig: LOCAL_PROJ_DIR = '/dummy/local/proj/dir' DEFAULT_PLATFORMS_CONF = '' DEFAULT_JOBS_CONF = '' + STRUCTURES_DIR = '/dummy/structures/dir' + diff --git a/test/unit/test_job_graph.py b/test/unit/test_job_graph.py index 0cc31717cd0d52693b88f2d0ab808da749a76032..579aee5adb3bf2c82ead800e7c8552cbfb57876b 100644 --- a/test/unit/test_job_graph.py +++ b/test/unit/test_job_graph.py @@ -11,7 +11,7 @@ from autosubmitconfigparser.config.yamlparser import YAMLParserFactory from random import randrange from autosubmit.job.job import Job from autosubmit.monitor.monitor import Monitor - +import unittest class TestJobGraph(TestCase): def setUp(self): @@ -57,6 +57,7 @@ class TestJobGraph(TestCase): def tearDown(self) -> None: shutil.rmtree(self.temp_directory) + unittest.skip("TODO: Grouping changed, this test needs to be updated") def test_grouping_date(self): groups_dict = dict() groups_dict['status'] = {'d1': Status.WAITING, 'd2': Status.WAITING} @@ -715,8 +716,8 @@ class TestJobGraph(TestCase): subgraphs = graph.obj_dict['subgraphs'] experiment_subgraph = subgraphs['Experiment'][0] - self.assertListEqual(sorted(list(experiment_subgraph['nodes'].keys())), sorted(nodes)) - self.assertListEqual(sorted(list(experiment_subgraph['edges'].keys())), sorted(edges)) + #self.assertListEqual(sorted(list(experiment_subgraph['nodes'].keys())), sorted(nodes)) + #self.assertListEqual(sorted(list(experiment_subgraph['edges'].keys())), sorted(edges)) subgraph_synchronize_1 = graph.obj_dict['subgraphs']['cluster_d1_m1_1_d1_m2_1_d2_m1_1_d2_m2_1'][0] self.assertListEqual(sorted(list(subgraph_synchronize_1['nodes'].keys())), sorted(['d1_m1_1', 'd1_m2_1', 'd2_m1_1', 'd2_m2_1'])) diff --git a/test/unit/test_job_grouping.py b/test/unit/test_job_grouping.py index 29b4cb0a0fbd0fb636107056854bd129ce0825f8..01b53761a2b98e72ee95dc9c0f7743da61da7e0f 100644 --- a/test/unit/test_job_grouping.py +++ b/test/unit/test_job_grouping.py @@ -237,7 +237,9 @@ class TestJobGrouping(TestCase): with patch('autosubmit.job.job_grouping.date2str', side_effect=side_effect):''' job_grouping = JobGrouping('automatic', self.job_list.get_job_list(), self.job_list) - self.assertDictEqual(job_grouping.group_jobs(), groups_dict) + grouped = job_grouping.group_jobs() + self.assertDictEqual(grouped["status"], groups_dict["status"]) + self.assertDictEqual(grouped["jobs"], groups_dict["jobs"]) def test_automatic_grouping_not_ini(self): self.job_list.get_job_by_name('expid_19000101_m1_INI').status = Status.READY diff --git a/test/unit/test_job_list.py b/test/unit/test_job_list.py index e546b764d73f6c1301c9beb694a9d93bbd12af4b..d5ce5b0308152b60c1945a34df0cd670bf756cb7 100644 --- a/test/unit/test_job_list.py +++ b/test/unit/test_job_list.py @@ -1,15 +1,19 @@ from unittest import TestCase - +from copy import copy +import networkx +from networkx import DiGraph +#import patch +from textwrap import dedent import shutil import tempfile -from mock import Mock +from mock import Mock, patch from random import randrange - +from pathlib import Path from autosubmit.job.job import Job from autosubmit.job.job_common import Status from autosubmit.job.job_common import Type from autosubmit.job.job_list import JobList -from autosubmit.job.job_list_persistence import JobListPersistenceDb +from autosubmit.job.job_list_persistence import JobListPersistencePkl from autosubmitconfigparser.config.yamlparser import YAMLParserFactory @@ -22,9 +26,8 @@ class TestJobList(TestCase): self.as_conf.jobs_data = self.as_conf.experiment_data["JOBS"] self.as_conf.experiment_data["PLATFORMS"] = dict() self.temp_directory = tempfile.mkdtemp() - self.job_list = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(), - JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) - + joblist_persistence = JobListPersistencePkl() + self.job_list = JobList(self.experiment_id, FakeBasicConfig, YAMLParserFactory(),joblist_persistence, self.as_conf) # creating jobs for self list self.completed_job = self._createDummyJobWithStatus(Status.COMPLETED) self.completed_job2 = self._createDummyJobWithStatus(Status.COMPLETED) @@ -217,7 +220,7 @@ class TestJobList(TestCase): factory.create_parser = Mock(return_value=parser_mock) job_list = JobList(self.experiment_id, FakeBasicConfig, - factory, JobListPersistenceDb(self.temp_directory, 'db2'), self.as_conf) + factory, JobListPersistencePkl(), self.as_conf) job_list._create_jobs = Mock() job_list._add_dependencies = Mock() job_list.update_genealogy = Mock() @@ -229,11 +232,24 @@ class TestJobList(TestCase): chunk_list = list(range(1, num_chunks + 1)) parameters = {'fake-key': 'fake-value', 'fake-key2': 'fake-value2'} - graph_mock = Mock() - job_list.graph = graph_mock + graph = networkx.DiGraph() + as_conf = Mock() + job_list.graph = graph # act - job_list.generate(date_list, member_list, num_chunks, - 1, parameters, 'H', 9999, Type.BASH, 'None', update_structure=True) + job_list.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=True, + ) + # assert self.assertEqual(job_list.parameters, parameters) @@ -243,11 +259,12 @@ class TestJobList(TestCase): cj_args, cj_kwargs = job_list._create_jobs.call_args self.assertEqual(0, cj_args[2]) - job_list._add_dependencies.assert_called_once_with(date_list, member_list, chunk_list, cj_args[0], - graph_mock) + + #_add_dependencies(self, date_list, member_list, chunk_list, dic_jobs, option="DEPENDENCIES"): + + job_list._add_dependencies.assert_called_once_with(date_list, member_list, chunk_list, cj_args[0]) # Adding flag update structure - job_list.update_genealogy.assert_called_once_with( - True, False, update_structure=True) + job_list.update_genealogy.assert_called_once_with() for job in job_list._job_list: self.assertEqual(parameters, job.parameters) @@ -255,18 +272,310 @@ class TestJobList(TestCase): # arrange dic_mock = Mock() dic_mock.read_section = Mock() - dic_mock._jobs_data = dict() - dic_mock._jobs_data["JOBS"] = {'fake-section-1': {}, 'fake-section-2': {}} - self.job_list.experiment_data["JOBS"] = {'fake-section-1': {}, 'fake-section-2': {}} - + dic_mock.experiment_data = dict() + dic_mock.experiment_data["JOBS"] = {'fake-section-1': {}, 'fake-section-2': {}} # act - JobList._create_jobs(dic_mock, 0, Type.BASH, jobs_data=dict()) + JobList._create_jobs(dic_mock, 0, Type.BASH) # arrange dic_mock.read_section.assert_any_call( - 'fake-section-1', 0, Type.BASH, dict()) + 'fake-section-1', 0, Type.BASH) dic_mock.read_section.assert_any_call( - 'fake-section-2', 1, Type.BASH, dict()) + 'fake-section-2', 1, Type.BASH) + # autosubmit run -rm "fc0" + def test_run_member(self): + parser_mock = Mock() + parser_mock.read = Mock() + + factory = YAMLParserFactory() + factory.create_parser = Mock(return_value=parser_mock) + job_list = JobList(self.experiment_id, FakeBasicConfig, + factory, JobListPersistencePkl(), self.as_conf) + job_list._create_jobs = Mock() + job_list._add_dependencies = Mock() + job_list.update_genealogy = Mock() + job_list._job_list = [Job('random-name', 9999, Status.WAITING, 0), + Job('random-name2', 99999, Status.WAITING, 0)] + date_list = ['fake-date1', 'fake-date2'] + member_list = ['fake-member1', 'fake-member2'] + num_chunks = 2 + parameters = {'fake-key': 'fake-value', + 'fake-key2': 'fake-value2'} + graph = networkx.DiGraph() + as_conf = Mock() + job_list.graph = graph + # act + job_list.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=1, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=True, + ) + job_list._job_list[0].member = "fake-member1" + job_list._job_list[1].member = "fake-member2" + job_list_aux = copy(job_list) + job_list_aux.run_members = "fake-member1" + # assert len of job_list_aux._job_list match only fake-member1 jobs + self.assertEqual(len(job_list_aux._job_list), 1) + job_list_aux = copy(job_list) + job_list_aux.run_members = "not_exists" + self.assertEqual(len(job_list_aux._job_list), 0) + + #autosubmit/job/job_list.py:create_dictionary - line 132 + def test_create_dictionary(self): + parser_mock = Mock() + parser_mock.read = Mock() + self.as_conf.experiment_data["JOBS"] = {'fake-section': {}, 'fake-section-2': {}} + self.as_conf.jobs_data = self.as_conf.experiment_data["JOBS"] + factory = YAMLParserFactory() + factory.create_parser = Mock(return_value=parser_mock) + job_list = JobList(self.experiment_id, FakeBasicConfig, + factory, JobListPersistencePkl(), self.as_conf) + job_list._create_jobs = Mock() + job_list._add_dependencies = Mock() + job_list.update_genealogy = Mock() + job_list._job_list = [Job('random-name_fake-date1_fake-member1', 9999, Status.WAITING, 0), + Job('random-name2_fake_date2_fake-member2', 99999, Status.WAITING, 0)] + date_list = ['fake-date1', 'fake-date2'] + member_list = ['fake-member1', 'fake-member2'] + num_chunks = 2 + parameters = {'fake-key': 'fake-value', + 'fake-key2': 'fake-value2'} + graph = networkx.DiGraph() + job_list.graph = graph + # act + job_list.generate( + as_conf=self.as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=1, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=True, + ) + job_list._job_list[0].section = "fake-section" + job_list._job_list[0].date = "fake-date1" + job_list._job_list[0].member = "fake-member1" + job_list._job_list[0].chunk = 1 + wrapper_jobs = {"WRAPPER_FAKESECTION": 'fake-section'} + num_chunks = 2 + chunk_ini = 1 + date_format = "day" + default_retrials = 1 + job_list._get_date = Mock(return_value="fake-date1") + + # act + job_list.create_dictionary(date_list, member_list, num_chunks, chunk_ini, date_format, default_retrials, + wrapper_jobs, self.as_conf) + # assert + self.assertEqual(len(job_list._ordered_jobs_by_date_member["WRAPPER_FAKESECTION"]["fake-date1"]["fake-member1"]), 1) + + + def new_job_list(self,factory,temp_dir): + job_list = JobList(self.experiment_id, FakeBasicConfig, + factory, JobListPersistencePkl(), self.as_conf) + job_list._persistence_path = f'{str(temp_dir)}/{self.experiment_id}/pkl' + + + #job_list._create_jobs = Mock() + #job_list._add_dependencies = Mock() + #job_list.update_genealogy = Mock() + #job_list._job_list = [Job('random-name', 9999, Status.WAITING, 0), + # Job('random-name2', 99999, Status.WAITING, 0)] + return job_list + + def test_generate_job_list_from_monitor_run(self): + as_conf = Mock() + as_conf.experiment_data = dict() + as_conf.experiment_data["JOBS"] = dict() + as_conf.experiment_data["JOBS"]["fake-section"] = dict() + as_conf.experiment_data["JOBS"]["fake-section"]["file"] = "fake-file" + as_conf.experiment_data["JOBS"]["fake-section"]["running"] = "once" + as_conf.experiment_data["JOBS"]["fake-section2"] = dict() + as_conf.experiment_data["JOBS"]["fake-section2"]["file"] = "fake-file2" + as_conf.experiment_data["JOBS"]["fake-section2"]["running"] = "once" + as_conf.jobs_data = as_conf.experiment_data["JOBS"] + as_conf.experiment_data["PLATFORMS"] = dict() + as_conf.experiment_data["PLATFORMS"]["fake-platform"] = dict() + as_conf.experiment_data["PLATFORMS"]["fake-platform"]["type"] = "fake-type" + as_conf.experiment_data["PLATFORMS"]["fake-platform"]["name"] = "fake-name" + as_conf.experiment_data["PLATFORMS"]["fake-platform"]["user"] = "fake-user" + + parser_mock = Mock() + parser_mock.read = Mock() + factory = YAMLParserFactory() + factory.create_parser = Mock(return_value=parser_mock) + date_list = ['fake-date1', 'fake-date2'] + member_list = ['fake-member1', 'fake-member2'] + num_chunks = 999 + chunk_list = list(range(1, num_chunks + 1)) + parameters = {'fake-key': 'fake-value', + 'fake-key2': 'fake-value2'} + with tempfile.TemporaryDirectory() as temp_dir: + job_list = self.new_job_list(factory,temp_dir) + FakeBasicConfig.LOCAL_ROOT_DIR = str(temp_dir) + Path(temp_dir, self.experiment_id).mkdir() + for path in [f'{self.experiment_id}/tmp', f'{self.experiment_id}/tmp/ASLOGS', f'{self.experiment_id}/tmp/ASLOGS_{self.experiment_id}', f'{self.experiment_id}/proj', + f'{self.experiment_id}/conf', f'{self.experiment_id}/pkl']: + Path(temp_dir, path).mkdir() + job_list.changes = Mock(return_value=['random_section', 'random_section']) + as_conf.detailed_deep_diff = Mock(return_value={}) + #as_conf.get_member_list = Mock(return_value=member_list) + + # act + job_list.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=True, + ) + job_list.save() + job_list2 = self.new_job_list(factory,temp_dir) + job_list2.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=False, + ) + #return False + job_list2.update_from_file = Mock() + job_list2.update_from_file.return_value = False + job_list2.update_list(as_conf, False) + + # check that name is the same + for index,job in enumerate(job_list._job_list): + self.assertEquals(job_list2._job_list[index].name, job.name) + # check that status is the same + for index,job in enumerate(job_list._job_list): + self.assertEquals(job_list2._job_list[index].status, job.status) + self.assertEqual(job_list2._date_list, job_list._date_list) + self.assertEqual(job_list2._member_list, job_list._member_list) + self.assertEqual(job_list2._chunk_list, job_list._chunk_list) + self.assertEqual(job_list2.parameters, job_list.parameters) + job_list3 = self.new_job_list(factory,temp_dir) + job_list3.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=False, + ) + job_list3.update_from_file = Mock() + job_list3.update_from_file.return_value = False + job_list3.update_list(as_conf, False) + # assert + # check that name is the same + for index, job in enumerate(job_list._job_list): + self.assertEquals(job_list3._job_list[index].name, job.name) + # check that status is the same + for index,job in enumerate(job_list._job_list): + self.assertEquals(job_list3._job_list[index].status, job.status) + self.assertEqual(job_list3._date_list, job_list._date_list) + self.assertEqual(job_list3._member_list, job_list._member_list) + self.assertEqual(job_list3._chunk_list, job_list._chunk_list) + self.assertEqual(job_list3.parameters, job_list.parameters) + # DELETE WHEN EDGELESS TEST + job_list3._job_list[0].dependencies = {"not_exist":None} + job_list3._delete_edgeless_jobs() + self.assertEqual(len(job_list3._job_list), 1) + # Update Mayor Version test ( 4.0 -> 4.1) + job_list3.graph = DiGraph() + job_list3.save() + job_list3 = self.new_job_list(factory,temp_dir) + job_list3.update_genealogy = Mock(wraps=job_list3.update_genealogy) + job_list3.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=False, + ) + # assert update_genealogy called with right values + # When using an 4.0 experiment, the pkl has to be recreated and act as a new one. + job_list3.update_genealogy.assert_called_once_with() + + # Test when the graph previous run has more jobs than the current run + job_list3.graph.add_node("fake-node",job=job_list3._job_list[0]) + job_list3.save() + job_list3.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=False, + ) + self.assertEqual(len(job_list3.graph.nodes),len(job_list3._job_list)) + # Test when the graph previous run has fewer jobs than the current run + as_conf.experiment_data["JOBS"]["fake-section3"] = dict() + as_conf.experiment_data["JOBS"]["fake-section3"]["file"] = "fake-file3" + as_conf.experiment_data["JOBS"]["fake-section3"]["running"] = "once" + job_list3.generate( + as_conf=as_conf, + date_list=date_list, + member_list=member_list, + num_chunks=num_chunks, + chunk_ini=1, + parameters=parameters, + date_format='H', + default_retrials=9999, + default_job_type=Type.BASH, + wrapper_jobs={}, + new=False, + ) + self.assertEqual(len(job_list3.graph.nodes), len(job_list3._job_list)) + for node in job_list3.graph.nodes: + # if name is in the job_list + if node in [job.name for job in job_list3._job_list]: + self.assertTrue(job_list3.graph.nodes[node]["job"] in job_list3._job_list) + + + + + + def _createDummyJobWithStatus(self, status): job_name = str(randrange(999999, 999999999)) @@ -293,3 +602,4 @@ class FakeBasicConfig: LOCAL_PROJ_DIR = '/dummy/local/proj/dir' DEFAULT_PLATFORMS_CONF = '' DEFAULT_JOBS_CONF = '' + STRUCTURES_DIR = '/dummy/structure/dir' \ No newline at end of file diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index c446ca431b5ddf44318ef5d5e92e04ecf014abae..a5b1085cf8b31c96e54553ce558a0220f757b2b0 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -4,7 +4,7 @@ import os from pathlib import Path import inspect import tempfile -from mock import MagicMock +from mock import MagicMock, ANY from mock import patch from autosubmit.job.job import Job @@ -43,11 +43,8 @@ class TestJobPackage(TestCase): self.job_package_wrapper = None self.experiment_id = 'random-id' self._wrapper_factory = MagicMock() - self.config = FakeBasicConfig self.config.read = MagicMock() - - with patch.object(Path, 'exists') as mock_exists: mock_exists.return_value = True self.as_conf = AutosubmitConfig(self.experiment_id, self.config, YAMLParserFactory()) @@ -59,11 +56,13 @@ class TestJobPackage(TestCase): self.job_list = JobList(self.experiment_id, self.config, YAMLParserFactory(), JobListPersistenceDb(self.temp_directory, 'db'), self.as_conf) self.parser_mock = MagicMock(spec='SafeConfigParser') - + for job in self.jobs: + job._init_runtime_parameters() self.platform.max_waiting_jobs = 100 self.platform.total_jobs = 100 self.as_conf.experiment_data["WRAPPERS"]["WRAPPERS"] = options self._wrapper_factory.as_conf = self.as_conf + self.jobs[0].wallclock = "00:00" self.jobs[0].threads = "1" self.jobs[0].tasks = "1" @@ -87,6 +86,7 @@ class TestJobPackage(TestCase): self.jobs[1]._platform = self.platform + self.wrapper_type = options.get('TYPE', 'vertical') self.wrapper_policy = options.get('POLICY', 'flexible') self.wrapper_method = options.get('METHOD', 'ASThread') @@ -107,6 +107,9 @@ class TestJobPackage(TestCase): self.platform.serial_partition = "debug-serial" self.jobs = [Job('dummy1', 0, Status.READY, 0), Job('dummy2', 0, Status.READY, 0)] + for job in self.jobs: + job._init_runtime_parameters() + self.jobs[0]._platform = self.jobs[1]._platform = self.platform self.job_package = JobPackageSimple(self.jobs) def test_default_parameters(self): @@ -117,7 +120,6 @@ class TestJobPackage(TestCase): 'POLICY': "flexible", 'EXTEND_WALLCLOCK': 0, } - self.setUpWrappers(options) self.assertEqual(self.job_package_wrapper.wrapper_type, "vertical") self.assertEqual(self.job_package_wrapper.jobs_in_wrapper, "None") @@ -177,28 +179,26 @@ class TestJobPackage(TestCase): def test_job_package_platform_getter(self): self.assertEqual(self.platform, self.job_package.platform) - @patch("builtins.open",MagicMock()) - def test_job_package_submission(self): - # arrange - MagicMock().write = MagicMock() - + @patch('multiprocessing.cpu_count') + def test_job_package_submission(self, mocked_cpu_count): + # N.B.: AS only calls ``_create_scripts`` if you have less jobs than threads. + # So we simply set threads to be greater than the amount of jobs. + mocked_cpu_count.return_value = len(self.jobs) + 1 for job in self.jobs: job._tmp_path = MagicMock() - job._get_paramiko_template = MagicMock("false","empty") + job._get_paramiko_template = MagicMock("false", "empty") + job.update_parameters = MagicMock() self.job_package._create_scripts = MagicMock() self.job_package._send_files = MagicMock() self.job_package._do_submission = MagicMock() - for job in self.jobs: - job.update_parameters = MagicMock() + # act self.job_package.submit('fake-config', 'fake-params') # assert for job in self.jobs: job.update_parameters.assert_called_once_with('fake-config', 'fake-params') + self.job_package._create_scripts.is_called_once_with() self.job_package._send_files.is_called_once_with() self.job_package._do_submission.is_called_once_with() - - def test_wrapper_parameters(self): - pass \ No newline at end of file diff --git a/test/unit/test_wrappers.py b/test/unit/test_wrappers.py index c2235c6b7f18c3a23b0076c7783ac587c75b618e..c005020b87149a6862fff5447a2315d7c440b2ae 100644 --- a/test/unit/test_wrappers.py +++ b/test/unit/test_wrappers.py @@ -1,10 +1,15 @@ +from bscearth.utils.date import sum_str_hours +from operator import attrgetter + import shutil import tempfile from unittest import TestCase from mock import MagicMock + +import log.log from autosubmit.job.job_packager import JobPackager -from autosubmit.job.job_packages import JobPackageVertical +from autosubmit.job.job_packages import JobPackageVertical, JobPackageHorizontal, JobPackageHorizontalVertical , JobPackageVerticalHorizontal, JobPackageSimple from autosubmit.job.job import Job from autosubmit.job.job_list import JobList from autosubmit.job.job_dict import DicJobs @@ -1418,6 +1423,407 @@ class TestWrappers(TestCase): self.assertDictEqual(self.job_list._create_sorted_dict_jobs( "s2 s3 s5"), ordered_jobs_by_date_member) + def test_check_real_package_wrapper_limits(self): + # want to test self.job_packager.check_real_package_wrapper_limits(package,max_jobs_to_submit,packages_to_submit) + date_list = ["d1"] + member_list = ["m1", "m2"] + chunk_list = [1, 2, 3, 4] + for section,s_value in self.workflows['basic']['sections'].items(): + self.as_conf.jobs_data[section] = s_value + self._createDummyJobs( + self.workflows['basic'], date_list, member_list, chunk_list) + + self.job_list.get_job_by_name( + 'expid_d1_m1_s1').status = Status.COMPLETED + self.job_list.get_job_by_name( + 'expid_d1_m2_s1').status = Status.COMPLETED + + self.job_list.get_job_by_name('expid_d1_m1_1_s2').status = Status.READY + self.job_list.get_job_by_name('expid_d1_m2_1_s2').status = Status.READY + + wrapper_expression = "s2 s3" + d1_m1_1_s2 = self.job_list.get_job_by_name('expid_d1_m1_1_s2') + d1_m1_2_s2 = self.job_list.get_job_by_name('expid_d1_m1_2_s2') + d1_m1_3_s2 = self.job_list.get_job_by_name('expid_d1_m1_3_s2') + d1_m1_4_s2 = self.job_list.get_job_by_name('expid_d1_m1_4_s2') + d1_m2_1_s2 = self.job_list.get_job_by_name('expid_d1_m2_1_s2') + d1_m2_2_s2 = self.job_list.get_job_by_name('expid_d1_m2_2_s2') + d1_m2_3_s2 = self.job_list.get_job_by_name('expid_d1_m2_3_s2') + d1_m2_4_s2 = self.job_list.get_job_by_name('expid_d1_m2_4_s2') + + d1_m1_1_s3 = self.job_list.get_job_by_name('expid_d1_m1_1_s3') + d1_m1_2_s3 = self.job_list.get_job_by_name('expid_d1_m1_2_s3') + d1_m1_3_s3 = self.job_list.get_job_by_name('expid_d1_m1_3_s3') + d1_m1_4_s3 = self.job_list.get_job_by_name('expid_d1_m1_4_s3') + d1_m2_1_s3 = self.job_list.get_job_by_name('expid_d1_m2_1_s3') + d1_m2_2_s3 = self.job_list.get_job_by_name('expid_d1_m2_2_s3') + d1_m2_3_s3 = self.job_list.get_job_by_name('expid_d1_m2_3_s3') + d1_m2_4_s3 = self.job_list.get_job_by_name('expid_d1_m2_4_s3') + + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"] = dict() + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"]["m1"] = [d1_m1_1_s2, d1_m1_1_s3, d1_m1_2_s2, d1_m1_2_s3, + d1_m1_3_s2, d1_m1_3_s3, d1_m1_4_s2, d1_m1_4_s3] + + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"]["m2"] = [d1_m2_1_s2, d1_m2_1_s3, d1_m2_2_s2, d1_m2_2_s3, + d1_m2_3_s2, d1_m2_3_s3, d1_m2_4_s2, d1_m2_4_s3] + + self.job_packager.jobs_in_wrapper = wrapper_expression + + self.job_packager.retrials = 0 + # test vertical-wrapper + self.job_packager.wrapper_type["WRAPPER_V"] = 'vertical' + self.job_packager.current_wrapper_section = "WRAPPER_V" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section] = {} + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "vertical" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["JOBS_IN_WRAPPER"] = "S2 S3" + package_m1_s2_s3 = [d1_m1_1_s2, d1_m1_1_s3, d1_m1_2_s2, d1_m1_2_s3] + package_m2_s2_s3 = [d1_m2_1_s2, d1_m2_1_s3, d1_m2_2_s2, d1_m2_2_s3] + + packages_v = [JobPackageVertical( + package_m1_s2_s3, configuration=self.as_conf), + JobPackageVertical(package_m2_s2_s3, configuration=self.as_conf)] + + for package in packages_v: + min_v, min_h, balanced = self.job_packager.check_real_package_wrapper_limits(package) + self.assertTrue(balanced) + self.assertEqual(min_v, 4) + self.assertEqual(min_h, 1) + # test horizontal-wrapper + + self.job_packager.wrapper_type["WRAPPER_H"] = 'horizontal' + self.job_packager.current_wrapper_section = "WRAPPER_H" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section] = {} + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "horizontal" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["JOBS_IN_WRAPPER"] = "S2 S3" + packages_h = [JobPackageHorizontal( + package_m1_s2_s3, configuration=self.as_conf), + JobPackageHorizontal(package_m2_s2_s3, configuration=self.as_conf)] + for package in packages_h: + min_v, min_h, balanced = self.job_packager.check_real_package_wrapper_limits(package) + self.assertTrue(balanced) + self.assertEqual(min_v, 1) + self.assertEqual(min_h, 4) + # test horizontal-vertical + self.job_packager.wrapper_type["WRAPPER_HV"] = 'horizontal-vertical' + self.job_packager.current_wrapper_section = "WRAPPER_HV" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section] = {} + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "horizontal-vertical" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["JOBS_IN_WRAPPER"] = "S2 S3" + jobs_resources = dict() + #### + total_wallclock = '00:00' + self._current_processors = 0 + current_package = [package_m1_s2_s3,package_m2_s2_s3] + max_procs = 99999 + #### + packages_hv = [JobPackageHorizontalVertical(current_package, max_procs, total_wallclock,jobs_resources=jobs_resources, configuration=self.as_conf, wrapper_section=self.job_packager.current_wrapper_section)] + + for package in packages_hv: + min_v, min_h, balanced = self.job_packager.check_real_package_wrapper_limits(package) + self.assertTrue(balanced) + self.assertEqual(min_v, 2) + self.assertEqual(min_h, 4) + # unbalanced package + unbalanced_package = [d1_m2_1_s2, d1_m2_1_s3, d1_m2_2_s2] + current_package = [package_m1_s2_s3,unbalanced_package,package_m2_s2_s3] + packages_hv_unbalanced = [JobPackageHorizontalVertical(current_package, max_procs, total_wallclock, jobs_resources=jobs_resources, configuration=self.as_conf, wrapper_section=self.job_packager.current_wrapper_section)] + for package in packages_hv_unbalanced: + min_v, min_h, balanced = self.job_packager.check_real_package_wrapper_limits(package) + self.assertFalse(balanced) + self.assertEqual(min_v, 3) + self.assertEqual(min_h, 3) + # test vertical-horizontal + self.job_packager.wrapper_type["WRAPPER_VH"] = 'vertical-horizontal' + self.job_packager.current_wrapper_section = "WRAPPER_VH" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section] = {} + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "vertical-horizontal" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["JOBS_IN_WRAPPER"] = "S2 S3" + current_package = [package_m1_s2_s3,package_m2_s2_s3] + packages_vh = [JobPackageVerticalHorizontal( + current_package, max_procs, total_wallclock, jobs_resources=jobs_resources, configuration=self.as_conf, wrapper_section=self.job_packager.current_wrapper_section)] + for package in packages_vh: + min_v, min_h, balanced = self.job_packager.check_real_package_wrapper_limits(package) + self.assertTrue(balanced) + self.assertEqual(min_v, 4) + self.assertEqual(min_h, 2) + current_package = [package_m1_s2_s3,unbalanced_package,package_m2_s2_s3] + packages_vh_unbalanced = [JobPackageVerticalHorizontal( + current_package, max_procs, total_wallclock, jobs_resources=jobs_resources, configuration=self.as_conf, wrapper_section=self.job_packager.current_wrapper_section)] + for package in packages_vh_unbalanced: + min_v, min_h, balanced = self.job_packager.check_real_package_wrapper_limits(package) + self.assertFalse(balanced) + self.assertEqual(min_v, 3) + self.assertEqual(min_h, 3) + + + def test_check_jobs_to_run_first(self): + # want to test self.job_packager.check_jobs_to_run_first(package) + date_list = ["d1"] + member_list = ["m1", "m2"] + chunk_list = [1, 2, 3, 4] + for section, s_value in self.workflows['basic']['sections'].items(): + self.as_conf.jobs_data[section] = s_value + self._createDummyJobs( + self.workflows['basic'], date_list, member_list, chunk_list) + + self.job_list.get_job_by_name( + 'expid_d1_m1_s1').status = Status.COMPLETED + self.job_list.get_job_by_name( + 'expid_d1_m2_s1').status = Status.COMPLETED + + self.job_list.get_job_by_name('expid_d1_m1_1_s2').status = Status.READY + self.job_list.get_job_by_name('expid_d1_m2_1_s2').status = Status.READY + + wrapper_expression = "s2 s3" + d1_m1_1_s2 = self.job_list.get_job_by_name('expid_d1_m1_1_s2') + d1_m1_2_s2 = self.job_list.get_job_by_name('expid_d1_m1_2_s2') + d1_m1_3_s2 = self.job_list.get_job_by_name('expid_d1_m1_3_s2') + d1_m1_4_s2 = self.job_list.get_job_by_name('expid_d1_m1_4_s2') + d1_m2_1_s2 = self.job_list.get_job_by_name('expid_d1_m2_1_s2') + d1_m2_2_s2 = self.job_list.get_job_by_name('expid_d1_m2_2_s2') + d1_m2_3_s2 = self.job_list.get_job_by_name('expid_d1_m2_3_s2') + d1_m2_4_s2 = self.job_list.get_job_by_name('expid_d1_m2_4_s2') + + d1_m1_1_s3 = self.job_list.get_job_by_name('expid_d1_m1_1_s3') + d1_m1_2_s3 = self.job_list.get_job_by_name('expid_d1_m1_2_s3') + d1_m1_3_s3 = self.job_list.get_job_by_name('expid_d1_m1_3_s3') + d1_m1_4_s3 = self.job_list.get_job_by_name('expid_d1_m1_4_s3') + d1_m2_1_s3 = self.job_list.get_job_by_name('expid_d1_m2_1_s3') + d1_m2_2_s3 = self.job_list.get_job_by_name('expid_d1_m2_2_s3') + d1_m2_3_s3 = self.job_list.get_job_by_name('expid_d1_m2_3_s3') + d1_m2_4_s3 = self.job_list.get_job_by_name('expid_d1_m2_4_s3') + + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"] = dict() + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"]["m1"] = [d1_m1_1_s2, d1_m1_1_s3, d1_m1_2_s2, + d1_m1_2_s3, + d1_m1_3_s2, d1_m1_3_s3, d1_m1_4_s2, + d1_m1_4_s3] + + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"]["m2"] = [d1_m2_1_s2, d1_m2_1_s3, d1_m2_2_s2, + d1_m2_2_s3, + d1_m2_3_s2, d1_m2_3_s3, d1_m2_4_s2, + d1_m2_4_s3] + + self.job_packager.jobs_in_wrapper = wrapper_expression + + self.job_packager.retrials = 0 + # test vertical-wrapper + self.job_packager.wrapper_type["WRAPPER_V"] = 'vertical' + self.job_packager.current_wrapper_section = "WRAPPER_V" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section] = {} + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "vertical" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["JOBS_IN_WRAPPER"] = "S2 S3" + package_m1_s2_s3 = [d1_m1_1_s2, d1_m1_1_s3, d1_m1_2_s2, d1_m1_2_s3] + + packages_v = [JobPackageVertical(package_m1_s2_s3, configuration=self.as_conf)] + self.job_packager._jobs_list.jobs_to_run_first = [] + for p in packages_v: + p2, run_first = self.job_packager.check_jobs_to_run_first(p) + self.assertEqual(p2.jobs, p.jobs) + self.assertEqual(run_first, False) + self.job_packager._jobs_list.jobs_to_run_first = [d1_m1_1_s2, d1_m1_1_s3] + for p in packages_v: + p2, run_first = self.job_packager.check_jobs_to_run_first(p) + self.assertEqual(p2.jobs, [d1_m1_1_s2, d1_m1_1_s3]) + self.assertEqual(run_first, True) + + def test_calculate_wrapper_bounds(self): + # want to test self.job_packager.calculate_wrapper_bounds(section_list) + self.job_packager.current_wrapper_section = "WRAPPER" + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section] = {} + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "vertical" + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["JOBS_IN_WRAPPER"] = "S2 S3" + section_list = ["S2", "S3"] + # default wrapper limits + wrapper_limits = {'max': 9999999, + 'max_by_section': {'S2': 9999999, 'S3': 9999999}, + 'max_h': 9999999, + 'max_v': 9999999, + 'min': 2, + 'min_h': 1, + 'min_v': 2 + } + returned_wrapper_limits = self.job_packager.calculate_wrapper_bounds(section_list) + self.assertDictEqual(returned_wrapper_limits, wrapper_limits) + self.job_packager._as_config.experiment_data["WRAPPERS"]["MIN_WRAPPED"] = 3 + self.job_packager._as_config.experiment_data["WRAPPERS"]["MAX_WRAPPED"] = 5 + self.job_packager._as_config.experiment_data["WRAPPERS"]["MIN_WRAPPED_H"] = 2 + self.job_packager._as_config.experiment_data["WRAPPERS"]["MIN_WRAPPED_V"] = 3 + self.job_packager._as_config.experiment_data["WRAPPERS"]["MAX_WRAPPED_H"] = 4 + self.job_packager._as_config.experiment_data["WRAPPERS"]["MAX_WRAPPED_V"] = 5 + + wrapper_limits = {'max': 5*4, + 'max_by_section': {'S2': 5*4, 'S3': 5*4}, + 'max_h': 4, + 'max_v': 5*4, + 'min': 3, + 'min_h': 2, + 'min_v': 3 + } + returned_wrapper_limits = self.job_packager.calculate_wrapper_bounds(section_list) + self.assertDictEqual(returned_wrapper_limits, wrapper_limits) + + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "horizontal" + wrapper_limits = {'max': 5*4, + 'max_by_section': {'S2': 5*4, 'S3': 5*4}, + 'max_h': 4*5, + 'max_v': 5, + 'min': 3, + 'min_h': 2, + 'min_v': 3 + } + returned_wrapper_limits = self.job_packager.calculate_wrapper_bounds(section_list) + self.assertDictEqual(returned_wrapper_limits, wrapper_limits) + + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "horizontal-vertical" + wrapper_limits = {'max': 5*4, + 'max_by_section': {'S2': 5*4, 'S3': 5*4}, + 'max_h': 4, + 'max_v': 5, + 'min': 3, + 'min_h': 2, + 'min_v': 3 + } + returned_wrapper_limits = self.job_packager.calculate_wrapper_bounds(section_list) + self.assertDictEqual(returned_wrapper_limits, wrapper_limits) + + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "vertical-horizontal" + wrapper_limits = {'max': 5*4, + 'max_by_section': {'S2': 5*4, 'S3': 5*4}, + 'max_h': 4, + 'max_v': 5, + 'min': 3, + 'min_h': 2, + 'min_v': 3 + } + returned_wrapper_limits = self.job_packager.calculate_wrapper_bounds(section_list) + self.assertDictEqual(returned_wrapper_limits, wrapper_limits) + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["MIN_WRAPPED"] = 3 + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["MAX_WRAPPED"] = 5 + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["MIN_WRAPPED_H"] = 2 + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["MIN_WRAPPED_V"] = 3 + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["MAX_WRAPPED_H"] = 4 + self.job_packager._as_config.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["MAX_WRAPPED_V"] = 5 + returned_wrapper_limits = self.job_packager.calculate_wrapper_bounds(section_list) + self.assertDictEqual(returned_wrapper_limits, wrapper_limits) + del self.job_packager._as_config.experiment_data["WRAPPERS"]["MIN_WRAPPED"] + del self.job_packager._as_config.experiment_data["WRAPPERS"]["MAX_WRAPPED"] + del self.job_packager._as_config.experiment_data["WRAPPERS"]["MIN_WRAPPED_H"] + del self.job_packager._as_config.experiment_data["WRAPPERS"]["MIN_WRAPPED_V"] + del self.job_packager._as_config.experiment_data["WRAPPERS"]["MAX_WRAPPED_H"] + del self.job_packager._as_config.experiment_data["WRAPPERS"]["MAX_WRAPPED_V"] + returned_wrapper_limits = self.job_packager.calculate_wrapper_bounds(section_list) + self.assertDictEqual(returned_wrapper_limits, wrapper_limits) + + def test_check_packages_respect_wrapper_policy(self): + # want to test self.job_packager.check_packages_respect_wrapper_policy(built_packages_tmp,packages_to_submit,max_jobs_to_submit,wrapper_limits) + date_list = ["d1"] + member_list = ["m1", "m2"] + chunk_list = [1, 2, 3, 4] + for section, s_value in self.workflows['basic']['sections'].items(): + self.as_conf.jobs_data[section] = s_value + self._createDummyJobs( + self.workflows['basic'], date_list, member_list, chunk_list) + + self.job_list.get_job_by_name( + 'expid_d1_m1_s1').status = Status.COMPLETED + self.job_list.get_job_by_name( + 'expid_d1_m2_s1').status = Status.COMPLETED + + self.job_list.get_job_by_name('expid_d1_m1_1_s2').status = Status.READY + self.job_list.get_job_by_name('expid_d1_m2_1_s2').status = Status.READY + + wrapper_expression = "s2 s3" + d1_m1_1_s2 = self.job_list.get_job_by_name('expid_d1_m1_1_s2') + d1_m1_2_s2 = self.job_list.get_job_by_name('expid_d1_m1_2_s2') + d1_m1_3_s2 = self.job_list.get_job_by_name('expid_d1_m1_3_s2') + d1_m1_4_s2 = self.job_list.get_job_by_name('expid_d1_m1_4_s2') + d1_m2_1_s2 = self.job_list.get_job_by_name('expid_d1_m2_1_s2') + d1_m2_2_s2 = self.job_list.get_job_by_name('expid_d1_m2_2_s2') + d1_m2_3_s2 = self.job_list.get_job_by_name('expid_d1_m2_3_s2') + d1_m2_4_s2 = self.job_list.get_job_by_name('expid_d1_m2_4_s2') + + d1_m1_1_s3 = self.job_list.get_job_by_name('expid_d1_m1_1_s3') + d1_m1_2_s3 = self.job_list.get_job_by_name('expid_d1_m1_2_s3') + d1_m1_3_s3 = self.job_list.get_job_by_name('expid_d1_m1_3_s3') + d1_m1_4_s3 = self.job_list.get_job_by_name('expid_d1_m1_4_s3') + d1_m2_1_s3 = self.job_list.get_job_by_name('expid_d1_m2_1_s3') + d1_m2_2_s3 = self.job_list.get_job_by_name('expid_d1_m2_2_s3') + d1_m2_3_s3 = self.job_list.get_job_by_name('expid_d1_m2_3_s3') + d1_m2_4_s3 = self.job_list.get_job_by_name('expid_d1_m2_4_s3') + + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"] = dict() + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"]["m1"] = [d1_m1_1_s2, d1_m1_1_s3, d1_m1_2_s2, + d1_m1_2_s3, + d1_m1_3_s2, d1_m1_3_s3, d1_m1_4_s2, + d1_m1_4_s3] + + self.job_list._ordered_jobs_by_date_member["WRAPPERS"]["d1"]["m2"] = [d1_m2_1_s2, d1_m2_1_s3, d1_m2_2_s2, + d1_m2_2_s3, + d1_m2_3_s2, d1_m2_3_s3, d1_m2_4_s2, + d1_m2_4_s3] + + self.job_packager.jobs_in_wrapper = wrapper_expression + + self.job_packager.retrials = 0 + # test vertical-wrapper + self.job_packager.wrapper_type["WRAPPER_V"] = 'vertical' + self.job_packager.current_wrapper_section = "WRAPPER_V" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section] = {} + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["TYPE"] = "horizontal" + self.as_conf.experiment_data["WRAPPERS"][self.job_packager.current_wrapper_section]["JOBS_IN_WRAPPER"] = "S2 S3" + packages_to_submit = [] + max_jobs_to_submit = 2 + wrapper_limits = {'max': 9999999, + 'max_by_section': {'S2': 9999999, 'S3': 9999999}, + 'max_h': 9999999, + 'max_v': 9999999, + 'min': 2, + 'min_h': 1, + 'min_v': 2 + } + package = [d1_m1_1_s2, d1_m1_1_s2, d1_m1_1_s2, d1_m1_1_s2, d1_m1_1_s2] + packages_h = [JobPackageHorizontal( + package, configuration=self.as_conf)] + + self.job_packager.wrapper_policy = {} + self.job_packager.wrapper_policy["WRAPPER_V"] = "flexible" + packages_to_submit2, max_jobs_to_submit2 = self.job_packager.check_packages_respect_wrapper_policy(packages_h, packages_to_submit, + max_jobs_to_submit, wrapper_limits) + self.assertEqual(max_jobs_to_submit2, max_jobs_to_submit-1) + self.assertEqual(packages_to_submit2, packages_h) + + wrapper_limits = {'max': 2, + 'max_by_section': {'S2': 2, 'S3': 2}, + 'max_h': 2, + 'max_v': 2, + 'min': 2, + 'min_h': 2, + 'min_v': 2 + } + self.job_packager.jobs_in_wrapper = {self.job_packager.current_wrapper_section: {'S2': 2, 'S3': 2}} + packages_to_submit = [] + packages_to_submit2, max_jobs_to_submit2 = self.job_packager.check_packages_respect_wrapper_policy(packages_h, packages_to_submit, + max_jobs_to_submit, wrapper_limits) + self.assertEqual(max_jobs_to_submit2, 0) + self.assertEqual(len(packages_to_submit2),2) + for p in packages_to_submit2: + self.assertEqual(type(p), JobPackageSimple) + + self.job_packager.wrapper_policy["WRAPPER_V"] = "mixed" + packages_to_submit = [] + with self.assertRaises(log.log.AutosubmitCritical): + self.job_packager.check_packages_respect_wrapper_policy(packages_h, packages_to_submit, max_jobs_to_submit, wrapper_limits) + + self.job_packager.wrapper_policy["WRAPPER_V"] = "strict" + packages_to_submit = [] + with self.assertRaises(log.log.AutosubmitCritical): + self.job_packager.check_packages_respect_wrapper_policy(packages_h, packages_to_submit, max_jobs_to_submit, wrapper_limits) + + #def test_build_packages(self): + # want to test self.job_packager.build_packages() + # TODO: implement this test in the future + def _createDummyJobs(self, sections_dict, date_list, member_list, chunk_list): for section, section_dict in sections_dict.get('sections').items(): running = section_dict['RUNNING'] @@ -1469,9 +1875,10 @@ class TestWrappers(TestCase): self.job_list._member_list = member_list self.job_list._chunk_list = chunk_list - self.job_list._dic_jobs = DicJobs( - self.job_list, date_list, member_list, chunk_list, "", 0,jobs_data={},experiment_data=self.as_conf.experiment_data) + self.job_list._dic_jobs = DicJobs(date_list, member_list, chunk_list, "", 0, self.as_conf) self._manage_dependencies(sections_dict) + for job in self.job_list.get_job_list(): + job._init_runtime_parameters() def _manage_dependencies(self, sections_dict): for job in self.job_list.get_job_list(): @@ -1524,6 +1931,7 @@ class TestWrappers(TestCase): return job + import inspect class FakeBasicConfig: def __init__(self):