diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 0218a8735c3a49f0fac2c1212134e18d7ea87eb7..e2d8f97b9d13499ddc188fdff8b4e2f90bdc7d52 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -41,7 +41,7 @@ import subprocess import json import tarfile import time -import pickle +import copy import os import sys import shutil @@ -70,6 +70,7 @@ from job.job_packages import JobPackageThread from job.job_package_persistence import JobPackagePersistence from job.job_list_persistence import JobListPersistenceDb from job.job_list_persistence import JobListPersistencePkl +from job.job_grouping import JobGrouping # noinspection PyPackageRequirements from bscearth.utils.log import Log from database.db_common import create_db @@ -170,12 +171,21 @@ class Autosubmit: subparser.add_argument('expid', help='experiment identifier') subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', help='chooses type of output for generated plot') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded') + group.add_argument('-fs', '--filter_status', type=str, + choices=('Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), + help='Select the original status to filter the list of jobs') group = subparser.add_mutually_exclusive_group(required=False) group.add_argument('-fl', '--list', type=str, - help='Supply the list of job names to be changed. Default = "Any". ' + help='Supply the list of job names to be filtered. Default = "Any". ' 'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"') group.add_argument('-fc', '--filter_chunks', type=str, - help='Supply the list of chunks to change the status. Default = "Any". ' + help='Supply the list of chunks to filter the list of jobs. Default = "Any". ' 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') group.add_argument('-fs', '--filter_status', type=str, choices=('Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'), @@ -218,6 +228,12 @@ class Autosubmit: subparser.add_argument('-s', '--save', action="store_true", default=False, help='Save changes to disk') subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded') # Migrate subparser = subparsers.add_parser('migrate', description="Migrate experiments from current user to another") @@ -238,6 +254,12 @@ class Autosubmit: help='hides plot window') subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf', help='chooses type of output for generated plot') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded') # Configure subparser = subparsers.add_parser('configure', description="configure database and path for autosubmit. It " @@ -287,6 +309,12 @@ class Autosubmit: help='Select the job type to filter the list of jobs') subparser.add_argument('--hide', action='store_true', default=False, help='hides plot window') + subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None, + help='Groups the jobs automatically or by date, member, chunk or split') + subparser.add_argument('-expand', type=str, + help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". ' + 'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"') + subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded') # Test Case subparser = subparsers.add_parser('testcase', description='create test case experiment') @@ -343,19 +371,19 @@ class Autosubmit: return Autosubmit.delete(args.expid, args.force) elif args.command == 'monitor': return Autosubmit.monitor(args.expid, args.output, args.list, args.filter_chunks, args.filter_status, - args.filter_type, args.hide, args.txt) + args.filter_type, args.hide, args.txt, args.group_by, args.expand, args.expand_status) elif args.command == 'stats': return Autosubmit.statistics(args.expid, args.filter_type, args.filter_period, args.output, args.hide) elif args.command == 'clean': return Autosubmit.clean(args.expid, args.project, args.plot, args.stats) elif args.command == 'recovery': - return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide) + return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide, args.group_by, args.expand, args.expand_status) elif args.command == 'check': return Autosubmit.check(args.expid) elif args.command == 'migrate': return Autosubmit.migrate(args.expid, args.offer, args.pickup) elif args.command == 'create': - return Autosubmit.create(args.expid, args.noplot, args.hide, args.output) + return Autosubmit.create(args.expid, args.noplot, args.hide, args.output, args.group_by, args.expand, args.expand_status) elif args.command == 'configure': if dialog is None or (args.databasepath is not None and args.localrootpath is not None): return Autosubmit.configure(args.databasepath, args.databasefilename, args.localrootpath, @@ -367,7 +395,8 @@ class Autosubmit: return Autosubmit.install() elif args.command == 'setstatus': return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list, - args.filter_chunks, args.filter_status, args.filter_type, args.hide) + args.filter_chunks, args.filter_status, args.filter_type, args.hide, + args.group_by, args.expand, args.expand_status) elif args.command == 'testcase': return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate, args.HPC, args.branch) @@ -741,6 +770,9 @@ class Autosubmit: package.submit(as_conf, job_list.parameters) + if hasattr(package, "name"): + job_list.packages_dict[package.name] = package.jobs + if remote_dependencies_dict and package.name in remote_dependencies_dict['name_to_id']: remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id @@ -757,7 +789,7 @@ class Autosubmit: return save @staticmethod - def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False): + def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False, group_by=None, expand=list(), expand_status=list()): """ Plots workflow graph for a given experiment with status of each job coded by node color. Plot is created in experiment's plot folder with name __