From 212ccb89f6b6df59112346d2f54355e80f25eca9 Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Wed, 13 Dec 2017 19:02:09 +0100
Subject: [PATCH 1/9] Initial solution for job SPLITS option

---
 autosubmit/job/job.py      |  1 +
 autosubmit/job/job_dict.py | 58 +++++++++++++++++++++++++++++---------
 autosubmit/job/job_list.py | 21 ++++++++++----
 test/unit/test_dic_jobs.py |  7 +++--
 4 files changed, 65 insertions(+), 22 deletions(-)

diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py
index c81a083aa..4e662971e 100644
--- a/autosubmit/job/job.py
+++ b/autosubmit/job/job.py
@@ -69,6 +69,7 @@ class Job(object):
         self.member = None
         self.date = None
         self.name = name
+        self.split = None
         self._long_name = None
         self.long_name = name
         self.date_format = ''
diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py
index 53bcdca4e..b0051577a 100644
--- a/autosubmit/job/job_dict.py
+++ b/autosubmit/job/job_dict.py
@@ -79,7 +79,8 @@ class DicJobs:
         elif running == 'chunk':
             synchronize = self.get_option(section, "SYNCHRONIZE", None)
             delay = int(self.get_option(section, "DELAY", -1))
-            self._create_jobs_chunk(section, priority, frequency, default_job_type, synchronize, delay, jobs_data)
+            splits = int(self.get_option(section, "SPLITS", 0))
+            self._create_jobs_chunk(section, priority, frequency, default_job_type, synchronize, delay, splits, jobs_data)
 
     def _create_jobs_once(self, section, priority, default_job_type, jobs_data=dict()):
         """
@@ -142,7 +143,7 @@ class DicJobs:
         conditional decision to the father which makes the call
     '''
 
-    def _create_jobs_chunk(self, section, priority, frequency, default_job_type, synchronize=None, delay=0, jobs_data=dict()):
+    def _create_jobs_chunk(self, section, priority, frequency, default_job_type, synchronize=None, delay=0, splits=0, jobs_data=dict()):
         """
         Create jobs to be run once per chunk
 
@@ -165,14 +166,27 @@ class DicJobs:
                 count += 1
                 if delay == -1 or delay < chunk:
                     if count % frequency == 0 or count == len(self._chunk_list):
-                        if synchronize == 'date':
-                            tmp_dic[chunk] = self.build_job(section, priority, None, None,
-                                                            chunk, default_job_type, jobs_data)
-                        elif synchronize == 'member':
-                            tmp_dic[chunk] = dict()
-                            for date in self._date_list:
-                                tmp_dic[chunk][date] = self.build_job(section, priority, date, None,
-                                                                  chunk, default_job_type, jobs_data)
+                        if splits > 0:
+                            if synchronize == 'date':
+                                tmp_dic[chunk] = []
+                                self._create_jobs_split(splits, section, None, None, chunk, priority,
+                                                   default_job_type, jobs_data, tmp_dic[chunk])
+                            elif synchronize == 'member':
+                                tmp_dic[chunk] = dict()
+                                for date in self._date_list:
+                                    tmp_dic[chunk][date] = []
+                                    self._create_jobs_split(splits, section, date, None, chunk, priority,
+                                                            default_job_type, jobs_data, tmp_dic[chunk][date])
+
+                        else:
+                            if synchronize == 'date':
+                                tmp_dic[chunk] = self.build_job(section, priority, None, None,
+                                                                chunk, default_job_type, jobs_data)
+                            elif synchronize == 'member':
+                                tmp_dic[chunk] = dict()
+                                for date in self._date_list:
+                                    tmp_dic[chunk][date] = self.build_job(section, priority, date, None,
+                                                                      chunk, default_job_type, jobs_data)
         # Real dic jobs assignment/creation
         self._dic[section] = dict()
         for date in self._date_list:
@@ -188,10 +202,22 @@ class DicJobs:
                                 self._dic[section][date][member][chunk] = tmp_dic[chunk]
                             elif synchronize == 'member':
                                 self._dic[section][date][member][chunk] = tmp_dic[chunk][date]
-                            else:
+
+                            if splits > 0 and synchronize is None:
+                                self._dic[section][date][member][chunk] = []
+                                self._create_jobs_split(splits, section, date, member, chunk, priority, default_job_type, jobs_data, self._dic[section][date][member][chunk])
+                            elif synchronize is None:
                                 self._dic[section][date][member][chunk] = self.build_job(section, priority, date, member,
-                                                                                         chunk, default_job_type, jobs_data)
-                            self._jobs_list.graph.add_node(self._dic[section][date][member][chunk].name)
+                                                                                             chunk, default_job_type, jobs_data)
+                                self._jobs_list.graph.add_node(self._dic[section][date][member][chunk].name)
+
+    def _create_jobs_split(self, splits, section, date, member, chunk, priority, default_job_type, jobs_data, dict):
+        total_jobs = 1
+        while total_jobs <= splits:
+            job = self.build_job(section, priority, date, member, chunk, default_job_type, jobs_data, total_jobs)
+            dict.append(job)
+            self._jobs_list.graph.add_node(job.name)
+            total_jobs += 1
 
     def get_jobs(self, section, date=None, member=None, chunk=None):
         """
@@ -257,7 +283,7 @@ class DicJobs:
                     jobs.append(dic[c])
         return jobs
 
-    def build_job(self, section, priority, date, member, chunk, default_job_type, jobs_data=dict()):
+    def build_job(self, section, priority, date, member, chunk, default_job_type, jobs_data=dict(), split=-1):
         name = self._jobs_list.expid
         if date is not None:
             name += "_" + date2str(date, self._date_format)
@@ -265,6 +291,8 @@ class DicJobs:
             name += "_" + member
         if chunk is not None:
             name += "_{0}".format(chunk)
+        if split > -1:
+            name += "_{0}".format(split)
         name += "_" + section
         if name in jobs_data:
             job = Job(name, jobs_data[name][1], jobs_data[name][2], priority)
@@ -277,6 +305,8 @@ class DicJobs:
         job.member = member
         job.chunk = chunk
         job.date_format = self._date_format
+        if split > -1:
+            job.split = split
 
         job.frequency = int(self.get_option(section, "FREQUENCY", 1))
         job.delay = int(self.get_option(section, "DELAY", -1))
diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py
index 74af5a139..8f7556077 100644
--- a/autosubmit/job/job_list.py
+++ b/autosubmit/job/job_list.py
@@ -66,7 +66,7 @@ class JobList:
         self._persistence = job_list_persistence
         self._graph = DiGraph()
 
-        self._packages_dict = dict()
+        self.packages_dict = dict()
         self._ordered_jobs_by_date_member = dict()
 
     @property
@@ -161,8 +161,13 @@ class JobList:
             dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs)
 
             for job in dic_jobs.get_jobs(job_section):
-                JobList._manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys,
-                                                 dependencies, graph)
+                num_jobs = 1
+                if isinstance(job, list):
+                    num_jobs = len(job)
+                for i in range(num_jobs):
+                    _job = job[i] if num_jobs > 1 else job
+                    JobList._manage_job_dependencies(dic_jobs, _job, date_list, member_list, chunk_list, dependencies_keys,
+                                                     dependencies, graph)
 
     @staticmethod
     def _manage_dependencies(dependencies_keys, dic_jobs):
@@ -189,6 +194,7 @@ class JobList:
                                  graph):
         for key in dependencies_keys:
             dependency = dependencies[key]
+
             skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, chunk_list,
                                                                                  job.member, member_list,
                                                                                  job.date, date_list,
@@ -199,8 +205,13 @@ class JobList:
             for parent in dic_jobs.get_jobs(dependency.section, date, member, chunk):
                 # only creates the dependency in the graph if the delay is not defined or if the chunk is greater than it
                 if dependency.delay == -1 or chunk > dependency.delay:
-                    job.add_parent(parent)
-                    graph.add_edge(parent.name, job.name)
+                    num_parents = 1
+                    if isinstance(parent, list):
+                        num_parents = len(parent)
+                    for i in range(num_parents):
+                        _parent = parent[i] if num_parents > 1 else parent
+                        job.add_parent(_parent)
+                        graph.add_edge(_parent.name, job.name)
 
             JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member,
                                                            member_list, dependency.section, graph)
diff --git a/test/unit/test_dic_jobs.py b/test/unit/test_dic_jobs.py
index 7448c02ff..6a595479f 100644
--- a/test/unit/test_dic_jobs.py
+++ b/test/unit/test_dic_jobs.py
@@ -98,9 +98,10 @@ class TestDicJobs(TestCase):
         frequency = 123
         synchronize = 'date'
         delay = -1
+        splits = 0
         self.parser_mock.has_option = Mock(return_value=True)
         self.parser_mock.get = Mock(return_value='chunk')
-        self.dictionary.get_option = Mock(side_effect=[frequency, synchronize, delay])
+        self.dictionary.get_option = Mock(side_effect=[frequency, synchronize, delay, splits])
         self.dictionary._create_jobs_once = Mock()
         self.dictionary._create_jobs_startdate = Mock()
         self.dictionary._create_jobs_member = Mock()
@@ -113,7 +114,7 @@ class TestDicJobs(TestCase):
         self.dictionary._create_jobs_once.assert_not_called()
         self.dictionary._create_jobs_startdate.assert_not_called()
         self.dictionary._create_jobs_member.assert_not_called()
-        self.dictionary._create_jobs_chunk.assert_called_once_with(section, priority, frequency, Type.BASH, synchronize, delay, {})
+        self.dictionary._create_jobs_chunk.assert_called_once_with(section, priority, frequency, Type.BASH, synchronize, delay, splits, {})
 
     def test_dic_creates_right_jobs_by_startdate(self):
         # arrange
@@ -158,7 +159,7 @@ class TestDicJobs(TestCase):
         self.dictionary.build_job = Mock(return_value=mock_section)
 
         # act
-        self.dictionary._create_jobs_chunk(mock_section.name, priority, frequency, Type.BASH, dict())
+        self.dictionary._create_jobs_chunk(mock_section.name, priority, frequency, Type.BASH)
 
         # assert
         self.assertEquals(len(self.date_list) * len(self.member_list) * len(self.chunk_list),
-- 
GitLab


From d403026de101edca78113177ace22b08da056747 Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Thu, 14 Dec 2017 19:09:19 +0100
Subject: [PATCH 2/9] Dependencies for SPLIT jobs

---
 autosubmit/job/job_dict.py  |  2 +-
 autosubmit/job/job_list.py  | 36 +++++++++++++++++++++++++++++++-----
 autosubmit/job/job_utils.py |  3 ++-
 3 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py
index b0051577a..5ef753436 100644
--- a/autosubmit/job/job_dict.py
+++ b/autosubmit/job/job_dict.py
@@ -140,7 +140,7 @@ class DicJobs:
 
     '''
         Maybe a good choice could be split this function or ascend the
-        conditional decision to the father which makes the call
+        conditional decision to the parent which makes the call
     '''
 
     def _create_jobs_chunk(self, section, priority, frequency, default_job_type, synchronize=None, delay=0, splits=0, jobs_data=dict()):
diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py
index 8f7556077..a96ea682d 100644
--- a/autosubmit/job/job_list.py
+++ b/autosubmit/job/job_list.py
@@ -180,15 +180,36 @@ class JobList:
                 sign = '-' if '-' in key else '+'
                 key_split = key.split(sign)
                 section = key_split[0]
+                splits = None
+                if '[' in section:
+                    section_name = section[0:section.find("[")]
+                    splits_section = int(dic_jobs.get_option(section_name, 'SPLITS', 0))
+                    splits = JobList._calculate_splits_dependencies(section, splits_section)
+                    section = section_name
                 distance = key_split[1]
                 dependency_running_type = dic_jobs.get_option(section, 'RUNNING', 'once').lower()
-                dependency = Dependency(section, int(distance), dependency_running_type, sign)
+                dependency = Dependency(section, int(distance), dependency_running_type, sign, splits=splits)
 
             delay = int(dic_jobs.get_option(section, 'DELAY', -1))
             dependency.delay = delay
             dependencies[key] = dependency
         return dependencies
 
+    @staticmethod
+    def _calculate_splits_dependencies(section, max_splits):
+        splits_list = section[section.find("[") + 1:section.find("]")]
+        splits = []
+        for str_split in splits_list.split(","):
+            if str_split.find(":") != -1:
+                numbers = str_split.split(":")
+                max_splits = min(int(numbers[1]), max_splits)
+                for count in range(int(numbers[0]), max_splits+1):
+                    splits.append(int(str(count).zfill(len(numbers[0]))))
+            else:
+                if int(str_split) <= max_splits:
+                    splits.append(int(str_split))
+        return splits
+
     @staticmethod
     def _manage_job_dependencies(dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies,
                                  graph):
@@ -203,13 +224,18 @@ class JobList:
                 continue
 
             for parent in dic_jobs.get_jobs(dependency.section, date, member, chunk):
-                # only creates the dependency in the graph if the delay is not defined or if the chunk is greater than it
                 if dependency.delay == -1 or chunk > dependency.delay:
                     num_parents = 1
                     if isinstance(parent, list):
-                        num_parents = len(parent)
+                        if job.split is not None:
+                            parent = filter(lambda _parent: _parent.split == job.split, parent)[0]
+                        else:
+                            if dependency.splits is not None:
+                                parent = filter(lambda _parent: _parent.split in dependency.splits, parent)
+                            num_parents = len(parent)
+
                     for i in range(num_parents):
-                        _parent = parent[i] if num_parents > 1 else parent
+                        _parent = parent[i] if isinstance(parent, list) else parent
                         job.add_parent(_parent)
                         graph.add_edge(_parent.name, job.name)
 
@@ -320,7 +346,7 @@ class JobList:
 
         sections_running_type_map = dict()
         for section in wrapper_expression.split(" "):
-            sections_running_type_map[section] = self._dic_jobs.get_option(section, "RUNNING", '')
+            sections_running_type_map[section] = self._dic_jobs.get_option(section, "RUNNING", 'once')
 
         for date in self._date_list:
             str_date = self._get_date(date)
diff --git a/autosubmit/job/job_utils.py b/autosubmit/job/job_utils.py
index 9b9c5efe6..c772f0e63 100644
--- a/autosubmit/job/job_utils.py
+++ b/autosubmit/job/job_utils.py
@@ -44,10 +44,11 @@ class Dependency(object):
 
     """
 
-    def __init__(self, section, distance=None, running=None, sign=None, delay=-1):
+    def __init__(self, section, distance=None, running=None, sign=None, delay=-1, splits=None):
         self.section = section
         self.distance = distance
         self.running = running
         self.sign = sign
         self.delay = delay
+        self.splits = splits
 
-- 
GitLab


From 25b9b009f234f795a40a0769cdc9ba75cf5ba9db Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Fri, 15 Dec 2017 11:59:48 +0100
Subject: [PATCH 3/9] Small refactoring

---
 autosubmit/job/job.py      | 19 ++++++++++++-------
 autosubmit/job/job_list.py | 23 ++++++++++++++---------
 2 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py
index 4e662971e..782cfd5b5 100644
--- a/autosubmit/job/job.py
+++ b/autosubmit/job/job.py
@@ -274,16 +274,21 @@ class Job(object):
         self.fail_count += 1
 
     # Maybe should be renamed to the plural?
-    def add_parent(self, *new_parent):
+    def add_parent(self, *parents):
         """
         Add parents for the job. It also adds current job as a child for all the new parents
 
-        :param new_parent: job's parents to add
-        :type new_parent: *Job
-        """
-        for parent in new_parent:
-            self._parents.add(parent)
-            parent.__add_child(self)
+        :param parents: job's parents to add
+        :type parents: *Job
+        """
+        for parent in parents:
+            num_parents = 1
+            if isinstance(parent, list):
+                num_parents = len(parent)
+            for i in range(num_parents):
+                new_parent = parent[i] if isinstance(parent, list) else parent
+                self._parents.add(new_parent)
+                new_parent.__add_child(self)
 
     def __add_child(self, new_child):
         """
diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py
index a96ea682d..40b8ca262 100644
--- a/autosubmit/job/job_list.py
+++ b/autosubmit/job/job_list.py
@@ -225,19 +225,15 @@ class JobList:
 
             for parent in dic_jobs.get_jobs(dependency.section, date, member, chunk):
                 if dependency.delay == -1 or chunk > dependency.delay:
-                    num_parents = 1
                     if isinstance(parent, list):
                         if job.split is not None:
                             parent = filter(lambda _parent: _parent.split == job.split, parent)[0]
                         else:
                             if dependency.splits is not None:
                                 parent = filter(lambda _parent: _parent.split in dependency.splits, parent)
-                            num_parents = len(parent)
 
-                    for i in range(num_parents):
-                        _parent = parent[i] if isinstance(parent, list) else parent
-                        job.add_parent(_parent)
-                        graph.add_edge(_parent.name, job.name)
+                    job.add_parent(parent)
+                    JobList._add_edge(graph, job, parent)
 
             JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member,
                                                            member_list, dependency.section, graph)
@@ -303,7 +299,7 @@ class JobList:
                 for distance in range(1, max_distance):
                     for parent in dic_jobs.get_jobs(section_name, date, member, chunk - distance):
                         job.add_parent(parent)
-                        graph.add_edge(parent.name, job.name)
+                        JobList._add_edge(graph, job, parent)
             elif job.member is not None:
                 member_index = member_list.index(job.member)
                 max_distance = (member_index + 1) % job.frequency
@@ -313,7 +309,7 @@ class JobList:
                     for parent in dic_jobs.get_jobs(section_name, date,
                                                     member_list[member_index - distance], chunk):
                         job.add_parent(parent)
-                        graph.add_edge(parent.name, job.name)
+                        JobList._add_edge(graph, job, parent)
             elif job.date is not None:
                 date_index = date_list.index(job.date)
                 max_distance = (date_index + 1) % job.frequency
@@ -323,7 +319,16 @@ class JobList:
                     for parent in dic_jobs.get_jobs(section_name, date_list[date_index - distance],
                                                     member, chunk):
                         job.add_parent(parent)
-                        graph.add_edge(parent.name, job.name)
+                        JobList._add_edge(graph, job, parent)
+
+    @staticmethod
+    def _add_edge(graph, job, parents):
+        num_parents = 1
+        if isinstance(parents, list):
+            num_parents = len(parents)
+        for i in range(num_parents):
+            parent = parents[i] if isinstance(parents, list) else parents
+            graph.add_edge(parent.name, job.name)
 
     @staticmethod
     def _create_jobs(dic_jobs, parser, priority, default_job_type, jobs_data=dict()):
-- 
GitLab


From c42c34ce0ec35c65e6517db3c9581d2a9f28eaf8 Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Fri, 5 Jan 2018 17:44:36 +0100
Subject: [PATCH 4/9] Grouping in visualization, missing status

---
 autosubmit/autosubmit.py      | 76 ++++++++++++++++++++++++++++++-----
 autosubmit/job/job_list.py    | 11 ++---
 autosubmit/monitor/monitor.py | 60 ++++++++++++++++++++-------
 3 files changed, 115 insertions(+), 32 deletions(-)

diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py
index 0218a8735..803e78eb8 100644
--- a/autosubmit/autosubmit.py
+++ b/autosubmit/autosubmit.py
@@ -170,12 +170,14 @@ class Autosubmit:
             subparser.add_argument('expid', help='experiment identifier')
             subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf',
                                    help='chooses type of output for generated plot')
+            subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split'), default=None,
+                                   help='Groups the jobs by date, member, chunk or split')
             group = subparser.add_mutually_exclusive_group(required=False)
             group.add_argument('-fl', '--list', type=str,
                                help='Supply the list of job names to be changed. Default = "Any". '
                                     'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"')
             group.add_argument('-fc', '--filter_chunks', type=str,
-                               help='Supply the list of chunks to change the status. Default = "Any". '
+                               help='Supply the list of chunks to filter the list of jobs. Default = "Any". '
                                     'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
             group.add_argument('-fs', '--filter_status', type=str,
                                choices=('Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
@@ -343,7 +345,7 @@ class Autosubmit:
                 return Autosubmit.delete(args.expid, args.force)
             elif args.command == 'monitor':
                 return Autosubmit.monitor(args.expid, args.output, args.list, args.filter_chunks, args.filter_status,
-                                          args.filter_type, args.hide, args.txt)
+                                          args.filter_type, args.hide, args.txt, args.group_by)
             elif args.command == 'stats':
                 return Autosubmit.statistics(args.expid, args.filter_type, args.filter_period, args.output, args.hide)
             elif args.command == 'clean':
@@ -741,6 +743,9 @@ class Autosubmit:
 
                     package.submit(as_conf, job_list.parameters)
 
+                    if hasattr(package, "name"):
+                        job_list.packages_dict[package.name] = package.jobs
+
                     if remote_dependencies_dict and package.name in remote_dependencies_dict['name_to_id']:
                         remote_dependencies_dict['name_to_id'][package.name] = package.jobs[0].id
 
@@ -757,7 +762,7 @@ class Autosubmit:
         return save
 
     @staticmethod
-    def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False):
+    def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False, group_by=None):
         """
         Plots workflow graph for a given experiment with status of each job coded by node color.
         Plot is created in experiment's plot folder with name <expid>_<date>_<time>.<file_format>
@@ -868,12 +873,14 @@ class Autosubmit:
             packages = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"),
                                                      "job_packages_" + expid).load()
 
+        groups_dict = Autosubmit._group_jobs(group_by, jobs, job_list)
+
         monitor_exp = Monitor()
 
         if txt_only:
             monitor_exp.generate_output_txt(expid, jobs, exp_path+"/tmp/LOG_"+expid)
         else:
-            monitor_exp.generate_output(expid, jobs, os.path.join(exp_path, "/tmp/LOG_", expid), file_format, packages, not hide)
+            monitor_exp.generate_output(expid, jobs, os.path.join(exp_path, "/tmp/LOG_", expid), file_format, packages, not hide, groups_dict)
 
         return True
 
@@ -1098,12 +1105,12 @@ class Autosubmit:
     @staticmethod
     def migrate(experiment_id, offer, pickup):
         """
-        Migrates experiment files from current to other user. 
+        Migrates experiment files from current to other user.
         It takes mapping information for new user from config files.
-        
+
         :param experiment_id: experiment identifier:
-        :param pickup: 
-        :param offer: 
+        :param pickup:
+        :param offer:
         """
         log_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, "ASlogs", 'migrate_{0}.log'.format(experiment_id))
         Log.set_file(log_file)
@@ -1591,8 +1598,8 @@ class Autosubmit:
         Archives an experiment: call clean (if experiment is of version 3 or later), compress folder
         to tar.gz and moves to year's folder
 
-        :param clean: 
-        :return: 
+        :param clean:
+        :return:
         :param expid: experiment identifier
         :type expid: str
         """
@@ -1819,6 +1826,7 @@ class Autosubmit:
                                   as_conf.get_retrials(),
                                   as_conf.get_default_job_type(),
                                   as_conf.get_wrapper_expression())
+
                 if rerun == "true":
                     chunk_list = Autosubmit._create_json(as_conf.get_chunk_list())
                     job_list.rerun(chunk_list)
@@ -1842,6 +1850,54 @@ class Autosubmit:
         except portalocker.AlreadyLocked:
             Autosubmit.show_lock_warning(expid)
 
+    @staticmethod
+    def _group_jobs(group_by, jobs, job_list):
+        groups_dict = dict()
+        date_format = job_list._dic_jobs._date_format
+
+        for date in job_list.get_date_list():
+            if group_by == 'date':
+                Autosubmit._create_group(jobs, groups_dict, group_by, date_format, date)
+            elif group_by in ['member', 'chunk', 'split']:
+                for member in job_list.get_member_list():
+                    if group_by in ['chunk', 'split']:
+                        for chunk in job_list.get_chunk_list():
+                            Autosubmit._create_group(jobs, groups_dict, group_by, date_format, date, member, chunk)
+                    else:
+                        Autosubmit._create_group(jobs, groups_dict, group_by, date_format, date, member)
+
+        return groups_dict
+
+    @staticmethod
+    def _create_group(jobs, groups_dict, group_by, date_format, date=None, member=None, chunk=None):
+        if chunk is not None:
+            name = date2str(date, date_format) + '_' + member + '_' +str(chunk)
+        elif member is not None:
+            name = date2str(date, date_format) + '_' + member
+        else:
+            name = date2str(date, date_format)
+
+        for i in reversed(range(len(jobs))):
+            job = jobs[i]
+
+            if group_by == 'date':
+                condition = (job.date == date or (job.date is None and job.chunk is not None))
+            elif group_by == 'member':
+                condition = (job.date == date and job.member == member) or (job.date in [date, None] and job.member is None and job.chunk is not None)
+            elif group_by in ['chunk']:
+                condition = (job.member in [member, None] and job.date in [date, None] and job.chunk == chunk)
+            elif group_by == 'split':
+                condition = job.split is not None
+                idx = job.name.rfind("_")
+                name = job.name[:idx - 1] + job.name[idx + 1:]
+
+            if (condition):
+                if (job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None):
+                    jobs.pop(i) #if synchronized does not remove
+                if job.name not in groups_dict:
+                    groups_dict[job.name] = list()
+                groups_dict[job.name].append(name)
+
     @staticmethod
     def _copy_code(as_conf, expid, project_type, force):
         """
diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py
index 40b8ca262..d12beb907 100644
--- a/autosubmit/job/job_list.py
+++ b/autosubmit/job/job_list.py
@@ -202,6 +202,7 @@ class JobList:
         for str_split in splits_list.split(","):
             if str_split.find(":") != -1:
                 numbers = str_split.split(":")
+                # change this to be checked in job_common.py
                 max_splits = min(int(numbers[1]), max_splits)
                 for count in range(int(numbers[0]), max_splits+1):
                     splits.append(int(str(count).zfill(len(numbers[0]))))
@@ -425,14 +426,10 @@ class JobList:
         return filtered_jobs_fake_date_member, fake_original_job_map
 
     def _get_date(self, date):
-        date_format = ''
-        if self.parameters.get('CHUNKSIZEUNIT') is 'hour':
+        if date.hour > 1:
             date_format = 'H'
-        for date in self._date_list:
-            if date.hour > 1:
-                date_format = 'H'
-            if date.minute > 1:
-                date_format = 'M'
+        if date.minute > 1:
+            date_format = 'M'
         str_date = date2str(date, date_format)
         return str_date
 
diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py
index b0f4e4f0b..22cba4065 100644
--- a/autosubmit/monitor/monitor.py
+++ b/autosubmit/monitor/monitor.py
@@ -77,7 +77,7 @@ class Monitor:
         else:
             return Monitor._table[Status.UNKNOWN]
 
-    def create_tree_list(self, expid, joblist, packages):
+    def create_tree_list(self, expid, joblist, packages, groups):
         """
         Create graph from joblist
 
@@ -127,10 +127,35 @@ class Monitor:
             if job.has_parents():
                 continue
 
-            node_job = pydotplus.Node(job.name, shape='box', style="filled",
-                                      fillcolor=self.color_status(job.status))
-            exp.add_node(node_job)
-            self._add_children(job, exp, node_job)
+            if groups is None or job.name not in groups or (job.name in groups and len(groups[job.name]) == 1):
+                node_job = pydotplus.Node(job.name, shape='box', style="filled",
+                                          fillcolor=self.color_status(job.status))
+                if job.name in groups:
+                    group = groups[job.name]
+                    node_job.obj_dict['name'] = group[0]
+                    node_job.obj_dict['attributes']['shape'] = 'box3d'
+                exp.add_node(node_job)
+                self._add_children(job, exp, node_job, groups)
+
+        if groups:
+            for job, group in groups.items():
+                if len(group) > 1:
+                    subgraph = pydotplus.graphviz.Cluster(graph_name='cluster_'+'_'.join(group))
+                    subgraph.obj_dict['attributes']['color'] = 'invis'
+
+                    previous_node = exp.get_node(group[0])[0]
+                    subgraph.add_node(previous_node)
+
+                    for i in range(1, len(group)):
+                        node = exp.get_node(group[i])[0]
+                        subgraph.add_node(node)
+                        # constraint false allows the horizontal alignment
+                        edge = pydotplus.Edge(previous_node, node, constraint='false', penwidth=4)
+                        edge.obj_dict['attributes']['dir'] = 'none'
+                        subgraph.add_edge(edge)
+                        previous_node = node
+                    graph.add_subgraph(subgraph)
+            graph.set_strict(True)
 
         graph.add_subgraph(exp)
 
@@ -150,7 +175,7 @@ class Monitor:
         Log.debug('Graph definition finalized')
         return graph
 
-    def _add_children(self, job, exp, node_job):
+    def _add_children(self, job, exp, node_job, groups):
         if job in self.nodes_ploted:
             return
         self.nodes_ploted.add(job)
@@ -158,18 +183,25 @@ class Monitor:
             for child in sorted(job.children, key=lambda k: k.name):
                 node_child = exp.get_node(child.name)
                 if len(node_child) == 0:
-                    node_child = pydotplus.Node(child.name, shape='box', style="filled",
-                                                fillcolor=self.color_status(child.status))
-                    exp.add_node(node_child)
                     flag = True
+                    if child.name not in groups or (child.name in groups and len(groups[child.name]) == 1):
+                        node_child = pydotplus.Node(child.name, shape='box', style="filled",
+                                                    fillcolor=self.color_status(child.status))
+                        if child.name in groups:
+                            group = groups[child.name][0]
+                            node_child.obj_dict['name'] = group
+                            node_child.obj_dict['attributes']['shape'] = 'box3d'
+
+                        exp.add_node(node_child)
+                        exp.add_edge(pydotplus.Edge(node_job, node_child))
                 else:
                     node_child = node_child[0]
+                    exp.add_edge(pydotplus.Edge(node_job, node_child))
                     flag = False
-                exp.add_edge(pydotplus.Edge(node_job, node_child))
                 if flag:
-                    self._add_children(child, exp, node_child)
+                    self._add_children(child, exp, node_child, groups)
 
-    def generate_output(self, expid, joblist, path, output_format="pdf", packages=None, show=False):
+    def generate_output(self, expid, joblist, path, output_format="pdf", packages=None, show=False, groups=dict()):
         """
         Plots graph for joblist and stores it in a file
 
@@ -188,7 +220,7 @@ class Monitor:
         output_file = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "plot", expid + "_" + output_date + "." +
                                    output_format)
 
-        graph = self.create_tree_list(expid, joblist, packages)
+        graph = self.create_tree_list(expid, joblist, packages, groups)
 
         Log.debug("Saving workflow plot at '{0}'", output_file)
         if output_format == "png":
@@ -225,8 +257,6 @@ class Monitor:
         if not os.path.exists(os.path.dirname(file_path)):
             os.makedirs(os.path.dirname(file_path))
 
-
-
         output_file = open(file_path, 'w+')
         for job in joblist:
             log_out = job.local_logs[0]
-- 
GitLab


From 2dad13b7c746fe5acca044d92e35d9c29272b415 Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Tue, 9 Jan 2018 18:21:00 +0100
Subject: [PATCH 5/9] Added status to groups

---
 autosubmit/autosubmit.py      | 51 +++++++++++++++++++++++++++++------
 autosubmit/job/job.py         |  3 ++-
 autosubmit/job/job_list.py    | 25 +++++++++--------
 autosubmit/monitor/monitor.py | 18 +++++++------
 4 files changed, 69 insertions(+), 28 deletions(-)

diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py
index 803e78eb8..9c9239705 100644
--- a/autosubmit/autosubmit.py
+++ b/autosubmit/autosubmit.py
@@ -174,7 +174,7 @@ class Autosubmit:
                                    help='Groups the jobs by date, member, chunk or split')
             group = subparser.add_mutually_exclusive_group(required=False)
             group.add_argument('-fl', '--list', type=str,
-                               help='Supply the list of job names to be changed. Default = "Any". '
+                               help='Supply the list of job names to be filtered. Default = "Any". '
                                     'LIST = "b037_20101101_fc3_21_sim b037_20111101_fc4_26_sim"')
             group.add_argument('-fc', '--filter_chunks', type=str,
                                help='Supply the list of chunks to filter the list of jobs. Default = "Any". '
@@ -1853,23 +1853,32 @@ class Autosubmit:
     @staticmethod
     def _group_jobs(group_by, jobs, job_list):
         groups_dict = dict()
+        jobs_group_dict = dict()
+        group_status_dict = dict()
         date_format = job_list._dic_jobs._date_format
 
         for date in job_list.get_date_list():
             if group_by == 'date':
-                Autosubmit._create_group(jobs, groups_dict, group_by, date_format, date)
+                Autosubmit._create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date)
             elif group_by in ['member', 'chunk', 'split']:
                 for member in job_list.get_member_list():
                     if group_by in ['chunk', 'split']:
                         for chunk in job_list.get_chunk_list():
-                            Autosubmit._create_group(jobs, groups_dict, group_by, date_format, date, member, chunk)
+                            Autosubmit._create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date, member, chunk)
                     else:
-                        Autosubmit._create_group(jobs, groups_dict, group_by, date_format, date, member)
+                        Autosubmit._create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date, member)
+
+        for group, statuses in group_status_dict.items():
+            status = Autosubmit._set_group_status(statuses)
+            group_status_dict[group] = status
+
+        groups_dict['jobs'] = jobs_group_dict
+        groups_dict['status'] = group_status_dict
 
         return groups_dict
 
     @staticmethod
-    def _create_group(jobs, groups_dict, group_by, date_format, date=None, member=None, chunk=None):
+    def _create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date=None, member=None, chunk=None):
         if chunk is not None:
             name = date2str(date, date_format) + '_' + member + '_' +str(chunk)
         elif member is not None:
@@ -1894,9 +1903,35 @@ class Autosubmit:
             if (condition):
                 if (job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None):
                     jobs.pop(i) #if synchronized does not remove
-                if job.name not in groups_dict:
-                    groups_dict[job.name] = list()
-                groups_dict[job.name].append(name)
+                if job.name not in jobs_group_dict:
+                    jobs_group_dict[job.name] = list()
+                jobs_group_dict[job.name].append(name)
+
+                if name not in group_status_dict:
+                    group_status_dict[name] = set()
+                group_status_dict[name].add(job.status)
+
+    @staticmethod
+    def _set_group_status(statuses):
+        if len(statuses) == 1:
+            return next(iter(statuses))
+        else:
+            if Status.FAILED in statuses:
+                return Status.FAILED
+            elif Status.RUNNING in statuses:
+                return Status.RUNNING
+            elif Status.SUBMITTED in statuses:
+                return Status.SUBMITTED
+            elif Status.QUEUING in statuses:
+                return Status.QUEUING
+            elif Status.READY in statuses:
+                return Status.READY
+            elif Status.WAITING in statuses:
+                return Status.WAITING
+            elif Status.SUSPENDED in statuses:
+                return Status.SUSPENDED
+            elif Status.UNKNOWN in statuses:
+                return Status.UNKNOWN
 
     @staticmethod
     def _copy_code(as_conf, expid, project_type, force):
diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py
index 782cfd5b5..31483197c 100644
--- a/autosubmit/job/job.py
+++ b/autosubmit/job/job.py
@@ -24,12 +24,13 @@ import os
 import re
 import time
 import json
+import datetime
 
 from autosubmit.job.job_common import Status, Type
 from autosubmit.job.job_common import StatisticsSnippetBash, StatisticsSnippetPython
 from autosubmit.job.job_common import StatisticsSnippetR, StatisticsSnippetEmpty
 from autosubmit.config.basicConfig import BasicConfig
-from bscearth.utils.date import *
+from bscearth.utils.date import date2str, parse_date, previous_day, chunk_end_date, chunk_start_date, Log, subs_dates
 
 
 class Job(object):
diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py
index d12beb907..9a549b3cd 100644
--- a/autosubmit/job/job_list.py
+++ b/autosubmit/job/job_list.py
@@ -173,25 +173,27 @@ class JobList:
     def _manage_dependencies(dependencies_keys, dic_jobs):
         dependencies = dict()
         for key in dependencies_keys:
+            distance = None
+            splits = None
+            sign = None
+
             if '-' not in key and '+' not in key:
-                dependency = Dependency(key)
                 section = key
             else:
                 sign = '-' if '-' in key else '+'
                 key_split = key.split(sign)
                 section = key_split[0]
-                splits = None
-                if '[' in section:
-                    section_name = section[0:section.find("[")]
-                    splits_section = int(dic_jobs.get_option(section_name, 'SPLITS', 0))
-                    splits = JobList._calculate_splits_dependencies(section, splits_section)
-                    section = section_name
-                distance = key_split[1]
-                dependency_running_type = dic_jobs.get_option(section, 'RUNNING', 'once').lower()
-                dependency = Dependency(section, int(distance), dependency_running_type, sign, splits=splits)
+                distance = int(key_split[1])
+
+            if '[' in section:
+                section_name = section[0:section.find("[")]
+                splits_section = int(dic_jobs.get_option(section_name, 'SPLITS', 0))
+                splits = JobList._calculate_splits_dependencies(section, splits_section)
+                section = section_name
 
+            dependency_running_type = dic_jobs.get_option(section, 'RUNNING', 'once').lower()
             delay = int(dic_jobs.get_option(section, 'DELAY', -1))
-            dependency.delay = delay
+            dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits)
             dependencies[key] = dependency
         return dependencies
 
@@ -426,6 +428,7 @@ class JobList:
         return filtered_jobs_fake_date_member, fake_original_job_map
 
     def _get_date(self, date):
+        date_format = ''
         if date.hour > 1:
             date_format = 'H'
         if date.minute > 1:
diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py
index 22cba4065..297ac52a1 100644
--- a/autosubmit/monitor/monitor.py
+++ b/autosubmit/monitor/monitor.py
@@ -127,18 +127,19 @@ class Monitor:
             if job.has_parents():
                 continue
 
-            if groups is None or job.name not in groups or (job.name in groups and len(groups[job.name]) == 1):
+            if not groups or job.name not in groups['jobs'] or (job.name in groups['jobs'] and len(groups['jobs'][job.name]) == 1):
                 node_job = pydotplus.Node(job.name, shape='box', style="filled",
                                           fillcolor=self.color_status(job.status))
-                if job.name in groups:
-                    group = groups[job.name]
-                    node_job.obj_dict['name'] = group[0]
+                if groups and job.name in groups['jobs']:
+                    group = groups['jobs'][job.name][0]
+                    node_job.obj_dict['name'] = group
+                    node_job.obj_dict['attributes']['fillcolor'] = self.color_status(groups['status'][group])
                     node_job.obj_dict['attributes']['shape'] = 'box3d'
                 exp.add_node(node_job)
                 self._add_children(job, exp, node_job, groups)
 
         if groups:
-            for job, group in groups.items():
+            for job, group in groups['jobs'].items():
                 if len(group) > 1:
                     subgraph = pydotplus.graphviz.Cluster(graph_name='cluster_'+'_'.join(group))
                     subgraph.obj_dict['attributes']['color'] = 'invis'
@@ -184,12 +185,13 @@ class Monitor:
                 node_child = exp.get_node(child.name)
                 if len(node_child) == 0:
                     flag = True
-                    if child.name not in groups or (child.name in groups and len(groups[child.name]) == 1):
+                    if not groups or child.name not in groups['jobs'] or (child.name in groups['jobs'] and len(groups['jobs'][child.name]) == 1):
                         node_child = pydotplus.Node(child.name, shape='box', style="filled",
                                                     fillcolor=self.color_status(child.status))
-                        if child.name in groups:
-                            group = groups[child.name][0]
+                        if groups and child.name in groups['jobs']:
+                            group = groups['jobs'][child.name][0]
                             node_child.obj_dict['name'] = group
+                            node_child.obj_dict['attributes']['fillcolor'] = self.color_status(groups['status'][group])
                             node_child.obj_dict['attributes']['shape'] = 'box3d'
 
                         exp.add_node(node_child)
-- 
GitLab


From f6e614c7439413269e5420f2e64d7d1df373dfa8 Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Mon, 15 Jan 2018 11:09:13 +0100
Subject: [PATCH 6/9] Separate class for job grouping

---
 autosubmit/autosubmit.py       |  96 ++----------------
 autosubmit/job/job_grouping.py | 171 +++++++++++++++++++++++++++++++++
 2 files changed, 180 insertions(+), 87 deletions(-)
 create mode 100644 autosubmit/job/job_grouping.py

diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py
index 9c9239705..a5080784b 100644
--- a/autosubmit/autosubmit.py
+++ b/autosubmit/autosubmit.py
@@ -41,7 +41,7 @@ import subprocess
 import json
 import tarfile
 import time
-import pickle
+import copy
 import os
 import sys
 import shutil
@@ -70,6 +70,7 @@ from job.job_packages import JobPackageThread
 from job.job_package_persistence import JobPackagePersistence
 from job.job_list_persistence import JobListPersistenceDb
 from job.job_list_persistence import JobListPersistencePkl
+from job.job_grouping import JobGrouping
 # noinspection PyPackageRequirements
 from bscearth.utils.log import Log
 from database.db_common import create_db
@@ -170,8 +171,8 @@ class Autosubmit:
             subparser.add_argument('expid', help='experiment identifier')
             subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf',
                                    help='chooses type of output for generated plot')
-            subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split'), default=None,
-                                   help='Groups the jobs by date, member, chunk or split')
+            subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+                                   help='Groups the jobs automatically or by date, member, chunk or split')
             group = subparser.add_mutually_exclusive_group(required=False)
             group.add_argument('-fl', '--list', type=str,
                                help='Supply the list of job names to be filtered. Default = "Any". '
@@ -873,7 +874,10 @@ class Autosubmit:
             packages = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"),
                                                      "job_packages_" + expid).load()
 
-        groups_dict = Autosubmit._group_jobs(group_by, jobs, job_list)
+        groups_dict = dict()
+        if group_by:
+            job_grouping = JobGrouping(group_by, copy.deepcopy(jobs), job_list, '')
+            groups_dict = job_grouping.group_jobs()
 
         monitor_exp = Monitor()
 
@@ -1838,6 +1842,7 @@ class Autosubmit:
 
                 JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"),
                                       "job_packages_" + expid).reset_table()
+
                 if not noplot:
                     Log.info("\nPlotting the jobs list...")
                     monitor_exp = Monitor()
@@ -1850,89 +1855,6 @@ class Autosubmit:
         except portalocker.AlreadyLocked:
             Autosubmit.show_lock_warning(expid)
 
-    @staticmethod
-    def _group_jobs(group_by, jobs, job_list):
-        groups_dict = dict()
-        jobs_group_dict = dict()
-        group_status_dict = dict()
-        date_format = job_list._dic_jobs._date_format
-
-        for date in job_list.get_date_list():
-            if group_by == 'date':
-                Autosubmit._create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date)
-            elif group_by in ['member', 'chunk', 'split']:
-                for member in job_list.get_member_list():
-                    if group_by in ['chunk', 'split']:
-                        for chunk in job_list.get_chunk_list():
-                            Autosubmit._create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date, member, chunk)
-                    else:
-                        Autosubmit._create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date, member)
-
-        for group, statuses in group_status_dict.items():
-            status = Autosubmit._set_group_status(statuses)
-            group_status_dict[group] = status
-
-        groups_dict['jobs'] = jobs_group_dict
-        groups_dict['status'] = group_status_dict
-
-        return groups_dict
-
-    @staticmethod
-    def _create_group(jobs, jobs_group_dict, group_by, date_format, group_status_dict, date=None, member=None, chunk=None):
-        if chunk is not None:
-            name = date2str(date, date_format) + '_' + member + '_' +str(chunk)
-        elif member is not None:
-            name = date2str(date, date_format) + '_' + member
-        else:
-            name = date2str(date, date_format)
-
-        for i in reversed(range(len(jobs))):
-            job = jobs[i]
-
-            if group_by == 'date':
-                condition = (job.date == date or (job.date is None and job.chunk is not None))
-            elif group_by == 'member':
-                condition = (job.date == date and job.member == member) or (job.date in [date, None] and job.member is None and job.chunk is not None)
-            elif group_by in ['chunk']:
-                condition = (job.member in [member, None] and job.date in [date, None] and job.chunk == chunk)
-            elif group_by == 'split':
-                condition = job.split is not None
-                idx = job.name.rfind("_")
-                name = job.name[:idx - 1] + job.name[idx + 1:]
-
-            if (condition):
-                if (job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None):
-                    jobs.pop(i) #if synchronized does not remove
-                if job.name not in jobs_group_dict:
-                    jobs_group_dict[job.name] = list()
-                jobs_group_dict[job.name].append(name)
-
-                if name not in group_status_dict:
-                    group_status_dict[name] = set()
-                group_status_dict[name].add(job.status)
-
-    @staticmethod
-    def _set_group_status(statuses):
-        if len(statuses) == 1:
-            return next(iter(statuses))
-        else:
-            if Status.FAILED in statuses:
-                return Status.FAILED
-            elif Status.RUNNING in statuses:
-                return Status.RUNNING
-            elif Status.SUBMITTED in statuses:
-                return Status.SUBMITTED
-            elif Status.QUEUING in statuses:
-                return Status.QUEUING
-            elif Status.READY in statuses:
-                return Status.READY
-            elif Status.WAITING in statuses:
-                return Status.WAITING
-            elif Status.SUSPENDED in statuses:
-                return Status.SUSPENDED
-            elif Status.UNKNOWN in statuses:
-                return Status.UNKNOWN
-
     @staticmethod
     def _copy_code(as_conf, expid, project_type, force):
         """
diff --git a/autosubmit/job/job_grouping.py b/autosubmit/job/job_grouping.py
new file mode 100644
index 000000000..4b9fc5910
--- /dev/null
+++ b/autosubmit/job/job_grouping.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+
+# Copyright 2017 Earth Sciences Department, BSC-CNS
+
+# This file is part of Autosubmit.
+
+# Autosubmit is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# Autosubmit is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Autosubmit.  If not, see <http://www.gnu.org/licenses/>.
+
+from autosubmit.job.job_common import Status
+from bscearth.utils.date import date2str
+import copy
+
+class JobGrouping(object):
+
+    def __init__(self, group_by, jobs, job_list, date_format, expand_list=list()):
+        self.group_by = group_by
+        self.automatic = False
+        self.jobs = jobs
+        self.job_list = job_list
+        self.date_format = date_format
+        self.expand_list = expand_list
+        if group_by == 'automatic':
+            self.group_by = 'chunk'
+            self.automatic = True
+
+    def group_jobs(self):
+        groups_dict = dict()
+
+        jobs_group_dict = dict()
+        self.group_status_dict = dict()
+        blacklist = list()
+        groups_map = dict()
+
+        for date in reversed(self.job_list.get_date_list()):
+            if self.group_by == 'date' or not self.job_list.get_member_list():
+                self.create_group(jobs_group_dict, date, blacklist=blacklist)
+            elif self.group_by in ['member', 'chunk', 'split']:
+                for member in reversed(self.job_list.get_member_list()):
+                    if self.group_by == 'member' or not self.job_list.get_chunk_list():
+                        self.create_group(jobs_group_dict, date, member, blacklist=blacklist)
+                    else:
+                        for chunk in reversed(self.job_list.get_chunk_list()):
+                            self.create_group(jobs_group_dict, date, member, chunk, blacklist=blacklist)
+
+        for group, statuses in self.group_status_dict.items():
+            status = self.set_group_status(statuses)
+            self.group_status_dict[group] = status
+
+        if self.automatic:
+            self.create_higher_level_group(self.group_status_dict.keys(), groups_map)
+
+        final_jobs_group = dict()
+        for job, groups in jobs_group_dict.items():
+            for group in groups:
+                if group not in blacklist and group in self.group_status_dict: #to remove the jobs belonging to groups that should be expanded
+                    while group in groups_map:
+                        group = groups_map[group]
+                    if job not in final_jobs_group:
+                        final_jobs_group[job] = list()
+                    final_jobs_group[job].append(group)
+
+        jobs_group_dict = final_jobs_group
+
+        groups_dict['jobs'] = jobs_group_dict
+        groups_dict['status'] = self.group_status_dict
+
+        return groups_dict
+
+    def create_group(self, jobs_group_dict, date=None, member=None, chunk=None, blacklist=list()):
+        if chunk is not None:
+            name = date2str(date, self.date_format) + '_' + member + '_' +str(chunk)
+        elif member is not None:
+            name = date2str(date, self.date_format) + '_' + member
+        else:
+            name = date2str(date, self.date_format)
+
+        for i in reversed(range(len(self.jobs))):
+            job = self.jobs[i]
+
+            if self.group_by == 'date':
+                condition = (job.date == date or (job.date is None and job.chunk is not None)) and name not in blacklist
+            elif self.group_by == 'member':
+                condition = (job.date == date and job.member == member) or (job.date in [date, None] and job.member is None and job.chunk is not None) and name not in blacklist
+            elif self.group_by  == 'chunk':
+                condition = (job.member in [member, None] and job.date in [date, None] and job.chunk == chunk) and name not in blacklist
+            elif self.group_by == 'split':
+                idx = job.name.rfind("_")
+                name = job.name[:idx - 1] + job.name[idx + 1:]
+                condition = job.split is not None and name not in blacklist
+
+            if (condition):
+                if (job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None):
+                    self.jobs.pop(i) #if synchronized does not remove
+                if name not in self.group_status_dict:
+                    self.group_status_dict[name] = set()
+                self.group_status_dict[name].add(job.status)
+
+                if job.status in self.expand_list or (self.automatic and name in self.group_status_dict and (len(self.group_status_dict[name]) > 1)):
+                    self.group_status_dict.pop(name)
+                    blacklist.append(name)
+                    break
+
+                if job.name not in jobs_group_dict:
+                    jobs_group_dict[job.name] = list()
+                jobs_group_dict[job.name].append(name)
+
+    def check_valid_group(self, groups_list, name, groups_map):
+        group_status = self.group_status_dict[groups_list[0]]
+        for group in groups_list[1:]:
+            status = self.group_status_dict[group]
+            if status != group_status:
+                return False
+
+        for group in groups_list:
+            self.group_status_dict.pop(group)
+            groups_map[group] = name
+        self.group_status_dict[name] = group_status
+        return True
+
+    def create_higher_level_group(self, groups_to_check, groups_map):
+        checked_groups = list()
+        for group in groups_to_check:
+            if group in self.group_status_dict:
+                split_count = len(group.split('_'))
+                if split_count > 0:
+                    new_group = group[:(group.rfind("_"))]
+
+                    num_groups = len(self.job_list.get_chunk_list()) if split_count == 3 else len(self.job_list.get_member_list())
+
+                    if new_group not in checked_groups:
+                        checked_groups.append(new_group)
+                        possible_groups = [existing_group for existing_group in list(self.group_status_dict.keys()) if
+                                              new_group in existing_group]
+
+                        if len(possible_groups) == num_groups:
+                            if self.check_valid_group(possible_groups, new_group, groups_map):
+                                groups_to_check.append(new_group)
+
+    def set_group_status(self, statuses):
+        if isinstance(statuses, int):
+            return statuses
+        if len(statuses) == 1:
+            return next(iter(statuses))
+        else:
+            if Status.FAILED in statuses:
+                return Status.FAILED
+            elif Status.RUNNING in statuses:
+                return Status.RUNNING
+            elif Status.SUBMITTED in statuses:
+                return Status.SUBMITTED
+            elif Status.QUEUING in statuses:
+                return Status.QUEUING
+            elif Status.READY in statuses:
+                return Status.READY
+            elif Status.WAITING in statuses:
+                return Status.WAITING
+            elif Status.SUSPENDED in statuses:
+                return Status.SUSPENDED
+            elif Status.UNKNOWN in statuses:
+                return Status.UNKNOWN
\ No newline at end of file
-- 
GitLab


From 8c66f72565e69e7e9fdaaf51df3f815ab0a71f4d Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Mon, 15 Jan 2018 19:07:28 +0100
Subject: [PATCH 7/9] Improvements to automatic grouping and code refactor

---
 autosubmit/job/job_grouping.py | 189 +++++++++++++++++++++------------
 1 file changed, 124 insertions(+), 65 deletions(-)

diff --git a/autosubmit/job/job_grouping.py b/autosubmit/job/job_grouping.py
index 4b9fc5910..ddd556723 100644
--- a/autosubmit/job/job_grouping.py
+++ b/autosubmit/job/job_grouping.py
@@ -19,65 +19,84 @@
 
 from autosubmit.job.job_common import Status
 from bscearth.utils.date import date2str
-import copy
 
 class JobGrouping(object):
 
     def __init__(self, group_by, jobs, job_list, date_format, expand_list=list()):
         self.group_by = group_by
-        self.automatic = False
         self.jobs = jobs
         self.job_list = job_list
         self.date_format = date_format
         self.expand_list = expand_list
-        if group_by == 'automatic':
-            self.group_by = 'chunk'
-            self.automatic = True
+        self.group_status_dict = dict()
 
     def group_jobs(self):
-        groups_dict = dict()
-
         jobs_group_dict = dict()
-        self.group_status_dict = dict()
         blacklist = list()
-        groups_map = dict()
-
-        for date in reversed(self.job_list.get_date_list()):
-            if self.group_by == 'date' or not self.job_list.get_member_list():
-                self.create_group(jobs_group_dict, date, blacklist=blacklist)
-            elif self.group_by in ['member', 'chunk', 'split']:
-                for member in reversed(self.job_list.get_member_list()):
-                    if self.group_by == 'member' or not self.job_list.get_chunk_list():
-                        self.create_group(jobs_group_dict, date, member, blacklist=blacklist)
-                    else:
-                        for chunk in reversed(self.job_list.get_chunk_list()):
-                            self.create_group(jobs_group_dict, date, member, chunk, blacklist=blacklist)
-
-        for group, statuses in self.group_status_dict.items():
-            status = self.set_group_status(statuses)
-            self.group_status_dict[group] = status
 
-        if self.automatic:
-            self.create_higher_level_group(self.group_status_dict.keys(), groups_map)
+        groups_map = dict()
+        if self.group_by == 'automatic':
+            self.automatic = True
+            jobs_group_dict = self._automatic_grouping(groups_map)
+        else:
+            self._group_jobs_by(jobs_group_dict, blacklist)
 
         final_jobs_group = dict()
         for job, groups in jobs_group_dict.items():
             for group in groups:
-                if group not in blacklist and group in self.group_status_dict: #to remove the jobs belonging to groups that should be expanded
+                if group not in blacklist:
                     while group in groups_map:
                         group = groups_map[group]
-                    if job not in final_jobs_group:
-                        final_jobs_group[job] = list()
-                    final_jobs_group[job].append(group)
+                    # to remove the jobs belonging to groups that should be expanded
+                    if group in self.group_status_dict:
+                        if job not in final_jobs_group:
+                            final_jobs_group[job] = list()
+                        final_jobs_group[job].append(group)
 
         jobs_group_dict = final_jobs_group
 
+        groups_dict = dict()
         groups_dict['jobs'] = jobs_group_dict
         groups_dict['status'] = self.group_status_dict
 
         return groups_dict
 
-    def create_group(self, jobs_group_dict, date=None, member=None, chunk=None, blacklist=list()):
+    def _set_group_status(self, statuses):
+        if isinstance(statuses, int):
+            return statuses
+        if len(statuses) == 1:
+            return next(iter(statuses))
+        else:
+            if Status.FAILED in statuses:
+                return Status.FAILED
+            elif Status.RUNNING in statuses:
+                return Status.RUNNING
+            elif Status.SUBMITTED in statuses:
+                return Status.SUBMITTED
+            elif Status.QUEUING in statuses:
+                return Status.QUEUING
+            elif Status.READY in statuses:
+                return Status.READY
+            elif Status.WAITING in statuses:
+                return Status.WAITING
+            elif Status.SUSPENDED in statuses:
+                return Status.SUSPENDED
+            elif Status.UNKNOWN in statuses:
+                return Status.UNKNOWN
+
+    def _group_jobs_by(self, jobs_group_dict, blacklist):
+        for date in reversed(self.job_list.get_date_list()):
+            if self.group_by == 'date' or not self.job_list.get_member_list():
+                self._create_group(jobs_group_dict, date, blacklist=blacklist)
+            elif self.group_by in ['member', 'chunk', 'split']:
+                for member in reversed(self.job_list.get_member_list()):
+                    if self.group_by == 'member' or not self.job_list.get_chunk_list():
+                        self._create_group(jobs_group_dict, date, member, blacklist=blacklist)
+                    else:
+                        for chunk in reversed(self.job_list.get_chunk_list()):
+                            self._create_group(jobs_group_dict, date, member, chunk, blacklist=blacklist)
+
+    def _create_group(self, jobs_group_dict, date=None, member=None, chunk=None, blacklist=list()):
         if chunk is not None:
             name = date2str(date, self.date_format) + '_' + member + '_' +str(chunk)
         elif member is not None:
@@ -89,24 +108,29 @@ class JobGrouping(object):
             job = self.jobs[i]
 
             if self.group_by == 'date':
-                condition = (job.date == date or (job.date is None and job.chunk is not None)) and name not in blacklist
+                condition = (job.date == date or (job.date is None and job.chunk is not None))
             elif self.group_by == 'member':
-                condition = (job.date == date and job.member == member) or (job.date in [date, None] and job.member is None and job.chunk is not None) and name not in blacklist
+                condition = (job.date == date and job.member == member) or \
+                            (job.date in [date, None] and job.member is None and job.chunk is not None)
             elif self.group_by  == 'chunk':
-                condition = (job.member in [member, None] and job.date in [date, None] and job.chunk == chunk) and name not in blacklist
+                condition = (job.member in [member, None] and job.date in [date, None] and job.chunk == chunk)
             elif self.group_by == 'split':
                 idx = job.name.rfind("_")
                 name = job.name[:idx - 1] + job.name[idx + 1:]
-                condition = job.split is not None and name not in blacklist
+                condition = job.split is not None
+
+            condition = condition and name not in blacklist
 
             if (condition):
-                if (job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None):
-                    self.jobs.pop(i) #if synchronized does not remove
+                #(not self.automatic and self.group_by == 'split') and \
+                if ((job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None)):
+                    self.jobs.pop(i) #if synchronized does not remove; neither if automatic and grouping the splits only
                 if name not in self.group_status_dict:
                     self.group_status_dict[name] = set()
                 self.group_status_dict[name].add(job.status)
 
-                if job.status in self.expand_list or (self.automatic and name in self.group_status_dict and (len(self.group_status_dict[name]) > 1)):
+                if job.status in self.expand_list or \
+                        (self.automatic and name in self.group_status_dict and (len(self.group_status_dict[name]) > 1)):
                     self.group_status_dict.pop(name)
                     blacklist.append(name)
                     break
@@ -115,7 +139,65 @@ class JobGrouping(object):
                     jobs_group_dict[job.name] = list()
                 jobs_group_dict[job.name].append(name)
 
-    def check_valid_group(self, groups_list, name, groups_map):
+    def _automatic_grouping(self, groups_map):
+        all_jobs = self.jobs
+        split_groups, split_groups_status = self._create_splits_groups()
+
+        blacklist = list()
+        jobs_group_dict = dict()
+        self.group_status_dict = dict()
+        self.group_by = 'chunk'
+        self.jobs = all_jobs
+
+        self._group_jobs_by(jobs_group_dict, blacklist)
+
+        for group, statuses in self.group_status_dict.items():
+            status = self._set_group_status(statuses)
+            self.group_status_dict[group] = status
+
+        self._create_higher_level_group(self.group_status_dict.keys(), groups_map)
+        self._fix_splits_automatic_grouping(split_groups, split_groups_status, jobs_group_dict)
+
+        # check if remaining jobs can be grouped
+        for i in reversed(range(len(self.jobs))):
+            job = self.jobs[i]
+            for group, status in self.group_status_dict.items():
+                if group in job.name and status == job.status:
+                    jobs_group_dict[job.name] = [group]
+                    self.jobs.pop(i)
+    
+        return jobs_group_dict
+
+    def _create_splits_groups(self):
+        jobs_group_dict = dict()
+
+        self.group_by = 'split'
+        self._group_jobs_by(jobs_group_dict, list())
+        return jobs_group_dict, self.group_status_dict
+
+    def _fix_splits_automatic_grouping(self, split_groups, split_groups_status, jobs_group_dict):
+        if split_groups and split_groups_status:
+            group_maps = dict()
+            for group in self.group_status_dict.keys():
+                matching_groups = [split_group for split_group in split_groups_status.keys() if group in split_group]
+                for matching_group in matching_groups:
+                    group_maps[matching_group] = group
+                    split_groups_status.pop(matching_group)
+
+            for split_group, statuses in split_groups_status.items():
+                status = self._set_group_status(statuses)
+                self.group_status_dict[split_group] = status
+
+            for job, groups in split_groups.items():
+                final_groups = list()
+                for group in groups:
+                    if group in group_maps:
+                        group = group_maps[group]
+                    final_groups.append(group)
+                if final_groups:
+                    jobs_group_dict[job] = final_groups
+
+    def _check_valid_group(self, groups_list, name, groups_map):
         group_status = self.group_status_dict[groups_list[0]]
         for group in groups_list[1:]:
             status = self.group_status_dict[group]
@@ -128,7 +210,7 @@ class JobGrouping(object):
         self.group_status_dict[name] = group_status
         return True
 
-    def create_higher_level_group(self, groups_to_check, groups_map):
+    def _create_higher_level_group(self, groups_to_check, groups_map):
         checked_groups = list()
         for group in groups_to_check:
             if group in self.group_status_dict:
@@ -144,28 +226,5 @@ class JobGrouping(object):
                                               new_group in existing_group]
 
                         if len(possible_groups) == num_groups:
-                            if self.check_valid_group(possible_groups, new_group, groups_map):
-                                groups_to_check.append(new_group)
-
-    def set_group_status(self, statuses):
-        if isinstance(statuses, int):
-            return statuses
-        if len(statuses) == 1:
-            return next(iter(statuses))
-        else:
-            if Status.FAILED in statuses:
-                return Status.FAILED
-            elif Status.RUNNING in statuses:
-                return Status.RUNNING
-            elif Status.SUBMITTED in statuses:
-                return Status.SUBMITTED
-            elif Status.QUEUING in statuses:
-                return Status.QUEUING
-            elif Status.READY in statuses:
-                return Status.READY
-            elif Status.WAITING in statuses:
-                return Status.WAITING
-            elif Status.SUSPENDED in statuses:
-                return Status.SUSPENDED
-            elif Status.UNKNOWN in statuses:
-                return Status.UNKNOWN
\ No newline at end of file
+                            if self._check_valid_group(possible_groups, new_group, groups_map):
+                                groups_to_check.append(new_group)
\ No newline at end of file
-- 
GitLab


From 0f93f0e919de7f929062a978a567d4290123b07d Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Thu, 18 Jan 2018 17:49:40 +0100
Subject: [PATCH 8/9] Added expand option

---
 autosubmit/autosubmit.py       | 17 +++++--
 autosubmit/job/job_grouping.py | 86 +++++++++++++++++++++++++++++++---
 autosubmit/job/job_list.py     |  9 ++++
 3 files changed, 103 insertions(+), 9 deletions(-)

diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py
index a5080784b..8bad1146f 100644
--- a/autosubmit/autosubmit.py
+++ b/autosubmit/autosubmit.py
@@ -173,6 +173,13 @@ class Autosubmit:
                                    help='chooses type of output for generated plot')
             subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
                                    help='Groups the jobs automatically or by date, member, chunk or split')
+            subparser.add_argument('-expand', type=str,
+                                    help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+                                    'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+            subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded')
+            group.add_argument('-fs', '--filter_status', type=str,
+                               choices=('Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
+                               help='Select the original status to filter the list of jobs')
             group = subparser.add_mutually_exclusive_group(required=False)
             group.add_argument('-fl', '--list', type=str,
                                help='Supply the list of job names to be filtered. Default = "Any". '
@@ -346,7 +353,7 @@ class Autosubmit:
                 return Autosubmit.delete(args.expid, args.force)
             elif args.command == 'monitor':
                 return Autosubmit.monitor(args.expid, args.output, args.list, args.filter_chunks, args.filter_status,
-                                          args.filter_type, args.hide, args.txt, args.group_by)
+                                          args.filter_type, args.hide, args.txt, args.group_by, args.expand, args.expand_status)
             elif args.command == 'stats':
                 return Autosubmit.statistics(args.expid, args.filter_type, args.filter_period, args.output, args.hide)
             elif args.command == 'clean':
@@ -763,7 +770,7 @@ class Autosubmit:
         return save
 
     @staticmethod
-    def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False, group_by=None):
+    def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False, group_by=None, expand=list(), expand_status=list()):
         """
         Plots workflow graph for a given experiment with status of each job coded by node color.
         Plot is created in experiment's plot folder with name <expid>_<date>_<time>.<file_format>
@@ -876,7 +883,11 @@ class Autosubmit:
 
         groups_dict = dict()
         if group_by:
-            job_grouping = JobGrouping(group_by, copy.deepcopy(jobs), job_list, '')
+            status = list()
+            if expand_status:
+                for s in expand_status.split():
+                    status.append(Autosubmit._get_status(s.upper()))
+            job_grouping = JobGrouping(group_by, copy.deepcopy(jobs), job_list, expand_list=expand, expanded_status=status)
             groups_dict = job_grouping.group_jobs()
 
         monitor_exp = Monitor()
diff --git a/autosubmit/job/job_grouping.py b/autosubmit/job/job_grouping.py
index ddd556723..fc6361f4a 100644
--- a/autosubmit/job/job_grouping.py
+++ b/autosubmit/job/job_grouping.py
@@ -22,15 +22,22 @@ from bscearth.utils.date import date2str
 
 class JobGrouping(object):
 
-    def __init__(self, group_by, jobs, job_list, date_format, expand_list=list()):
+    def __init__(self, group_by, jobs, job_list, expand_list=list(), expanded_status=list()):
         self.group_by = group_by
         self.jobs = jobs
         self.job_list = job_list
-        self.date_format = date_format
+        self.date_format = job_list.get_date_format()
         self.expand_list = expand_list
+        self.expand_status = expanded_status
+        self.automatic = False
         self.group_status_dict = dict()
+        self.ungrouped_jobs = list()
 
     def group_jobs(self):
+        if self.expand_list:
+            self._set_ungrouped_jobs()
+            self.jobs = list(set(self.jobs) - set(self.ungrouped_jobs))
+
         jobs_group_dict = dict()
         blacklist = list()
 
@@ -41,6 +48,10 @@ class JobGrouping(object):
         else:
             self._group_jobs_by(jobs_group_dict, blacklist)
 
+            for group, statuses in self.group_status_dict.items():
+                status = self._set_group_status(statuses)
+                self.group_status_dict[group] = status
+
         final_jobs_group = dict()
         for job, groups in jobs_group_dict.items():
             for group in groups:
@@ -61,6 +72,70 @@ class JobGrouping(object):
 
         return groups_dict
 
+    def _set_ungrouped_jobs(self):
+        text = self.expand_list
+
+        from pyparsing import nestedExpr
+        """
+        Function to parse rerun specification from json format
+
+        :param text: text to parse
+        :type text: list
+        :return: parsed output
+        """
+        count = 0
+
+        out = nestedExpr('[', ']').parseString(text).asList()
+
+        depth = lambda L: isinstance(L, list) and max(map(depth, L)) + 1
+
+        if self.group_by == 'date':
+            if depth(out) == 2:
+                dates = list()
+                for date in out[0]:
+                    dates.append(date)
+                self.ungrouped_jobs = [job for job in self.jobs if date2str(job.date, self.date_format) in dates]
+            else:
+                raise ValueError("Please check the syntax of the expand parameter including only dates")
+        elif self.group_by == 'member':
+            if depth(out) == 3:
+                for element in out[0]:
+                    if count % 2 == 0:
+                        date = out[0][count]
+                        members = out[0][count + 1]
+                        self.ungrouped_jobs = self.ungrouped_jobs + [job for job in self.jobs if date2str(job.date, self.date_format) == date and job.member in members]
+                        count += 1
+                    else:
+                        count += 1
+            else:
+                raise ValueError("Please check the syntax of the expand parameter including dates and the corresponding members")
+        elif self.group_by == 'chunk':
+            if depth(out) == 4:
+                for element in out[0]:
+                    if count % 2 == 0:
+                        date = out[0][count]
+                        member_chunks = out[0][count + 1]
+                        member_count = 0
+                        for element_member in member_chunks:
+                            if member_count % 2 == 0:
+                                member = member_chunks[member_count]
+                                chunks = list()
+                                for chunk in member_chunks[member_count + 1]:
+                                    if chunk.find("-") != -1:
+                                        numbers = chunk.split("-")
+                                        for count in range(int(numbers[0]), int(numbers[1]) + 1):
+                                            chunks.append(count)
+                                    else:
+                                        chunks.append(int(chunk))
+
+                                self.ungrouped_jobs = self.ungrouped_jobs + \
+                                                      [job for job in self.jobs if date2str(job.date, self.date_format) == date
+                                                       and job.member == member and job.chunk in chunks]
+                            member_count += 1
+                    count += 1
+            else:
+                raise ValueError("Please check the syntax of the expand parameter including dates and the corresponding members and chunks")
+
     def _set_group_status(self, statuses):
         if isinstance(statuses, int):
             return statuses
@@ -119,17 +194,16 @@ class JobGrouping(object):
                 name = job.name[:idx - 1] + job.name[idx + 1:]
                 condition = job.split is not None
 
-            condition = condition and name not in blacklist
+            condition = condition and name not in blacklist# and job not in self.ungrouped_jobs
 
             if (condition):
-                #(not self.automatic and self.group_by == 'split') and \
                 if ((job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None)):
                     self.jobs.pop(i) #if synchronized does not remove; neither if automatic and grouping the splits only
                 if name not in self.group_status_dict:
                     self.group_status_dict[name] = set()
                 self.group_status_dict[name].add(job.status)
 
-                if job.status in self.expand_list or \
+                if job.status in self.expand_status or \
                         (self.automatic and name in self.group_status_dict and (len(self.group_status_dict[name]) > 1)):
                     self.group_status_dict.pop(name)
                     blacklist.append(name)
@@ -215,7 +289,7 @@ class JobGrouping(object):
         for group in groups_to_check:
             if group in self.group_status_dict:
                 split_count = len(group.split('_'))
-                if split_count > 0:
+                if split_count > 1:
                     new_group = group[:(group.rfind("_"))]
 
                     num_groups = len(self.job_list.get_chunk_list()) if split_count == 3 else len(self.job_list.get_member_list())
diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py
index 9a549b3cd..4a034e377 100644
--- a/autosubmit/job/job_list.py
+++ b/autosubmit/job/job_list.py
@@ -476,6 +476,15 @@ class JobList:
         """
         return self._job_list
 
+    def get_date_format(self):
+        date_format = ''
+        for date in self.get_date_list():
+            if date.hour > 1:
+                date_format = 'H'
+            if date.minute > 1:
+                date_format = 'M'
+        return date_format
+
     def get_ordered_jobs_by_date_member(self):
         """
         Get the dictionary of jobs ordered according to wrapper's expression divided by date and member
-- 
GitLab


From 74ad40179bda1a03c21bf64992a1f664e58d73db Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Fri, 19 Jan 2018 18:13:58 +0100
Subject: [PATCH 9/9] Added grouping to create, setstatus, recovery; and unit
 tests

---
 autosubmit/autosubmit.py       |  73 +++-
 autosubmit/job/job_grouping.py |   2 +-
 autosubmit/monitor/monitor.py  |  21 +-
 test/unit/test_job_grouping.py | 678 +++++++++++++++++++++++++++++++++
 4 files changed, 754 insertions(+), 20 deletions(-)
 create mode 100644 test/unit/test_job_grouping.py

diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py
index 8bad1146f..e2d8f97b9 100644
--- a/autosubmit/autosubmit.py
+++ b/autosubmit/autosubmit.py
@@ -228,6 +228,12 @@ class Autosubmit:
             subparser.add_argument('-s', '--save', action="store_true", default=False, help='Save changes to disk')
             subparser.add_argument('--hide', action='store_true', default=False,
                                    help='hides plot window')
+            subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+                                   help='Groups the jobs automatically or by date, member, chunk or split')
+            subparser.add_argument('-expand', type=str,
+                                   help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+                                        'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+            subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded')
 
             # Migrate
             subparser = subparsers.add_parser('migrate', description="Migrate experiments from current user to another")
@@ -248,6 +254,12 @@ class Autosubmit:
                                    help='hides plot window')
             subparser.add_argument('-o', '--output', choices=('pdf', 'png', 'ps', 'svg'), default='pdf',
                                    help='chooses type of output for generated plot')
+            subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+                                   help='Groups the jobs automatically or by date, member, chunk or split')
+            subparser.add_argument('-expand', type=str,
+                                   help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+                                        'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+            subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded')
 
             # Configure
             subparser = subparsers.add_parser('configure', description="configure database and path for autosubmit. It "
@@ -297,6 +309,12 @@ class Autosubmit:
                                help='Select the job type to filter the list of jobs')
             subparser.add_argument('--hide', action='store_true', default=False,
                                    help='hides plot window')
+            subparser.add_argument('-group_by', choices=('date', 'member', 'chunk', 'split', 'automatic'), default=None,
+                                   help='Groups the jobs automatically or by date, member, chunk or split')
+            subparser.add_argument('-expand', type=str,
+                                   help='Supply the list of dates/members/chunks to filter the list of jobs. Default = "Any". '
+                                        'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
+            subparser.add_argument('-expand_status', type=str, help='Select the statuses to be expanded')
 
             # Test Case
             subparser = subparsers.add_parser('testcase', description='create test case experiment')
@@ -359,13 +377,13 @@ class Autosubmit:
             elif args.command == 'clean':
                 return Autosubmit.clean(args.expid, args.project, args.plot, args.stats)
             elif args.command == 'recovery':
-                return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide)
+                return Autosubmit.recovery(args.expid, args.noplot, args.save, args.all, args.hide, args.group_by, args.expand, args.expand_status)
             elif args.command == 'check':
                 return Autosubmit.check(args.expid)
             elif args.command == 'migrate':
                 return Autosubmit.migrate(args.expid, args.offer, args.pickup)
             elif args.command == 'create':
-                return Autosubmit.create(args.expid, args.noplot, args.hide, args.output)
+                return Autosubmit.create(args.expid, args.noplot, args.hide, args.output, args.group_by, args.expand, args.expand_status)
             elif args.command == 'configure':
                 if dialog is None or (args.databasepath is not None and args.localrootpath is not None):
                     return Autosubmit.configure(args.databasepath, args.databasefilename, args.localrootpath,
@@ -377,7 +395,8 @@ class Autosubmit:
                 return Autosubmit.install()
             elif args.command == 'setstatus':
                 return Autosubmit.set_status(args.expid, args.noplot, args.save, args.status_final, args.list,
-                                             args.filter_chunks, args.filter_status, args.filter_type, args.hide)
+                                             args.filter_chunks, args.filter_status, args.filter_type, args.hide,
+                                             args.group_by, args.expand, args.expand_status)
             elif args.command == 'testcase':
                 return Autosubmit.testcase(args.copy, args.description, args.chunks, args.member, args.stardate,
                                            args.HPC, args.branch)
@@ -887,6 +906,7 @@ class Autosubmit:
             if expand_status:
                 for s in expand_status.split():
                     status.append(Autosubmit._get_status(s.upper()))
+
             job_grouping = JobGrouping(group_by, copy.deepcopy(jobs), job_list, expand_list=expand, expanded_status=status)
             groups_dict = job_grouping.group_jobs()
 
@@ -1019,7 +1039,7 @@ class Autosubmit:
         return True
 
     @staticmethod
-    def recovery(expid, noplot, save, all_jobs, hide):
+    def recovery(expid, noplot, save, all_jobs, hide, group_by=None, expand=list(), expand_status=list()):
         """
         Method to check all active jobs. If COMPLETED file is found, job status will be changed to COMPLETED,
         otherwise it will be set to WAITING. It will also update the jobs list.
@@ -1110,10 +1130,21 @@ class Autosubmit:
         packages = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"),
                               "job_packages_" + expid).load()
 
+        groups_dict = dict()
+        if group_by:
+            status = list()
+            if expand_status:
+                for s in expand_status.split():
+                    status.append(Autosubmit._get_status(s.upper()))
+
+            job_grouping = JobGrouping(group_by, copy.deepcopy(job_list.get_job_list()), job_list, expand_list=expand,
+                                       expanded_status=status)
+            groups_dict = job_grouping.group_jobs()
+
         if not noplot:
             Log.info("\nPlotting the jobs list...")
             monitor_exp = Monitor()
-            monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), packages=packages, show=not hide)
+            monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), packages=packages, show=not hide, groups=groups_dict)
 
         return True
 
@@ -1757,7 +1788,7 @@ class Autosubmit:
                                     jobs_destiny)
 
     @staticmethod
-    def create(expid, noplot, hide, output='pdf'):
+    def create(expid, noplot, hide, output='pdf', group_by=None, expand=list(), expand_status=list()):
         """
         Creates job list for given experiment. Configuration files must be valid before realizaing this process.
 
@@ -1854,10 +1885,24 @@ class Autosubmit:
                 JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"),
                                       "job_packages_" + expid).reset_table()
 
+                groups_dict = dict()
+
                 if not noplot:
+                    if group_by:
+                        status = list()
+                        if expand_status:
+                            for s in expand_status.split():
+                                status.append(Autosubmit._get_status(s.upper()))
+
+                        job_grouping = JobGrouping(group_by, copy.deepcopy(job_list.get_job_list()), job_list, expand_list=expand,
+                                                   expanded_status=status)
+                        groups_dict = job_grouping.group_jobs()
+
+                        Log.info("\nFINISHED GROUPING...")
+
                     Log.info("\nPlotting the jobs list...")
                     monitor_exp = Monitor()
-                    monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), output, None, not hide)
+                    monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), output, None, not hide, groups=groups_dict)
 
                 Log.result("\nJob list created successfully")
                 Log.user_warning("Remember to MODIFY the MODEL config files!")
@@ -1949,7 +1994,7 @@ class Autosubmit:
         Log.info("CHANGED: job: " + job.name + " status to: " + final)
 
     @staticmethod
-    def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, hide):
+    def set_status(expid, noplot, save, final, lst, filter_chunks, filter_status, filter_section, hide, group_by=None, expand=list(), expand_status=list()):
         """
         Set status
 
@@ -2075,9 +2120,19 @@ class Autosubmit:
                                       "job_packages_" + expid).load()
 
                 if not noplot:
+                    groups_dict = dict()
+                    if group_by:
+                        status = list()
+                        if expand_status:
+                            for s in expand_status.split():
+                                status.append(Autosubmit._get_status(s.upper()))
+
+                        job_grouping = JobGrouping(group_by, copy.deepcopy(job_list.get_job_list()), job_list, expand_list=expand,
+                                                   expanded_status=status)
+                        groups_dict = job_grouping.group_jobs()
                     Log.info("\nPloting joblist...")
                     monitor_exp = Monitor()
-                    monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), packages=packages, show=not hide)
+                    monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), packages=packages, show=not hide, groups=groups_dict)
 
                 return True
 
diff --git a/autosubmit/job/job_grouping.py b/autosubmit/job/job_grouping.py
index fc6361f4a..84ea417aa 100644
--- a/autosubmit/job/job_grouping.py
+++ b/autosubmit/job/job_grouping.py
@@ -194,7 +194,7 @@ class JobGrouping(object):
                 name = job.name[:idx - 1] + job.name[idx + 1:]
                 condition = job.split is not None
 
-            condition = condition and name not in blacklist# and job not in self.ungrouped_jobs
+            condition = condition and name not in blacklist
 
             if (condition):
                 if ((job.chunk is None) or (job.member is not None and job.date is not None and job.chunk is not None)):
diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py
index 297ac52a1..ab794192b 100644
--- a/autosubmit/monitor/monitor.py
+++ b/autosubmit/monitor/monitor.py
@@ -121,6 +121,7 @@ class Monitor:
         if packages != None and packages:
             for (exp_id, package_name, job_name) in packages:
                 jobs_packages_dict[job_name] = package_name
+
         packages_subgraphs_dict = dict()
 
         for job in joblist:
@@ -182,20 +183,20 @@ class Monitor:
         self.nodes_ploted.add(job)
         if job.has_children() != 0:
             for child in sorted(job.children, key=lambda k: k.name):
-                node_child = exp.get_node(child.name)
+                if groups and child.name in groups['jobs']:
+                    node_child = exp.get_node(groups['jobs'][child.name][0])
+                else:
+                    node_child = exp.get_node(child.name)
                 if len(node_child) == 0:
                     flag = True
-                    if not groups or child.name not in groups['jobs'] or (child.name in groups['jobs'] and len(groups['jobs'][child.name]) == 1):
+                    if groups and child.name in groups['jobs'] and len(groups['jobs'][child.name]) == 1:
+                        group = groups['jobs'][child.name][0]
+                        node_child = pydotplus.Node(group, shape='box3d', style="filled", fillcolor=self.color_status(groups['status'][group]))
+                    elif not groups or child.name not in groups['jobs']:
                         node_child = pydotplus.Node(child.name, shape='box', style="filled",
                                                     fillcolor=self.color_status(child.status))
-                        if groups and child.name in groups['jobs']:
-                            group = groups['jobs'][child.name][0]
-                            node_child.obj_dict['name'] = group
-                            node_child.obj_dict['attributes']['fillcolor'] = self.color_status(groups['status'][group])
-                            node_child.obj_dict['attributes']['shape'] = 'box3d'
-
-                        exp.add_node(node_child)
-                        exp.add_edge(pydotplus.Edge(node_job, node_child))
+                    exp.add_node(node_child)
+                    exp.add_edge(pydotplus.Edge(node_job, node_child))
                 else:
                     node_child = node_child[0]
                     exp.add_edge(pydotplus.Edge(node_job, node_child))
diff --git a/test/unit/test_job_grouping.py b/test/unit/test_job_grouping.py
new file mode 100644
index 000000000..9cda66c63
--- /dev/null
+++ b/test/unit/test_job_grouping.py
@@ -0,0 +1,678 @@
+from unittest import TestCase
+from mock import Mock
+from autosubmit.job.job_list import JobList
+from bscearth.utils.config_parser import ConfigParserFactory
+from autosubmit.job.job_list_persistence import JobListPersistenceDb
+from autosubmit.job.job_common import Status
+from random import randrange
+from autosubmit.job.job import Job
+from mock import patch
+
+from autosubmit.job.job_grouping import JobGrouping
+
+class TestJobGrouping(TestCase):
+
+    def setUp(self):
+        self.experiment_id = 'random-id'
+        self.job_list = JobList(self.experiment_id, FakeBasicConfig, ConfigParserFactory(),
+                                JobListPersistenceDb('.', '.'))
+        self.parser_mock = Mock(spec='SafeConfigParser')
+
+        # Basic workflow with SETUP, INI, SIM, POST, CLEAN
+        self._createDummyJob('expid_SETUP', Status.READY)
+
+        for date in ['d1', 'd2']:
+            for member in ['m1', 'm2']:
+                job = self._createDummyJob('expid_' + date + '_' + member + '_' + 'INI', Status.WAITING, date, member)
+                self.job_list.get_job_list().append(job)
+
+        sections = ['SIM', 'POST', 'CLEAN']
+        for section in sections:
+            for date in ['d1', 'd2']:
+                for member in ['m1', 'm2']:
+                    for chunk in [1, 2]:
+                        job = self._createDummyJob('expid_' + date + '_' + member + '_' + str(chunk) + '_' + section, Status.WAITING, date, member, chunk)
+                        self.job_list.get_job_list().append(job)
+
+    @patch('autosubmit.job.job_grouping.date2str', side_effect=['d2', 'd1'])
+    def test_group_by_date(self, *patches):
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1' : Status.WAITING, 'd2' : Status.WAITING}
+        groups_dict['jobs'] = {
+                                'expid_d1_m1_INI' : ['d1'], 'expid_d1_m2_INI' : ['d1'], 'expid_d2_m1_INI' : ['d2'], 'expid_d2_m2_INI' : ['d2'],
+
+                               'expid_d1_m1_1_SIM': ['d1'], 'expid_d1_m1_2_SIM': ['d1'], 'expid_d1_m2_1_SIM': ['d1'], 'expid_d1_m2_2_SIM': ['d1'],
+                               'expid_d2_m1_1_SIM': ['d2'], 'expid_d2_m1_2_SIM': ['d2'], 'expid_d2_m2_1_SIM': ['d2'], 'expid_d2_m2_2_SIM': ['d2'],
+
+                               'expid_d1_m1_1_POST': ['d1'], 'expid_d1_m1_2_POST': ['d1'], 'expid_d1_m2_1_POST': ['d1'], 'expid_d1_m2_2_POST': ['d1'],
+                               'expid_d2_m1_1_POST': ['d2'], 'expid_d2_m1_2_POST': ['d2'], 'expid_d2_m2_1_POST': ['d2'], 'expid_d2_m2_2_POST': ['d2'],
+
+                               'expid_d1_m1_1_CLEAN': ['d1'], 'expid_d1_m1_2_CLEAN': ['d1'], 'expid_d1_m2_1_CLEAN': ['d1'],  'expid_d1_m2_2_CLEAN': ['d1'],
+                               'expid_d2_m1_1_CLEAN': ['d2'], 'expid_d2_m1_2_CLEAN': ['d2'], 'expid_d2_m2_1_CLEAN': ['d2'], 'expid_d2_m2_2_CLEAN': ['d2']
+                               }
+        
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('date', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str', side_effect=['d2', 'd2', 'd1', 'd1'])
+    def test_group_by_member(self, *patches):
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1': Status.WAITING, 'd1_m2': Status.WAITING, 'd2_m1': Status.WAITING, 'd2_m2' : Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_INI': ['d1_m1'], 'expid_d1_m2_INI': ['d1_m2'], 'expid_d2_m1_INI': ['d2_m1'], 'expid_d2_m2_INI': ['d2_m2'],
+
+            'expid_d1_m1_1_SIM': ['d1_m1'], 'expid_d1_m1_2_SIM': ['d1_m1'], 'expid_d1_m2_1_SIM': ['d1_m2'],
+            'expid_d1_m2_2_SIM': ['d1_m2'],
+            'expid_d2_m1_1_SIM': ['d2_m1'], 'expid_d2_m1_2_SIM': ['d2_m1'], 'expid_d2_m2_1_SIM': ['d2_m2'],
+            'expid_d2_m2_2_SIM': ['d2_m2'],
+
+            'expid_d1_m1_1_POST': ['d1_m1'], 'expid_d1_m1_2_POST': ['d1_m1'], 'expid_d1_m2_1_POST': ['d1_m2'],
+            'expid_d1_m2_2_POST': ['d1_m2'],
+            'expid_d2_m1_1_POST': ['d2_m1'], 'expid_d2_m1_2_POST': ['d2_m1'], 'expid_d2_m2_1_POST': ['d2_m2'],
+            'expid_d2_m2_2_POST': ['d2_m2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1'], 'expid_d1_m1_2_CLEAN': ['d1_m1'], 'expid_d1_m2_1_CLEAN': ['d1_m2'],
+            'expid_d1_m2_2_CLEAN': ['d1_m2'],
+            'expid_d2_m1_1_CLEAN': ['d2_m1'], 'expid_d2_m1_2_CLEAN': ['d2_m1'], 'expid_d2_m2_1_CLEAN': ['d2_m2'],
+            'expid_d2_m2_2_CLEAN': ['d2_m2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('member', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str', side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_group_by_chunk(self, *patches):
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1_1': Status.WAITING, 'd1_m1_2': Status.WAITING,
+                                 'd1_m2_1': Status.WAITING, 'd1_m2_2': Status.WAITING,
+                                 'd2_m1_1': Status.WAITING, 'd2_m1_2': Status.WAITING,
+                                 'd2_m2_1': Status.WAITING, 'd2_m2_2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_1_SIM': ['d1_m1_1'], 'expid_d1_m1_2_SIM': ['d1_m1_2'], 'expid_d1_m2_1_SIM': ['d1_m2_1'],
+            'expid_d1_m2_2_SIM': ['d1_m2_2'],
+            'expid_d2_m1_1_SIM': ['d2_m1_1'], 'expid_d2_m1_2_SIM': ['d2_m1_2'], 'expid_d2_m2_1_SIM': ['d2_m2_1'],
+            'expid_d2_m2_2_SIM': ['d2_m2_2'],
+
+            'expid_d1_m1_1_POST': ['d1_m1_1'], 'expid_d1_m1_2_POST': ['d1_m1_2'], 'expid_d1_m2_1_POST': ['d1_m2_1'],
+            'expid_d1_m2_2_POST': ['d1_m2_2'],
+            'expid_d2_m1_1_POST': ['d2_m1_1'], 'expid_d2_m1_2_POST': ['d2_m1_2'], 'expid_d2_m2_1_POST': ['d2_m2_1'],
+            'expid_d2_m2_2_POST': ['d2_m2_2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1_1'], 'expid_d1_m1_2_CLEAN': ['d1_m1_2'], 'expid_d1_m2_1_CLEAN': ['d1_m2_1'],
+            'expid_d1_m2_2_CLEAN': ['d1_m2_2'],
+            'expid_d2_m1_1_CLEAN': ['d2_m1_1'], 'expid_d2_m1_2_CLEAN': ['d2_m1_2'], 'expid_d2_m2_1_CLEAN': ['d2_m2_1'],
+            'expid_d2_m2_2_CLEAN': ['d2_m2_2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('chunk', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str', side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_group_by_split(self, *patches):
+        for date in ['d1', 'd2']:
+            for member in ['m1', 'm2']:
+                for chunk in [1, 2]:
+                    for split in [1, 2]:
+                        job = self._createDummyJob('expid_' + date + '_' + member + '_' + str(chunk) + '_' + str(split) + '_CMORATM',
+                                                   Status.WAITING, date, member, chunk, split)
+                        self.job_list.get_job_list().append(job)
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {
+            'expid_d1_m1_1_CMORATM': Status.WAITING,
+            'expid_d1_m1_2_CMORATM': Status.WAITING,
+            'expid_d1_m2_1_CMORATM': Status.WAITING,
+            'expid_d1_m2_2_CMORATM': Status.WAITING,
+            'expid_d2_m1_1_CMORATM': Status.WAITING,
+            'expid_d2_m1_2_CMORATM': Status.WAITING,
+            'expid_d2_m2_1_CMORATM': Status.WAITING,
+            'expid_d2_m2_2_CMORATM': Status.WAITING,
+        }
+        
+        groups_dict['jobs'] =  {
+            'expid_d1_m1_1_1_CMORATM' : ['expid_d1_m1_1_CMORATM'],
+            'expid_d1_m1_1_2_CMORATM' : ['expid_d1_m1_1_CMORATM'],
+            'expid_d1_m1_2_1_CMORATM' : ['expid_d1_m1_2_CMORATM'],
+            'expid_d1_m1_2_2_CMORATM' : ['expid_d1_m1_2_CMORATM'],
+            'expid_d1_m2_1_1_CMORATM': ['expid_d1_m2_1_CMORATM'],
+            'expid_d1_m2_1_2_CMORATM': ['expid_d1_m2_1_CMORATM'],
+            'expid_d1_m2_2_1_CMORATM': ['expid_d1_m2_2_CMORATM'],
+            'expid_d1_m2_2_2_CMORATM': ['expid_d1_m2_2_CMORATM'],
+            'expid_d2_m1_1_1_CMORATM': ['expid_d2_m1_1_CMORATM'],
+            'expid_d2_m1_1_2_CMORATM': ['expid_d2_m1_1_CMORATM'],
+            'expid_d2_m1_2_1_CMORATM': ['expid_d2_m1_2_CMORATM'],
+            'expid_d2_m1_2_2_CMORATM': ['expid_d2_m1_2_CMORATM'],
+            'expid_d2_m2_1_1_CMORATM': ['expid_d2_m2_1_CMORATM'],
+            'expid_d2_m2_1_2_CMORATM': ['expid_d2_m2_1_CMORATM'],
+            'expid_d2_m2_2_1_CMORATM': ['expid_d2_m2_2_CMORATM'],
+            'expid_d2_m2_2_2_CMORATM': ['expid_d2_m2_2_CMORATM']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('split', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_automatic_grouping_all(self, *patches):
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1': Status.WAITING, 'd2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_INI': ['d1'], 'expid_d1_m2_INI': ['d1'], 'expid_d2_m1_INI': ['d2'], 'expid_d2_m2_INI': ['d2'],
+
+            'expid_d1_m1_1_SIM': ['d1'], 'expid_d1_m1_2_SIM': ['d1'], 'expid_d1_m2_1_SIM': ['d1'],
+            'expid_d1_m2_2_SIM': ['d1'],
+            'expid_d2_m1_1_SIM': ['d2'], 'expid_d2_m1_2_SIM': ['d2'], 'expid_d2_m2_1_SIM': ['d2'],
+            'expid_d2_m2_2_SIM': ['d2'],
+
+            'expid_d1_m1_1_POST': ['d1'], 'expid_d1_m1_2_POST': ['d1'], 'expid_d1_m2_1_POST': ['d1'],
+            'expid_d1_m2_2_POST': ['d1'],
+            'expid_d2_m1_1_POST': ['d2'], 'expid_d2_m1_2_POST': ['d2'], 'expid_d2_m2_1_POST': ['d2'],
+            'expid_d2_m2_2_POST': ['d2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1'], 'expid_d1_m1_2_CLEAN': ['d1'], 'expid_d1_m2_1_CLEAN': ['d1'],
+            'expid_d1_m2_2_CLEAN': ['d1'],
+            'expid_d2_m1_1_CLEAN': ['d2'], 'expid_d2_m1_2_CLEAN': ['d2'], 'expid_d2_m2_1_CLEAN': ['d2'],
+            'expid_d2_m2_2_CLEAN': ['d2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('automatic', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_automatic_grouping_not_ini(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.READY
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.READY
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.READY
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.READY
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1': Status.WAITING, 'd2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_1_SIM': ['d1'], 'expid_d1_m1_2_SIM': ['d1'], 'expid_d1_m2_1_SIM': ['d1'],
+            'expid_d1_m2_2_SIM': ['d1'],
+            'expid_d2_m1_1_SIM': ['d2'], 'expid_d2_m1_2_SIM': ['d2'], 'expid_d2_m2_1_SIM': ['d2'],
+            'expid_d2_m2_2_SIM': ['d2'],
+
+            'expid_d1_m1_1_POST': ['d1'], 'expid_d1_m1_2_POST': ['d1'], 'expid_d1_m2_1_POST': ['d1'],
+            'expid_d1_m2_2_POST': ['d1'],
+            'expid_d2_m1_1_POST': ['d2'], 'expid_d2_m1_2_POST': ['d2'], 'expid_d2_m2_1_POST': ['d2'],
+            'expid_d2_m2_2_POST': ['d2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1'], 'expid_d1_m1_2_CLEAN': ['d1'], 'expid_d1_m2_1_CLEAN': ['d1'],
+            'expid_d1_m2_2_CLEAN': ['d1'],
+            'expid_d2_m1_1_CLEAN': ['d2'], 'expid_d2_m1_2_CLEAN': ['d2'], 'expid_d2_m2_1_CLEAN': ['d2'],
+            'expid_d2_m2_2_CLEAN': ['d2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('automatic', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_automatic_grouping_splits(self, *patches):
+        for date in ['d1', 'd2']:
+            for member in ['m1', 'm2']:
+                for chunk in [1, 2]:
+                    for split in [1, 2]:
+                        job = self._createDummyJob(
+                            'expid_' + date + '_' + member + '_' + str(chunk) + '_' + str(split) + '_CMORATM',
+                            Status.WAITING, date, member, chunk, split)
+                        self.job_list.get_job_list().append(job)
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1': Status.WAITING, 'd2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_INI': ['d1'], 'expid_d1_m2_INI': ['d1'], 'expid_d2_m1_INI': ['d2'], 'expid_d2_m2_INI': ['d2'],
+
+            'expid_d1_m1_1_SIM': ['d1'], 'expid_d1_m1_2_SIM': ['d1'], 'expid_d1_m2_1_SIM': ['d1'],
+            'expid_d1_m2_2_SIM': ['d1'],
+            'expid_d2_m1_1_SIM': ['d2'], 'expid_d2_m1_2_SIM': ['d2'], 'expid_d2_m2_1_SIM': ['d2'],
+            'expid_d2_m2_2_SIM': ['d2'],
+
+            'expid_d1_m1_1_POST': ['d1'], 'expid_d1_m1_2_POST': ['d1'], 'expid_d1_m2_1_POST': ['d1'],
+            'expid_d1_m2_2_POST': ['d1'],
+            'expid_d2_m1_1_POST': ['d2'], 'expid_d2_m1_2_POST': ['d2'], 'expid_d2_m2_1_POST': ['d2'],
+            'expid_d2_m2_2_POST': ['d2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1'], 'expid_d1_m1_2_CLEAN': ['d1'], 'expid_d1_m2_1_CLEAN': ['d1'],
+            'expid_d1_m2_2_CLEAN': ['d1'],
+            'expid_d2_m1_1_CLEAN': ['d2'], 'expid_d2_m1_2_CLEAN': ['d2'], 'expid_d2_m2_1_CLEAN': ['d2'],
+            'expid_d2_m2_2_CLEAN': ['d2'],
+
+            'expid_d1_m1_1_1_CMORATM': ['d1'],
+            'expid_d1_m1_1_2_CMORATM': ['d1'],
+            'expid_d1_m1_2_1_CMORATM': ['d1'],
+            'expid_d1_m1_2_2_CMORATM': ['d1'],
+            'expid_d1_m2_1_1_CMORATM': ['d1'],
+            'expid_d1_m2_1_2_CMORATM': ['d1'],
+            'expid_d1_m2_2_1_CMORATM': ['d1'],
+            'expid_d1_m2_2_2_CMORATM': ['d1'],
+            'expid_d2_m1_1_1_CMORATM': ['d2'],
+            'expid_d2_m1_1_2_CMORATM': ['d2'],
+            'expid_d2_m1_2_1_CMORATM': ['d2'],
+            'expid_d2_m1_2_2_CMORATM': ['d2'],
+            'expid_d2_m2_1_1_CMORATM': ['d2'],
+            'expid_d2_m2_1_2_CMORATM': ['d2'],
+            'expid_d2_m2_2_1_CMORATM': ['d2'],
+            'expid_d2_m2_2_2_CMORATM': ['d2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('automatic', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_automatic_grouping_different_status_member(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_2_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_2_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_CLEAN').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_2_CLEAN').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m2_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_CLEAN').status = Status.RUNNING
+
+        self.job_list.get_job_by_name('expid_d1_m2_2_SIM').status = Status.READY
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1': Status.COMPLETED, 'd2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_INI' : ['d1_m1'],
+
+            'expid_d1_m1_1_SIM': ['d1_m1'], 'expid_d1_m1_2_SIM': ['d1_m1'],
+            'expid_d2_m1_1_SIM': ['d2'], 'expid_d2_m1_2_SIM': ['d2'], 'expid_d2_m2_1_SIM': ['d2'],
+            'expid_d2_m2_2_SIM': ['d2'],
+
+            'expid_d1_m1_1_POST': ['d1_m1'], 'expid_d1_m1_2_POST': ['d1_m1'],
+            'expid_d2_m1_1_POST': ['d2'], 'expid_d2_m1_2_POST': ['d2'], 'expid_d2_m2_1_POST': ['d2'],
+            'expid_d2_m2_2_POST': ['d2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1'], 'expid_d1_m1_2_CLEAN': ['d1_m1'],
+            'expid_d2_m1_1_CLEAN': ['d2'], 'expid_d2_m1_2_CLEAN': ['d2'], 'expid_d2_m2_1_CLEAN': ['d2'],
+            'expid_d2_m2_2_CLEAN': ['d2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('automatic', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_automatic_grouping_different_status_chunk(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_CLEAN').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_2_SIM').status = Status.READY
+
+        self.job_list.get_job_by_name('expid_d1_m2_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_CLEAN').status = Status.RUNNING
+
+        self.job_list.get_job_by_name('expid_d1_m2_2_SIM').status = Status.READY
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1_1': Status.COMPLETED, 'd2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_1_SIM': ['d1_m1_1'],
+            'expid_d2_m1_1_SIM': ['d2'], 'expid_d2_m1_2_SIM': ['d2'], 'expid_d2_m2_1_SIM': ['d2'],
+            'expid_d2_m2_2_SIM': ['d2'],
+
+            'expid_d1_m1_1_POST': ['d1_m1_1'],
+            'expid_d2_m1_1_POST': ['d2'], 'expid_d2_m1_2_POST': ['d2'], 'expid_d2_m2_1_POST': ['d2'],
+            'expid_d2_m2_2_POST': ['d2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1_1'],
+            'expid_d2_m1_1_CLEAN': ['d2'], 'expid_d2_m1_2_CLEAN': ['d2'], 'expid_d2_m2_1_CLEAN': ['d2'],
+            'expid_d2_m2_2_CLEAN': ['d2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('automatic', self.job_list.get_job_list(), self.job_list)
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d2', 'd2', 'd1', 'd1'])
+    def test_group_by_member_expand_running(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_CLEAN').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_2_SIM').status = Status.READY
+
+        self.job_list.get_job_by_name('expid_d1_m2_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_CLEAN').status = Status.RUNNING
+
+        self.job_list.get_job_by_name('expid_d1_m2_2_SIM').status = Status.READY
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1': Status.READY, 'd2_m1': Status.WAITING,
+                                 'd2_m2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_INI': ['d1_m1'], 'expid_d2_m1_INI': ['d2_m1'],
+            'expid_d2_m2_INI': ['d2_m2'],
+
+            'expid_d1_m1_1_SIM': ['d1_m1'], 'expid_d1_m1_2_SIM': ['d1_m1'],
+            'expid_d2_m1_1_SIM': ['d2_m1'], 'expid_d2_m1_2_SIM': ['d2_m1'], 'expid_d2_m2_1_SIM': ['d2_m2'],
+            'expid_d2_m2_2_SIM': ['d2_m2'],
+
+            'expid_d1_m1_1_POST': ['d1_m1'], 'expid_d1_m1_2_POST': ['d1_m1'],
+            'expid_d2_m1_1_POST': ['d2_m1'], 'expid_d2_m1_2_POST': ['d2_m1'], 'expid_d2_m2_1_POST': ['d2_m2'],
+            'expid_d2_m2_2_POST': ['d2_m2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1'], 'expid_d1_m1_2_CLEAN': ['d1_m1'],
+            'expid_d2_m1_1_CLEAN': ['d2_m1'], 'expid_d2_m1_2_CLEAN': ['d2_m1'], 'expid_d2_m2_1_CLEAN': ['d2_m2'],
+            'expid_d2_m2_2_CLEAN': ['d2_m2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('member', self.job_list.get_job_list(), self.job_list, expanded_status=[Status.RUNNING])
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str', side_effect=['d2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_group_by_chunk_expand_failed_running(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_CLEAN').status = Status.FAILED
+
+        self.job_list.get_job_by_name('expid_d1_m1_2_SIM').status = Status.READY
+
+        self.job_list.get_job_by_name('expid_d1_m2_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_CLEAN').status = Status.RUNNING
+
+        self.job_list.get_job_by_name('expid_d1_m2_2_SIM').status = Status.READY
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1_2': Status.READY, 'd1_m2_2': Status.READY,
+                                 'd2_m1_1': Status.WAITING, 'd2_m1_2': Status.WAITING,
+                                 'd2_m2_1': Status.WAITING, 'd2_m2_2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_2_SIM': ['d1_m1_2'],
+            'expid_d1_m2_2_SIM': ['d1_m2_2'],
+            'expid_d2_m1_1_SIM': ['d2_m1_1'], 'expid_d2_m1_2_SIM': ['d2_m1_2'], 'expid_d2_m2_1_SIM': ['d2_m2_1'],
+            'expid_d2_m2_2_SIM': ['d2_m2_2'],
+
+            'expid_d1_m1_2_POST': ['d1_m1_2'],
+            'expid_d1_m2_2_POST': ['d1_m2_2'],
+            'expid_d2_m1_1_POST': ['d2_m1_1'], 'expid_d2_m1_2_POST': ['d2_m1_2'], 'expid_d2_m2_1_POST': ['d2_m2_1'],
+            'expid_d2_m2_2_POST': ['d2_m2_2'],
+
+            'expid_d1_m1_2_CLEAN': ['d1_m1_2'],
+            'expid_d1_m2_2_CLEAN': ['d1_m2_2'],
+            'expid_d2_m1_1_CLEAN': ['d2_m1_1'], 'expid_d2_m1_2_CLEAN': ['d2_m1_2'], 'expid_d2_m2_1_CLEAN': ['d2_m2_1'],
+            'expid_d2_m2_2_CLEAN': ['d2_m2_2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('chunk', self.job_list.get_job_list(), self.job_list, expanded_status=[Status.RUNNING, Status.FAILED])
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d1', 'd1', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+
+                        'd1', 'd1', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+
+                        'd2', 'd2', 'd1', 'd1'])
+    def test_group_by_member_expand(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_CLEAN').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_2_SIM').status = Status.READY
+
+        self.job_list.get_job_by_name('expid_d1_m2_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_CLEAN').status = Status.RUNNING
+
+        self.job_list.get_job_by_name('expid_d1_m2_2_SIM').status = Status.READY
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1': Status.READY}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_INI': ['d1_m1'],
+
+            'expid_d1_m1_1_SIM': ['d1_m1'], 'expid_d1_m1_2_SIM': ['d1_m1'],
+
+            'expid_d1_m1_1_POST': ['d1_m1'], 'expid_d1_m1_2_POST': ['d1_m1'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1'], 'expid_d1_m1_2_CLEAN': ['d1_m1'],
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('member', self.job_list.get_job_list(), self.job_list,
+                                   expand_list="[ d1 [m2] d2 [m1 m2] ]")
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d1', 'd1', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+
+                        'd1', 'd1', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+
+                        'd2', 'd2', 'd1', 'd1'])
+    def test_group_by_member_expand_running(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_CLEAN').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_2_SIM').status = Status.READY
+
+        self.job_list.get_job_by_name('expid_d1_m2_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_CLEAN').status = Status.RUNNING
+
+        self.job_list.get_job_by_name('expid_d1_m2_2_SIM').status = Status.READY
+
+        self.job_list.get_job_by_name('expid_d2_m1_1_SIM').status = Status.RUNNING
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1': Status.READY}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_INI': ['d1_m1'],
+
+            'expid_d1_m1_1_SIM': ['d1_m1'], 'expid_d1_m1_2_SIM': ['d1_m1'],
+
+            'expid_d1_m1_1_POST': ['d1_m1'], 'expid_d1_m1_2_POST': ['d1_m1'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1'], 'expid_d1_m1_2_CLEAN': ['d1_m1'],
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('member', self.job_list.get_job_list(), self.job_list,
+                                   expand_list="[ d1 [m2] d2 [m2] ]", expanded_status=[Status.RUNNING])
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    @patch('autosubmit.job.job_grouping.date2str',
+           side_effect=['d1', 'd1', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+
+                        'd1', 'd1', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+
+                        'd1', 'd1', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+                        'd1', 'd1', 'd1', 'd1', 'd2', 'd2', 'd2', 'd2',
+
+                        'd2', 'd2', 'd2', 'd2', 'd1', 'd1', 'd1', 'd1'])
+    def test_group_by_chunk_expand(self, *patches):
+        self.job_list.get_job_by_name('expid_d1_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m1_INI').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d2_m2_INI').status = Status.COMPLETED
+
+        self.job_list.get_job_by_name('expid_d1_m1_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m1_1_CLEAN').status = Status.FAILED
+
+        self.job_list.get_job_by_name('expid_d1_m1_2_SIM').status = Status.READY
+
+        self.job_list.get_job_by_name('expid_d1_m2_1_SIM').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_POST').status = Status.COMPLETED
+        self.job_list.get_job_by_name('expid_d1_m2_1_CLEAN').status = Status.RUNNING
+
+        groups_dict = dict()
+
+        groups_dict['status'] = {'d1_m1_1': Status.FAILED, 'd1_m1_2': Status.READY, 'd1_m2_1': Status.RUNNING, 'd2_m2_2': Status.WAITING}
+        groups_dict['jobs'] = {
+            'expid_d1_m1_1_SIM': ['d1_m1_1'], 'expid_d1_m1_2_SIM': ['d1_m1_2'], 'expid_d1_m2_1_SIM': ['d1_m2_1'],
+            'expid_d2_m2_2_SIM': ['d2_m2_2'],
+
+            'expid_d1_m1_1_POST': ['d1_m1_1'], 'expid_d1_m1_2_POST': ['d1_m1_2'], 'expid_d1_m2_1_POST': ['d1_m2_1'],
+            'expid_d2_m2_2_POST': ['d2_m2_2'],
+
+            'expid_d1_m1_1_CLEAN': ['d1_m1_1'], 'expid_d1_m1_2_CLEAN': ['d1_m1_2'], 'expid_d1_m2_1_CLEAN': ['d1_m2_1'],
+            'expid_d2_m2_2_CLEAN': ['d2_m2_2']
+        }
+
+        self.job_list.get_date_list = Mock(return_value=['d1', 'd2'])
+        self.job_list.get_member_list = Mock(return_value=['m1', 'm2'])
+        self.job_list.get_chunk_list = Mock(return_value=[1, 2])
+        self.job_list.get_date_format = Mock(return_value='')
+
+        job_grouping = JobGrouping('chunk', self.job_list.get_job_list(), self.job_list,
+                                   expand_list="[ d1 [m2 [2] ] d2 [m1 [1 2] m2 [1] ] ]")
+        self.assertDictEqual(job_grouping.group_jobs(), groups_dict)
+
+    def _createDummyJob(self, name, status, date=None, member=None, chunk=None, split=None):
+        job_id = randrange(1, 999)
+        job = Job(name, job_id, status, 0)
+        job.type = randrange(0, 2)
+
+        job.date = date
+        job.member = member
+        job.chunk = chunk
+        job.split = split
+
+        return job
+
+class FakeBasicConfig:
+    def __init__(self):
+        pass
+
+    DB_DIR = '/dummy/db/dir'
+    DB_FILE = '/dummy/db/file'
+    DB_PATH = '/dummy/db/path'
+    LOCAL_ROOT_DIR = '/dummy/local/root/dir'
+    LOCAL_TMP_DIR = '/dummy/local/temp/dir'
+    LOCAL_PROJ_DIR = '/dummy/local/proj/dir'
+    DEFAULT_PLATFORMS_CONF = ''
+    DEFAULT_JOBS_CONF = ''
+
+
+
-- 
GitLab