From 81e4e7103b0c8097d1d959e3a979f6451299ec44 Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Mon, 27 Nov 2017 14:51:06 +0100
Subject: [PATCH 1/2] Added option for only txt and paths to log files

---
 autosubmit/autosubmit.py      | 39 +++++++++++++++++++++--------------
 autosubmit/monitor/monitor.py | 11 ++++++----
 2 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py
index bf434b4ef..0bc5ca317 100644
--- a/autosubmit/autosubmit.py
+++ b/autosubmit/autosubmit.py
@@ -182,6 +182,8 @@ class Autosubmit:
                                help='Select the job type to filter the list of jobs')
             subparser.add_argument('--hide', action='store_true', default=False,
                                    help='hides plot window')
+            subparser.add_argument('--txt', action='store_true', default=False,
+                                   help='Generates only txt status file')
 
             # Stats
             subparser = subparsers.add_parser('stats', description="plots statistics for specified experiment")
@@ -277,10 +279,10 @@ class Autosubmit:
                                help='Supply the list of chunks to change the status. Default = "Any". '
                                     'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
             group.add_argument('-fs', '--filter_status', type=str,
-                               choices=('Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
-                               help='Select the original status to filter the list of jobs')
+                               help='Select the status (one or more) to filter the list of jobs.'
+                                    "Valid values = ['Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN']")
             group.add_argument('-ft', '--filter_type', type=str,
-                               help='Select the job type to filter the list of jobs')
+                               help='Select the job type(s) to filter from the list of jobs')
             subparser.add_argument('--hide', action='store_true', default=False,
                                    help='hides plot window')
 
@@ -339,7 +341,7 @@ class Autosubmit:
                 return Autosubmit.delete(args.expid, args.force)
             elif args.command == 'monitor':
                 return Autosubmit.monitor(args.expid, args.output, args.list, args.filter_chunks, args.filter_status,
-                                          args.filter_type, args.hide)
+                                          args.filter_type, args.hide, args.txt)
             elif args.command == 'stats':
                 return Autosubmit.statistics(args.expid, args.filter_type, args.filter_period, args.output, args.hide)
             elif args.command == 'clean':
@@ -738,7 +740,7 @@ class Autosubmit:
         return save
 
     @staticmethod
-    def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide):
+    def monitor(expid, file_format, lst, filter_chunks, filter_status, filter_section, hide, txt_only=False):
         """
         Plots workflow graph for a given experiment with status of each job coded by node color.
         Plot is created in experiment's plot folder with name <expid>_<date>_<time>.<file_format>
@@ -845,7 +847,11 @@ class Autosubmit:
             job.parents = job.parents - referenced_jobs_to_remove
 
         monitor_exp = Monitor()
-        monitor_exp.generate_output(expid, jobs, file_format, not hide)
+        if txt_only:
+            monitor_exp.generate_output_txt(expid, jobs, exp_path+"/tmp/LOG_"+expid)
+        else:
+            monitor_exp.generate_output(expid, jobs, file_format, not hide)
+
         return True
 
     @staticmethod
@@ -1970,26 +1976,29 @@ class Autosubmit:
                                         Autosubmit.change_status(final, final_status, job)
 
                 if filter_status:
+                    status_list = filter_status.split()
+
                     Log.debug("Filtering jobs with status {0}", filter_status)
-                    if filter_status == 'Any':
+                    if status_list == 'Any':
                         for job in job_list.get_job_list():
                             Autosubmit.change_status(final, final_status, job)
                     else:
-                        fs = Autosubmit._get_status(filter_status)
-                        for job in filter(lambda j: j.status == fs, job_list.get_job_list()):
-                            Autosubmit.change_status(final, final_status, job)
+                        for status in status_list:
+                            fs = Autosubmit._get_status(status)
+                            for job in filter(lambda j: j.status == fs, job_list.get_job_list()):
+                                Autosubmit.change_status(final, final_status, job)
 
                 if filter_section:
-                    ft = filter_section
-                    Log.debug(ft)
+                    ft = filter_section.split()
 
                     if ft == 'Any':
                         for job in job_list.get_job_list():
                             Autosubmit.change_status(final, final_status, job)
                     else:
-                        for job in job_list.get_job_list():
-                            if job.section == ft:
-                                Autosubmit.change_status(final, final_status, job)
+                        for section in ft:
+                            for job in job_list.get_job_list():
+                                if job.section == section:
+                                    Autosubmit.change_status(final, final_status, job)
 
                 if lst:
                     jobs = lst.split()
diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py
index d85b59803..c21734035 100644
--- a/autosubmit/monitor/monitor.py
+++ b/autosubmit/monitor/monitor.py
@@ -194,7 +194,7 @@ class Monitor:
 
         self.generate_output_txt(expid, joblist)
 
-    def generate_output_txt(self, expid, joblist):
+    def generate_output_txt(self, expid, joblist, path):
         Log.info('Writing status txt...')
 
         now = time.localtime()
@@ -204,11 +204,14 @@ class Monitor:
         if not os.path.exists(os.path.dirname(file_path)):
             os.makedirs(os.path.dirname(file_path))
 
+
+
         output_file = open(file_path, 'w+')
         for job in joblist:
-            output = job.name + " " + Status().VALUE_TO_KEY[job.status]
-            if job != joblist[-1]:
-                output += "\n"
+            log_out = job.local_logs[0]
+            log_err = job.local_logs[1]
+
+            output = job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + path + "/" + log_out + " " + path + "/" + log_err + "\n"
             output_file.write(output)
         output_file.close()
 
-- 
GitLab


From 5ff227134cafda3daebc5950a571a23bf4d3f289 Mon Sep 17 00:00:00 2001
From: Larissa Batista Leite <larissa.batista@bsc.es>
Date: Mon, 27 Nov 2017 15:36:44 +0100
Subject: [PATCH 2/2] Adding path to all generate_output calls

---
 autosubmit/autosubmit.py      | 35 ++++++++++++++++-------------------
 autosubmit/monitor/monitor.py |  4 ++--
 2 files changed, 18 insertions(+), 21 deletions(-)

diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py
index 0bc5ca317..119ad03aa 100644
--- a/autosubmit/autosubmit.py
+++ b/autosubmit/autosubmit.py
@@ -279,10 +279,10 @@ class Autosubmit:
                                help='Supply the list of chunks to change the status. Default = "Any". '
                                     'LIST = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 19651101 [ fc0 [16-30] ] ]"')
             group.add_argument('-fs', '--filter_status', type=str,
-                               help='Select the status (one or more) to filter the list of jobs.'
-                                    "Valid values = ['Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN']")
+                               choices=('Any', 'READY', 'COMPLETED', 'WAITING', 'SUSPENDED', 'FAILED', 'UNKNOWN'),
+                               help='Select the original status to filter the list of jobs')
             group.add_argument('-ft', '--filter_type', type=str,
-                               help='Select the job type(s) to filter from the list of jobs')
+                               help='Select the job type to filter the list of jobs')
             subparser.add_argument('--hide', action='store_true', default=False,
                                    help='hides plot window')
 
@@ -850,7 +850,7 @@ class Autosubmit:
         if txt_only:
             monitor_exp.generate_output_txt(expid, jobs, exp_path+"/tmp/LOG_"+expid)
         else:
-            monitor_exp.generate_output(expid, jobs, file_format, not hide)
+            monitor_exp.generate_output(expid, jobs, os.path.join(exp_path, "/tmp/LOG_", expid), file_format, not hide)
 
         return True
 
@@ -1065,7 +1065,7 @@ class Autosubmit:
         if not noplot:
             Log.info("\nPlotting the jobs list...")
             monitor_exp = Monitor()
-            monitor_exp.generate_output(expid, job_list.get_job_list(), show=not hide)
+            monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), show=not hide)
 
         return True
 
@@ -1803,7 +1803,7 @@ class Autosubmit:
                 if not noplot:
                     Log.info("\nPlotting the jobs list...")
                     monitor_exp = Monitor()
-                    monitor_exp.generate_output(expid, job_list.get_job_list(), output, not hide)
+                    monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), output, not hide)
 
                 Log.result("\nJob list created successfully")
                 Log.user_warning("Remember to MODIFY the MODEL config files!")
@@ -1976,29 +1976,26 @@ class Autosubmit:
                                         Autosubmit.change_status(final, final_status, job)
 
                 if filter_status:
-                    status_list = filter_status.split()
-
                     Log.debug("Filtering jobs with status {0}", filter_status)
-                    if status_list == 'Any':
+                    if filter_status == 'Any':
                         for job in job_list.get_job_list():
                             Autosubmit.change_status(final, final_status, job)
                     else:
-                        for status in status_list:
-                            fs = Autosubmit._get_status(status)
-                            for job in filter(lambda j: j.status == fs, job_list.get_job_list()):
-                                Autosubmit.change_status(final, final_status, job)
+                        fs = Autosubmit._get_status(filter_status)
+                        for job in filter(lambda j: j.status == fs, job_list.get_job_list()):
+                            Autosubmit.change_status(final, final_status, job)
 
                 if filter_section:
-                    ft = filter_section.split()
+                    ft = filter_section
+                    Log.debug(ft)
 
                     if ft == 'Any':
                         for job in job_list.get_job_list():
                             Autosubmit.change_status(final, final_status, job)
                     else:
-                        for section in ft:
-                            for job in job_list.get_job_list():
-                                if job.section == section:
-                                    Autosubmit.change_status(final, final_status, job)
+                        for job in job_list.get_job_list():
+                            if job.section == ft:
+                                Autosubmit.change_status(final, final_status, job)
 
                 if lst:
                     jobs = lst.split()
@@ -2023,7 +2020,7 @@ class Autosubmit:
                 if not noplot:
                     Log.info("\nPloting joblist...")
                     monitor_exp = Monitor()
-                    monitor_exp.generate_output(expid, job_list.get_job_list(), show=not hide)
+                    monitor_exp.generate_output(expid, job_list.get_job_list(), os.path.join(exp_path, "/tmp/LOG_", expid), show=not hide)
 
                 return True
 
diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py
index c21734035..048690c66 100644
--- a/autosubmit/monitor/monitor.py
+++ b/autosubmit/monitor/monitor.py
@@ -148,7 +148,7 @@ class Monitor:
                 if flag:
                     self._add_children(child, exp, node_child)
 
-    def generate_output(self, expid, joblist, output_format="pdf", show=False):
+    def generate_output(self, expid, joblist, path, output_format="pdf", show=False):
         """
         Plots graph for joblist and stores it in a file
 
@@ -192,7 +192,7 @@ class Monitor:
             except subprocess.CalledProcessError:
                 Log.error('File {0} could not be opened', output_file)
 
-        self.generate_output_txt(expid, joblist)
+        self.generate_output_txt(expid, joblist, path)
 
     def generate_output_txt(self, expid, joblist, path):
         Log.info('Writing status txt...')
-- 
GitLab