From 9fa4b7a78bf7f1960e89ae1b73a11b01b32f73f8 Mon Sep 17 00:00:00 2001 From: Daniel Beltran Mora Date: Mon, 3 Dec 2018 15:16:09 +0100 Subject: [PATCH 1/7] Git --recursive option should be optional now, need more testing , #297 --- autosubmit/config/config_common.py | 8 ++++++++ autosubmit/git/autosubmit_git.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 884e9d618..58d0a53d0 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -436,6 +436,7 @@ class AutosubmitConfig(object): if project_type == 'git': result = result and parser.check_exists('git', 'PROJECT_ORIGIN') result = result and parser.check_exists('git', 'PROJECT_BRANCH') + result = result and parser.check_is_boolean('git', 'RECURSIVE') elif project_type == 'svn': result = result and parser.check_exists('svn', 'PROJECT_URL') result = result and parser.check_exists('svn', 'PROJECT_REVISION') @@ -610,7 +611,14 @@ class AutosubmitConfig(object): :rtype: str """ return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) + def get_git_recursive(self): + """ + Returns git recursive boolean from experiment's config file + :return: git commit + :rtype: str + """ + return self._exp_parser.get_option('git', 'RECURSIVE', True) def get_project_destination(self): """ Returns git commit from experiment's config file diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 283644fc4..adcb8f460 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -95,6 +95,10 @@ class AutosubmitGit: git_project_origin = as_conf.get_git_project_origin() git_project_branch = as_conf.get_git_project_branch() git_project_commit = as_conf.get_git_project_commit() + if as_conf.get_git_project_recursive() : + git_project_recursive = "--recursive" + else: + git_project_recursive = "" project_destination = as_conf.get_project_destination() project_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, as_conf.expid, BasicConfig.LOCAL_PROJ_DIR) git_path = as_conf.get_project_dir() @@ -113,8 +117,8 @@ class AutosubmitGit: Log.info("Fetching {0} into {1}", git_project_commit + " " + git_project_origin, project_path) try: command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3}; " \ - "git submodule update --init --recursive".format(project_path, git_project_origin, git_path, - git_project_commit, project_destination) + "git submodule update --init {5}".format(project_path, git_project_origin, git_path, + git_project_commit, project_destination,git_project_recursive) output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError: Log.error("Can not checkout commit {0}: {1}", git_project_commit, output) @@ -124,8 +128,8 @@ class AutosubmitGit: else: Log.info("Cloning {0} into {1}", git_project_branch + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone --recursive -b {1} {2} {3}".format(project_path, git_project_branch, - git_project_origin, project_destination) + command = "cd {0}; git clone {4} -b {1} {2} {3}".format(project_path, git_project_branch, + git_project_origin, project_destination,git_project_recursive) output = subprocess.check_output(command, shell=True) Log.debug('{0}:{1}', command, output) except subprocess.CalledProcessError: -- GitLab From 913d0017631838d0256d55febc5e8269b64acbd1 Mon Sep 17 00:00:00 2001 From: Daniel Beltran Mora Date: Wed, 5 Dec 2018 16:35:56 +0100 Subject: [PATCH 2/7] mail notifier to a job failed , #303 --- autosubmit/autosubmit.py | 5 +++++ autosubmit/config/config_common.py | 18 +++++++++--------- autosubmit/git/autosubmit_git.py | 13 +++++-------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index ca3a2a955..d6b444ce6 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -796,6 +796,11 @@ class Autosubmit: Log.info("No more jobs to run.") if len(job_list.get_failed()) > 0: + #TODO: Send Mail Notification + if as_conf.get_notifications(): + Notifier.notify_status_change(exp_id,job_name,prev_status,status,as_conf.get_mails_to()) + + Log.info("Some jobs have failed and reached maximum retrials") return False else: diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 58d0a53d0..f1aa6f83f 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -436,7 +436,7 @@ class AutosubmitConfig(object): if project_type == 'git': result = result and parser.check_exists('git', 'PROJECT_ORIGIN') result = result and parser.check_exists('git', 'PROJECT_BRANCH') - result = result and parser.check_is_boolean('git', 'RECURSIVE') + #result = result and parser.check_is_boolean('git', 'RECURSIVE') elif project_type == 'svn': result = result and parser.check_exists('svn', 'PROJECT_URL') result = result and parser.check_exists('svn', 'PROJECT_REVISION') @@ -611,14 +611,14 @@ class AutosubmitConfig(object): :rtype: str """ return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) - def get_git_recursive(self): - """ - Returns git recursive boolean from experiment's config file - - :return: git commit - :rtype: str - """ - return self._exp_parser.get_option('git', 'RECURSIVE', True) + #def get_git_recursive(self): + #""" + #Returns git recursive boolean from experiment's config file + + #:return: git commit + #:rtype: str + #""" + #return self._exp_parser.get_option('git', 'RECURSIVE', True) def get_project_destination(self): """ Returns git commit from experiment's config file diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index adcb8f460..78bd26691 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -95,10 +95,7 @@ class AutosubmitGit: git_project_origin = as_conf.get_git_project_origin() git_project_branch = as_conf.get_git_project_branch() git_project_commit = as_conf.get_git_project_commit() - if as_conf.get_git_project_recursive() : - git_project_recursive = "--recursive" - else: - git_project_recursive = "" + project_destination = as_conf.get_project_destination() project_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, as_conf.expid, BasicConfig.LOCAL_PROJ_DIR) git_path = as_conf.get_project_dir() @@ -117,8 +114,8 @@ class AutosubmitGit: Log.info("Fetching {0} into {1}", git_project_commit + " " + git_project_origin, project_path) try: command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3}; " \ - "git submodule update --init {5}".format(project_path, git_project_origin, git_path, - git_project_commit, project_destination,git_project_recursive) + "git submodule update --init ".format(project_path, git_project_origin, git_path, + git_project_commit, project_destination) output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError: Log.error("Can not checkout commit {0}: {1}", git_project_commit, output) @@ -128,8 +125,8 @@ class AutosubmitGit: else: Log.info("Cloning {0} into {1}", git_project_branch + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone {4} -b {1} {2} {3}".format(project_path, git_project_branch, - git_project_origin, project_destination,git_project_recursive) + command = "cd {0}; git clone -b {1} {2} {3}".format(project_path, git_project_branch, + git_project_origin, project_destination) output = subprocess.check_output(command, shell=True) Log.debug('{0}:{1}', command, output) except subprocess.CalledProcessError: -- GitLab From ae12f5d497dd98fb8ec4666e762b65476538408f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 10 Dec 2018 16:31:13 +0100 Subject: [PATCH 3/7] Add an boolean for distinguish between setstatus and update on fly , #327 closed --- autosubmit/autosubmit.py | 6 ++---- autosubmit/job/job_list.py | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d6b444ce6..1a08e33df 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -796,7 +796,7 @@ class Autosubmit: Log.info("No more jobs to run.") if len(job_list.get_failed()) > 0: - #TODO: Send Mail Notification + if as_conf.get_notifications(): Notifier.notify_status_change(exp_id,job_name,prev_status,status,as_conf.get_mails_to()) @@ -2253,12 +2253,10 @@ class Autosubmit: Autosubmit.change_status(final, final_status, job) sys.setrecursionlimit(50000) - + job_list.update_list(as_conf,False) if save: - job_list.update_list(as_conf) job_list.save() else: - job_list.update_list(as_conf) Log.warning("Changes NOT saved to the JobList!!!!: use -s option to save") packages = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 8bce2b82d..11d14a9f8 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -777,7 +777,7 @@ class JobList: def parameters(self, value): self._parameters = value - def update_list(self, as_conf): + def update_list(self, as_conf,store_change=True): """ Updates job list, resetting failed jobs and changing to READY all WAITING jobs with all parents COMPLETED @@ -788,8 +788,8 @@ class JobList: """ # load updated file list save = False - if self.update_from_file(): - save = True + if self.update_from_file(store_change): + save = store_change # reset jobs that has failed less than 10 times Log.debug('Updating FAILED jobs') -- GitLab From 5435d60b4a9cc5f29ce7630c35c5ed044d24d646 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 11 Dec 2018 10:35:36 +0100 Subject: [PATCH 4/7] Need testing in a Virtual machine, actually it does not delete anything just simulate it , #360 --- autosubmit/autosubmit.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1a08e33df..dc4fa3377 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -457,17 +457,21 @@ class Autosubmit: Log.info("Experiment directory does not exist.") else: Log.info("Removing experiment directory...") - try: - shutil.rmtree(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)) - except OSError as e: - Log.warning('Can not delete experiment folder: {0}', e) - return False - Log.info("Deleting experiment from database...") - ret = delete_experiment(expid_delete) - if ret: - Log.result("Experiment {0} deleted".format(expid_delete)) - return ret - + ret = False + if pwd.getpwuid(os.stat(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)).st_uid).pw_name == os.getlogin(): + try: + Log.warning("Fake Deleted") + #shutil.rmtree(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)) + except OSError as e: + Log.warning('Can not delete experiment folder: {0}', e) + return ret + Log.info("Deleting experiment from database...") + #ret = delete_experiment(expid_delete) + # if ret: + # Log.result("Experiment {0} deleted".format(expid_delete)) + else: + Log.warning("Current User is not the Owner {0} can not be deleted!",expid_delete) + return ret @staticmethod def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False): """ -- GitLab From f44de5f32f2e0aff99c5c4e77d53d93c1f3c8795 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 11 Dec 2018 15:40:04 +0100 Subject: [PATCH 5/7] Tested, delete should work only if the user is the same, #360 still not merged in master or/and production --- autosubmit/autosubmit.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index dc4fa3377..d9a31d76d 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -460,15 +460,15 @@ class Autosubmit: ret = False if pwd.getpwuid(os.stat(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)).st_uid).pw_name == os.getlogin(): try: - Log.warning("Fake Deleted") - #shutil.rmtree(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)) + + shutil.rmtree(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)) except OSError as e: Log.warning('Can not delete experiment folder: {0}', e) return ret Log.info("Deleting experiment from database...") - #ret = delete_experiment(expid_delete) - # if ret: - # Log.result("Experiment {0} deleted".format(expid_delete)) + ret = delete_experiment(expid_delete) + if ret: + Log.result("Experiment {0} deleted".format(expid_delete)) else: Log.warning("Current User is not the Owner {0} can not be deleted!",expid_delete) return ret -- GitLab From f9e2e725a10c8f0697784041a443a554b4ff8936 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 12 Dec 2018 12:43:51 +0100 Subject: [PATCH 6/7] Improve visibility when using autosubmit monitor expid --txt, #245 #353 --- autosubmit/monitor/monitor.py | 41 ++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index 3f6b214f2..194e6d154 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -287,9 +287,9 @@ class Monitor: except subprocess.CalledProcessError: Log.error('File {0} could not be opened', output_file) - self.generate_output_txt(expid, joblist, path) + self.generate_output_txt(expid, joblist, path, "default") - def generate_output_txt(self, expid, joblist, path): + def generate_output_txt(self, expid, joblist, path,show="recursive"): Log.info('Writing status txt...') now = time.localtime() @@ -300,19 +300,34 @@ class Monitor: os.makedirs(os.path.dirname(file_path)) output_file = open(file_path, 'w+') - for job in joblist: - log_out = "" - log_err = "" - if job.status in [Status.FAILED, Status.COMPLETED]: - log_out = path + "/" + job.local_logs[0] - log_err = path + "/" + job.local_logs[1] - - output = job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + log_out + " " + log_err + "\n" - output_file.write(output) - output_file.close() - + if show != "recursive": + for job in joblist: + log_out = "" + log_err = "" + if job.status in [Status.FAILED, Status.COMPLETED]: + log_out = path + "/" + job.local_logs[0] + log_err = path + "/" + job.local_logs[1] + + output = job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + log_out + " " + log_err + "\n" + output_file.write(output) + else: + output_file.write("Writing jobs right to left prioritizing deep \n") + self.write_output_txt_recursive(joblist[0],output_file,"") + output_file.close() Log.result('Status txt created at {0}', output_file) + def write_output_txt_recursive(self,job,output_file,level): + log_out = "" + log_err = "" + if job.status in [Status.FAILED, Status.COMPLETED]: + log_out = path + "/" + job.local_logs[0] + log_err = path + "/" + job.local_logs[1] + output = level+job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + log_out + " " + log_err + "\n" + output_file.write(output) + if job.has_children() > 0: + for child in job.children: + self.write_output_txt_recursive(child,output_file,"_"+level) + def generate_output_stats(self, expid, joblist, output_format="pdf", period_ini=None, period_fi=None, show=False): """ Plots stats for joblist and stores it in a file -- GitLab From e6a496636add9f1d6474cbb3faf5403193a21a04 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 12 Dec 2018 13:01:48 +0100 Subject: [PATCH 7/7] Improved visibility when using autosubmit monitor expid --txt, #245 #353 --- autosubmit/monitor/monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index 194e6d154..f95e12723 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -311,7 +311,7 @@ class Monitor: output = job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + log_out + " " + log_err + "\n" output_file.write(output) else: - output_file.write("Writing jobs right to left prioritizing deep \n") + output_file.write("Writing jobs, they're grouped by [FC and DATE] \n") self.write_output_txt_recursive(joblist[0],output_file,"") output_file.close() Log.result('Status txt created at {0}', output_file) -- GitLab