diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index ca3a2a955df4f77e46220be993b79bfc6dabbffc..d9a31d76d2dd5e790a0785c3eae05c5cea058398 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -457,17 +457,21 @@ class Autosubmit: Log.info("Experiment directory does not exist.") else: Log.info("Removing experiment directory...") - try: - shutil.rmtree(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)) - except OSError as e: - Log.warning('Can not delete experiment folder: {0}', e) - return False - Log.info("Deleting experiment from database...") - ret = delete_experiment(expid_delete) - if ret: - Log.result("Experiment {0} deleted".format(expid_delete)) - return ret + ret = False + if pwd.getpwuid(os.stat(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)).st_uid).pw_name == os.getlogin(): + try: + shutil.rmtree(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid_delete)) + except OSError as e: + Log.warning('Can not delete experiment folder: {0}', e) + return ret + Log.info("Deleting experiment from database...") + ret = delete_experiment(expid_delete) + if ret: + Log.result("Experiment {0} deleted".format(expid_delete)) + else: + Log.warning("Current User is not the Owner {0} can not be deleted!",expid_delete) + return ret @staticmethod def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False): """ @@ -796,6 +800,11 @@ class Autosubmit: Log.info("No more jobs to run.") if len(job_list.get_failed()) > 0: + + if as_conf.get_notifications(): + Notifier.notify_status_change(exp_id,job_name,prev_status,status,as_conf.get_mails_to()) + + Log.info("Some jobs have failed and reached maximum retrials") return False else: @@ -2248,12 +2257,10 @@ class Autosubmit: Autosubmit.change_status(final, final_status, job) sys.setrecursionlimit(50000) - + job_list.update_list(as_conf,False) if save: - job_list.update_list(as_conf) job_list.save() else: - job_list.update_list(as_conf) Log.warning("Changes NOT saved to the JobList!!!!: use -s option to save") packages = JobPackagePersistence(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl"), diff --git a/autosubmit/config/config_common.py b/autosubmit/config/config_common.py index 884e9d6181354ba4325b071d187a96823ad9c481..f1aa6f83f9e47dfca9b3dd9ef0d20fd87d83358b 100644 --- a/autosubmit/config/config_common.py +++ b/autosubmit/config/config_common.py @@ -436,6 +436,7 @@ class AutosubmitConfig(object): if project_type == 'git': result = result and parser.check_exists('git', 'PROJECT_ORIGIN') result = result and parser.check_exists('git', 'PROJECT_BRANCH') + #result = result and parser.check_is_boolean('git', 'RECURSIVE') elif project_type == 'svn': result = result and parser.check_exists('svn', 'PROJECT_URL') result = result and parser.check_exists('svn', 'PROJECT_REVISION') @@ -610,7 +611,14 @@ class AutosubmitConfig(object): :rtype: str """ return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) - + #def get_git_recursive(self): + #""" + #Returns git recursive boolean from experiment's config file + + #:return: git commit + #:rtype: str + #""" + #return self._exp_parser.get_option('git', 'RECURSIVE', True) def get_project_destination(self): """ Returns git commit from experiment's config file diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 283644fc42df272c84e2716f387715e4e2d315c7..78bd26691fef35e4ee5a24c405558f8a2911fd68 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -95,6 +95,7 @@ class AutosubmitGit: git_project_origin = as_conf.get_git_project_origin() git_project_branch = as_conf.get_git_project_branch() git_project_commit = as_conf.get_git_project_commit() + project_destination = as_conf.get_project_destination() project_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, as_conf.expid, BasicConfig.LOCAL_PROJ_DIR) git_path = as_conf.get_project_dir() @@ -113,7 +114,7 @@ class AutosubmitGit: Log.info("Fetching {0} into {1}", git_project_commit + " " + git_project_origin, project_path) try: command = "cd {0}; git clone {1} {4}; cd {2}; git checkout {3}; " \ - "git submodule update --init --recursive".format(project_path, git_project_origin, git_path, + "git submodule update --init ".format(project_path, git_project_origin, git_path, git_project_commit, project_destination) output = subprocess.check_output(command, shell=True) except subprocess.CalledProcessError: @@ -124,8 +125,8 @@ class AutosubmitGit: else: Log.info("Cloning {0} into {1}", git_project_branch + " " + git_project_origin, project_path) try: - command = "cd {0}; git clone --recursive -b {1} {2} {3}".format(project_path, git_project_branch, - git_project_origin, project_destination) + command = "cd {0}; git clone -b {1} {2} {3}".format(project_path, git_project_branch, + git_project_origin, project_destination) output = subprocess.check_output(command, shell=True) Log.debug('{0}:{1}', command, output) except subprocess.CalledProcessError: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 8bce2b82d2a3d88b60df4b339a868f494ae04058..11d14a9f8fabacc1a08a09c53151e60cd4d22b73 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -777,7 +777,7 @@ class JobList: def parameters(self, value): self._parameters = value - def update_list(self, as_conf): + def update_list(self, as_conf,store_change=True): """ Updates job list, resetting failed jobs and changing to READY all WAITING jobs with all parents COMPLETED @@ -788,8 +788,8 @@ class JobList: """ # load updated file list save = False - if self.update_from_file(): - save = True + if self.update_from_file(store_change): + save = store_change # reset jobs that has failed less than 10 times Log.debug('Updating FAILED jobs') diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index 3f6b214f24f567da38fd349d34f8e17d421a69b7..f95e127234b2095a6841270accf3451b34744300 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -287,9 +287,9 @@ class Monitor: except subprocess.CalledProcessError: Log.error('File {0} could not be opened', output_file) - self.generate_output_txt(expid, joblist, path) + self.generate_output_txt(expid, joblist, path, "default") - def generate_output_txt(self, expid, joblist, path): + def generate_output_txt(self, expid, joblist, path,show="recursive"): Log.info('Writing status txt...') now = time.localtime() @@ -300,19 +300,34 @@ class Monitor: os.makedirs(os.path.dirname(file_path)) output_file = open(file_path, 'w+') - for job in joblist: - log_out = "" - log_err = "" - if job.status in [Status.FAILED, Status.COMPLETED]: - log_out = path + "/" + job.local_logs[0] - log_err = path + "/" + job.local_logs[1] - - output = job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + log_out + " " + log_err + "\n" - output_file.write(output) - output_file.close() - + if show != "recursive": + for job in joblist: + log_out = "" + log_err = "" + if job.status in [Status.FAILED, Status.COMPLETED]: + log_out = path + "/" + job.local_logs[0] + log_err = path + "/" + job.local_logs[1] + + output = job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + log_out + " " + log_err + "\n" + output_file.write(output) + else: + output_file.write("Writing jobs, they're grouped by [FC and DATE] \n") + self.write_output_txt_recursive(joblist[0],output_file,"") + output_file.close() Log.result('Status txt created at {0}', output_file) + def write_output_txt_recursive(self,job,output_file,level): + log_out = "" + log_err = "" + if job.status in [Status.FAILED, Status.COMPLETED]: + log_out = path + "/" + job.local_logs[0] + log_err = path + "/" + job.local_logs[1] + output = level+job.name + " " + Status().VALUE_TO_KEY[job.status] + " " + log_out + " " + log_err + "\n" + output_file.write(output) + if job.has_children() > 0: + for child in job.children: + self.write_output_txt_recursive(child,output_file,"_"+level) + def generate_output_stats(self, expid, joblist, output_format="pdf", period_ini=None, period_fi=None, show=False): """ Plots stats for joblist and stores it in a file