From 3dfc50ada6894e7b810054a74ebe27bfa2d67a59 Mon Sep 17 00:00:00 2001 From: Irene Simo Munoz Date: Tue, 29 Oct 2024 16:24:48 +0100 Subject: [PATCH] Add all expid_dir_path functions to autosubmit.py and fix tests !509 All previous definitions all expid_path, tmp_path, log_path and aslogs_path (and similar) have been removed and their calls have been switched for BasicConfig.foo() calls. Impacted tests are mainly affected in terms of adding Mock objects since BasicConfig.foo() functions are not available in autosubmitconfigparser yet. No new tests added --- autosubmit/autosubmit.py | 176 ++++++++-------------- autosubmit/notifications/mail_notifier.py | 55 ++++--- test/unit/conftest.py | 29 ++-- test/unit/test_catlog.py | 46 +++--- test/unit/test_describe.py | 13 +- test/unit/test_scheduler_general.py | 1 + 6 files changed, 155 insertions(+), 165 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 797929a5f..c9027d15b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -864,17 +864,14 @@ class Autosubmit: raise AutosubmitCritical( "Experiment {0} has no yml data. Please, if you really wish to use AS 4 prompt:\nautosubmit upgrade {0}".format( expid), 7012) - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) - aslogs_path = os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR) - if not os.path.exists(exp_path): + if not BasicConfig.expid_dir(expid).exists: raise AutosubmitCritical("Experiment does not exist", 7012) # delete is treated differently owner, eadmin, current_owner = Autosubmit._check_ownership_and_set_last_command(as_conf, expid, args.command) - if not os.path.exists(tmp_path): - os.mkdir(tmp_path) - if not os.path.exists(aslogs_path): - os.mkdir(aslogs_path) + if not BasicConfig.expid_tmp_dir(expid).exists(): + BasicConfig.expid_tmp_dir(expid).mkdir() + if not BasicConfig.expid_aslog_dir(expid).exists(): + BasicConfig.expid_aslog_dir(expid).mkdir() if args.command == "stop": exp_id = "_".join(expids) Log.set_file(os.path.join(BasicConfig.GLOBAL_LOG_DIR, @@ -883,25 +880,25 @@ class Autosubmit: args.command + exp_id + '_err.log'), "err") else: if owner: - os.chmod(tmp_path, 0o775) + os.chmod(BasicConfig.expid_tmp_dir(expid), 0o775) with suppress(PermissionError, FileNotFoundError, Exception): # for -txt option - os.chmod(f'{exp_path}/status', 0o775) + os.chmod(f'{BasicConfig.expid_dir(expid)}/status', 0o775) - Log.set_file(os.path.join(aslogs_path, args.command + '.log'), "out", log_level) - Log.set_file(os.path.join(aslogs_path, args.command + '_err.log'), "err") + Log.set_file(BasicConfig.expid_aslog_dir(expid).joinpath(args.command + '.log'), "out", log_level) + Log.set_file(BasicConfig.expid_aslog_dir(expid).joinpath(args.command + '_err.log'), "err") if args.command in ["run"]: - if os.path.exists(os.path.join(aslogs_path, 'jobs_active_status.log')): - os.remove(os.path.join(aslogs_path, 'jobs_active_status.log')) - if os.path.exists(os.path.join(aslogs_path, 'jobs_failed_status.log')): - os.remove(os.path.join(aslogs_path, 'jobs_failed_status.log')) - Log.set_file(os.path.join(aslogs_path, 'jobs_active_status.log'), "status") - Log.set_file(os.path.join(aslogs_path, 'jobs_failed_status.log'), "status_failed") + if os.path.exists(BasicConfig.expid_aslog_dir(expid).joinpath('jobs_active_status.log')): + os.remove(BasicConfig.expid_aslog_dir(expid).joinpath('jobs_active_status.log')) + if os.path.exists(BasicConfig.expid_aslog_dir(expid).joinpath('jobs_failed_status.log')): + os.remove(BasicConfig.expid_aslog_dir(expid).joinpath('jobs_failed_status.log')) + Log.set_file(BasicConfig.expid_aslog_dir(expid).joinpath('jobs_active_status.log'), "status") + Log.set_file(BasicConfig.expid_aslog_dir(expid).joinpath('jobs_failed_status.log'), "status_failed") else: - st = os.stat(tmp_path) + st = os.stat(BasicConfig.expid_tmp_dir(expid)) oct_perm = str(oct(st.st_mode))[-3:] if int(oct_perm[1]) in [6, 7] or int(oct_perm[2]) in [6, 7]: - Log.set_file(os.path.join(tmp_path, args.command + '.log'), "out", log_level) - Log.set_file(os.path.join(tmp_path, args.command + '_err.log'), "err") + Log.set_file(BasicConfig.expid_tmp_dir(expid).joinpath(args.command + '.log'), "out", log_level) + Log.set_file(BasicConfig.expid_tmp_dir(expid).joinpath(args.command + '_err.log'), "err") else: Log.set_file(os.path.join(BasicConfig.GLOBAL_LOG_DIR, args.command + expid + '.log'), "out", log_level) @@ -909,8 +906,8 @@ class Autosubmit: args.command + expid + '_err.log'), "err") Log.printlog( "Permissions of {0} are {1}. The log is being written in the {2} path instead of {1}. Please tell to the owner to fix the permissions".format( - tmp_path, oct_perm, BasicConfig.GLOBAL_LOG_DIR)) - Log.file_path = tmp_path + BasicConfig.expid_tmp_dir(expid), oct_perm, BasicConfig.GLOBAL_LOG_DIR)) + Log.file_path = BasicConfig.expid_tmp_dir(expid) if owner: if "update_version" in args: force_update_version = args.update_version @@ -938,8 +935,7 @@ class Autosubmit: else: exp_id = "_" + expid if args.command not in expid_less: - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - if not os.path.exists(exp_path): + if not BasicConfig.expid_dir(expid).exists(): raise AutosubmitCritical("Experiment does not exist", 7012) Log.set_file(os.path.join(BasicConfig.GLOBAL_LOG_DIR, args.command + exp_id + '.log'), "out", log_level) @@ -1459,9 +1455,7 @@ class Autosubmit: try: Log.info(f"Inspecting experiment {expid}") Autosubmit._check_ownership(expid, raise_error=True) - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) - if os.path.exists(os.path.join(tmp_path, 'autosubmit.lock')): + if BasicConfig.expid_tmp_dir(expid).joinpath('autosubmit.lock').exists(): locked = True else: locked = False @@ -2120,17 +2114,15 @@ class Autosubmit: profiler.start() # Initialize common folders - try: - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) - except BaseException as e: - raise AutosubmitCritical("Failure during the loading of the experiment configuration, check file paths", + if not BasicConfig.expid_tmp_dir(expid).exists(): + with BaseException as e: + raise AutosubmitCritical("Failure during the loading of the experiment configuration, check file paths", 7014, str(e)) # checking if there is a lock file to avoid multiple running on the same expid try: # Portalocker is used to avoid multiple autosubmit running on the same experiment, we have to change this system in #806 - with portalocker.Lock(os.path.join(tmp_path, 'autosubmit.lock'), timeout=1): + with portalocker.Lock(BasicConfig.expid_tmp_dir(expid).joinpath('autosubmit.lock'), timeout=1): try: Log.debug("Preparing run") # This function is called only once, when the experiment is started. It is used to initialize the experiment and to check the correctness of the configuration files. @@ -2598,7 +2590,6 @@ class Autosubmit: profiler.start() try: - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) Log.info("Getting job list...") as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) as_conf.check_conf_files(False) @@ -2730,8 +2721,7 @@ class Autosubmit: monitor_exp = Monitor() try: if txt_only or txt_logfiles or file_format == "txt": - monitor_exp.generate_output_txt(expid, jobs, os.path.join( - exp_path, "/tmp/LOG_" + expid), txt_logfiles, job_list_object=job_list) + monitor_exp.generate_output_txt(expid, jobs, BasicConfig.expid_log_dir(expid), txt_logfiles, job_list_object=job_list) if txt_only: current_length = len(job_list.get_job_list()) if current_length > 1000: @@ -2744,8 +2734,7 @@ class Autosubmit: # if file_format is set, use file_format, otherwise use conf value monitor_exp.generate_output(expid, jobs, - os.path.join( - exp_path, "/tmp/LOG_", expid), + BasicConfig.expid_log_dir(expid), output_format=file_format if file_format is not None and len( str(file_format)) > 0 else output_type, packages=packages, @@ -2855,8 +2844,6 @@ class Autosubmit: :param stats: set True to delete outdated stats """ try: - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - if project: autosubmit_config = AutosubmitConfig( expid, BasicConfig, YAMLParserFactory()) @@ -2913,7 +2900,6 @@ class Autosubmit: try: Autosubmit._check_ownership(expid, raise_error=True) - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) as_conf.check_conf_files(True) @@ -3040,8 +3026,7 @@ class Autosubmit: monitor_exp = Monitor() monitor_exp.generate_output(expid, job_list.get_job_list(), - os.path.join( - exp_path, "/tmp/LOG_", expid), + BasicConfig.expid_log_dir(expid), output_format=output_type, packages=packages, show=not hide, @@ -3116,7 +3101,6 @@ class Autosubmit: :type experiment_id: str """ try: - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) as_conf = AutosubmitConfig( experiment_id, BasicConfig, YAMLParserFactory()) @@ -3178,8 +3162,6 @@ class Autosubmit: try: ignore_performance_keys = ["error_message", "warnings_job_data", "considered"] - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) if folder_path is not None and len(str(folder_path)) > 0: tmp_path = folder_path import platform @@ -3340,7 +3322,6 @@ class Autosubmit: for experiment_id in experiments_ids: try: experiment_id = experiment_id.strip(" ") - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) as_conf = AutosubmitConfig( experiment_id, BasicConfig, YAMLParserFactory()) @@ -3375,9 +3356,8 @@ class Autosubmit: hpc = as_conf.get_platform() description = get_experiment_descrip(experiment_id) Log.result("Describing {0}", experiment_id) - Log.result("Owner: {0}", user) - Log.result("Location: {0}", exp_path) + Log.result("Location: {0}", BasicConfig.expid_dir(experiment_id)) Log.result("Created: {0}", created) Log.result("Model: {0}", model) Log.result("Branch: {0}", branch) @@ -4007,15 +3987,13 @@ class Autosubmit: :return: :rtype: """ - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) - pkl_folder_path = os.path.join(exp_path, "pkl") + pkl_folder_path = BasicConfig.expid_dir(expid).joinpath("pkl") current_pkl_path = os.path.join( pkl_folder_path, "job_list_{}.pkl".format(expid)) backup_pkl_path = os.path.join( pkl_folder_path, "job_list_{}_backup.pkl".format(expid)) try: - with portalocker.Lock(os.path.join(tmp_path, 'autosubmit.lock'), timeout=1): + with portalocker.Lock(BasicConfig.expid_tmp_dir(expid).joinpath('autosubmit.lock'), timeout=1): # Not locked Log.info("Looking for backup file {}".format(backup_pkl_path)) if os.path.exists(backup_pkl_path): @@ -4233,10 +4211,6 @@ class Autosubmit: :rtype: bool """ - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - - exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - if not noclean: # Cleaning to reduce file size. version = get_autosubmit_version(expid) @@ -4246,17 +4220,16 @@ class Autosubmit: # Getting year of last completed. If not, year of expid folder year = None - tmp_folder = os.path.join(exp_folder, BasicConfig.LOCAL_TMP_DIR) - if os.path.isdir(tmp_folder): - for filename in os.listdir(tmp_folder): + if BasicConfig.expid_tmp_dir(expid).is_dir(): + for filename in os.listdir(BasicConfig.expid_tmp_dir(expid)): if filename.endswith("COMPLETED"): file_year = time.localtime(os.path.getmtime( - os.path.join(tmp_folder, filename))).tm_year + BasicConfig.expid_tmp_dir(expid).joinpath(filename))).tm_year if year is None or year < file_year: year = file_year if year is None: - year = time.localtime(os.path.getmtime(exp_folder)).tm_year + year = time.localtime(os.path.getmtime(BasicConfig.expid_dir(expid))).tm_year try: year_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, str(year)) if not os.path.exists(year_path): @@ -4280,7 +4253,7 @@ class Autosubmit: compress_type = "w" output_filepath = '{0}.tar'.format(expid) with tarfile.open(os.path.join(year_path, output_filepath), compress_type) as tar: - tar.add(exp_folder, arcname='') + tar.add(BasicConfig.expid_dir(expid), arcname='') tar.close() os.chmod(os.path.join(year_path, output_filepath), 0o775) except Exception as e: @@ -4289,18 +4262,16 @@ class Autosubmit: Log.info("Tar file created!") try: - shutil.rmtree(exp_folder) + shutil.rmtree(BasicConfig.expid_dir(expid)) except Exception as e: Log.warning( "Can not fully remove experiments folder: {0}".format(str(e))) - if os.stat(exp_folder): + if os.stat(BasicConfig.expid_dir(expid)): try: - tmp_folder = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, "tmp") - tmp_expid = os.path.join(tmp_folder, expid + "_to_delete") - os.rename(exp_folder, tmp_expid) + tmp_expid = BasicConfig.expid_tmp_dir(expid).joinpath(expid + "_to_delete") + BasicConfig.expid_dir(expid).rename(tmp_expid) Log.warning("Experiment folder renamed to: {0}".format( - exp_folder + "_to_delete ")) + BasicConfig.expid_dir(expid) + "_to_delete ")) except Exception as e: Autosubmit.unarchive(expid, uncompressed=False, rocrate=rocrate) raise AutosubmitCritical( @@ -4321,7 +4292,6 @@ class Autosubmit: :param rocrate: flag to enable RO-Crate :type rocrate: bool """ - exp_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR, experiment_id) # Searching by year. We will store it on database year = datetime.datetime.today().year @@ -4349,19 +4319,19 @@ class Autosubmit: # Creating tar file Log.info("Unpacking tar file ... ") - if not os.path.isdir(exp_folder): - os.mkdir(exp_folder) + if not BasicConfig.expid_dir(experiment_id).is_dir(): + BasicConfig.expid_dir(experiment_id).mkdir() try: if rocrate: import zipfile with zipfile.ZipFile(archive_path, 'r') as zip: - zip.extractall(exp_folder) + zip.extractall(BasicConfig.expid_dir(experiment_id)) else: with tarfile.open(os.path.join(archive_path), compress_type) as tar: - tar.extractall(exp_folder) + tar.extractall(BasicConfig.expid_dir(experiment_id)) tar.close() except Exception as e: - shutil.rmtree(exp_folder, ignore_errors=True) + shutil.rmtree(BasicConfig.exid_dir(experiment_id), ignore_errors=True) Log.printlog("Can not extract file: {0}".format(str(e)), 6012) return False @@ -4443,10 +4413,8 @@ class Autosubmit: # checking if there is a lock file to avoid multiple running on the same expid try: Autosubmit._check_ownership(expid, raise_error=True) - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) # Encapsulating the lock - with portalocker.Lock(os.path.join(tmp_path, 'autosubmit.lock'), timeout=1) as fh: + with portalocker.Lock(BasicConfig.expid_tmp_dir(expid).joinpath('autosubmit.lock'), timeout=1) as fh: try: Log.info( "Preparing .lock file to avoid multiple instances with same expid.") @@ -4468,14 +4436,14 @@ class Autosubmit: raise AutosubmitCritical(f'Job list is empty\nCheck if there are YML files in {as_conf.experiment_data.get("DEFAULT","").get("CUSTOM_CONFIG","")}', code=7015) output_type = as_conf.get_output_type() - if not os.path.exists(os.path.join(exp_path, "pkl")): + if not BasicConfig.expid_dir(expid).joinpath("pkl").exists(): raise AutosubmitCritical( "The pkl folder doesn't exists. Make sure that the 'pkl' folder exists in the following path: {}".format( - exp_path), code=6013) - if not os.path.exists(os.path.join(exp_path, "plot")): + BasicConfig.expid_dir(expid)), code=6013) + if not BasicConfig.expid_dir(expid).joinpath("plot").exists(): raise AutosubmitCritical( "The plot folder doesn't exists. Make sure that the 'plot' folder exists in the following path: {}".format( - exp_path), code=6013) + BasicConfig.expid_dir(expid)), code=6013) update_job = not os.path.exists(os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid, "pkl", "job_list_" + expid + ".pkl")) @@ -4592,8 +4560,7 @@ class Autosubmit: monitor_exp = Monitor() # if output is set, use output monitor_exp.generate_output(expid, job_list.get_job_list(), - os.path.join( - exp_path, "/tmp/LOG_", expid), + BasicConfig.expid_log_dir(expid), output if output is not None else output_type, packages, not hide, @@ -5149,13 +5116,11 @@ class Autosubmit: :return: """ Autosubmit._check_ownership(expid, raise_error=True) - exp_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, expid) - tmp_path = os.path.join(exp_path, BasicConfig.LOCAL_TMP_DIR) section_validation_message = " " job_validation_message = " " # checking if there is a lock file to avoid multiple running on the same expid try: - with portalocker.Lock(os.path.join(tmp_path, 'autosubmit.lock'), timeout=1): + with portalocker.Lock(BasicConfig.expid_tmp_dir(expid).joinpath('autosubmit.lock'), timeout=1): Log.info( "Preparing .lock file to avoid multiple instances with same expid.") @@ -5386,8 +5351,7 @@ class Autosubmit: monitor_exp = Monitor() monitor_exp.generate_output(expid, job_list.get_job_list(), - os.path.join( - exp_path, "/tmp/LOG_", expid), + BasicConfig.expid_log_dir(expid), output_format=output_type, packages=packages, show=not hide, @@ -5903,22 +5867,6 @@ class Autosubmit: expid = exp_or_job_id if is_workflow else exp_or_job_id[:4] - # Workflow folder. - # e.g. ~/autosubmit/a000 - exp_path = Path(BasicConfig.LOCAL_ROOT_DIR, expid) - # Directory with workflow temporary/volatile files. Contains the output of commands such as inspect, - # and also STAT/COMPLETED files for each workflow task. - # e.g. ~/autosubmit/a000/tmp - tmp_path = exp_path / BasicConfig.LOCAL_TMP_DIR - # Directory with logs for Autosubmit executed commands (create, run, etc.) and jobs statuses files. - # e.g. ~/autosubmit/a000/tmp/ASLOGS - aslogs_path = tmp_path / BasicConfig.LOCAL_ASLOG_DIR - # Directory with the logs of the workflow run, for each workflow task. Includes the generated - # .cmd files, and STAT/COMPLETED files for the run. The files with similar names in the parent - # directory are generated with inspect, while these are with the run subcommand. - # e.g. ~/autosubmit/a000/tmp/LOG_a000 - exp_logs_path = tmp_path / f'LOG_{expid}' - if is_workflow: if file not in ['o', 'e', 's']: raise AutosubmitCritical(f'Invalid arguments for cat-log: workflow logs only support o(output), ' @@ -5926,30 +5874,30 @@ class Autosubmit: if file in ['e', 'o']: search_pattern = '*_run_err.log' if file == 'e' else '*_run.log' - workflow_log_files = sorted(aslogs_path.glob(search_pattern)) + workflow_log_files = sorted(BasicConfig.expid_aslog_dir(expid).glob(search_pattern)) else: search_pattern = f'{expid}_*.txt' - status_files_path = exp_path / 'status' - workflow_log_files = sorted(status_files_path.glob(search_pattern)) + status_files_path = os.path.join(BasicConfig.expid_dir(expid), 'status') + workflow_log_files = sorted(Path(status_files_path).glob(search_pattern)) if not workflow_log_files: Log.info('No logs found.') return True workflow_log_file = workflow_log_files[-1] - if not workflow_log_file.is_file(): + if not os.path.isfile(workflow_log_file): raise AutosubmitCritical(f'The workflow log file found is not a file: {workflow_log_file}', 7011) return view_file(workflow_log_file, mode) == 0 else: - job_logs_path = tmp_path if inspect else exp_logs_path + job_logs_path = BasicConfig.expid_tmp_dir(expid) if inspect else BasicConfig.expid_log_dir(expid) if file == 'j': workflow_log_file = job_logs_path / f'{exp_or_job_id}.cmd' elif file == 's': workflow_log_file = job_logs_path / f'{exp_or_job_id}_TOTAL_STATS' else: search_pattern = f'{exp_or_job_id}.*.{"err" if file == "e" else "out"}' - workflow_log_files = sorted(job_logs_path.glob(search_pattern)) + workflow_log_files = sorted(Path(job_logs_path).glob(search_pattern)) if not workflow_log_files: Log.info('No logs found.') return True @@ -5959,7 +5907,7 @@ class Autosubmit: Log.info('No logs found.') return True - if not workflow_log_file.is_file(): + if not os.path.isfile(workflow_log_file): raise AutosubmitCritical(f'The job log file {file} found is not a file: {workflow_log_file}', 7011) return view_file(workflow_log_file, mode) == 0 diff --git a/autosubmit/notifications/mail_notifier.py b/autosubmit/notifications/mail_notifier.py index 45c01c1c9..1562163e5 100644 --- a/autosubmit/notifications/mail_notifier.py +++ b/autosubmit/notifications/mail_notifier.py @@ -20,7 +20,11 @@ import smtplib import email.utils from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart +from email.mime.application import MIMEApplication from log.log import Log +from pathlib import Path +from autosubmitconfigparser.config.basicconfig import BasicConfig class MailNotifier: def __init__(self, basic_config): @@ -28,16 +32,25 @@ class MailNotifier: def notify_experiment_status(self, exp_id,mail_to,platform): message_text = self._generate_message_experiment_status(exp_id, platform) - message = MIMEText(message_text) + message = MIMEMultipart() message['From'] = email.utils.formataddr(('Autosubmit', self.config.MAIL_FROM)) - message['Subject'] = f'[Autosubmit] Warning a remote platform is malfunctioning' + message['Subject'] = f'[Autosubmit] Warning: a remote platform is malfunctioning' message['Date'] = email.utils.formatdate(localtime=True) + message.attach(MIMEText(message_text)) + + try: + files = [f for f in BasicConfig.expid_aslog_dir(exp_id).glob('*_run.log') if Path(f).is_file()] + mn._attach_files(message, files) + except BaseException as e: + Log.printlog('An error has occurred while sending a mail for warn about remote_platform', 6011) + for mail in mail_to: message['To'] = email.utils.formataddr((mail, mail)) try: self._send_mail(self.config.MAIL_FROM, mail, message) except BaseException as e: Log.printlog('An error has occurred while sending a mail for warn about remote_platform', 6011) + def notify_status_change(self, exp_id, job_name, prev_status, status, mail_to): message_text = self._generate_message_text(exp_id, job_name, prev_status, status) message = MIMEText(message_text) @@ -52,26 +65,34 @@ class MailNotifier: Log.printlog('Trace:{0}\nAn error has occurred while sending a mail for the job {0}'.format(e,job_name), 6011) def _send_mail(self, mail_from, mail_to, message): - server = smtplib.SMTP(self.config.SMTP_SERVER,timeout=60) + server = smtplib.SMTP(self.config.SMTP_SERVER, timeout=60) server.sendmail(mail_from, mail_to, message.as_string()) server.quit() + def _attach_files(self, message, files): + for f in files or []: + with open(f, "rb") as file: + part = MIMEApplication(file.read(), Name = Path(f).name) + part['Content-Disposition'] = 'attachment; filename="%s"' % Path(f).name + message.attach(part) + + @staticmethod def _generate_message_text(exp_id, job_name, prev_status, status): - return f'Autosubmit notification\n' \ - f'-------------------------\n\n' \ - f'Experiment id: {str(exp_id)} \n\n' \ - + f'Job name: {str(job_name)} \n\n' \ - f'The job status has changed from: {str(prev_status)} to {str(status)} \n\n\n\n\n' \ - f'INFO: This message was auto generated by Autosubmit, '\ - f'remember that you can disable these messages on Autosubmit config file. \n' + return f'''Autosubmit notification\n + -------------------------\n\n + Experiment id: {str(exp_id)} \n\n + Job name: {str(job_name)} \n\n + The job status has changed from: {str(prev_status)} to {str(status)} \n\n\n\n\n + INFO: This message was auto generated by Autosubmit, + remember that you can disable these messages on Autosubmit config file. \n''' @staticmethod def _generate_message_experiment_status(exp_id, platform=""): - return f'Autosubmit notification: Remote Platform issues\n' \ - f'-------------------------\n\n' \ - f'Experiment id:{str(exp_id)} \n\n' \ - + f'Platform affected:{str(platform.name)} using as host:{str(platform.host)} \n\n' \ - f'[WARN] Autosubmit encountered an issue with an remote_platform.\n It will resume itself, whenever is possible\n If issue persist, you can change the host IP or put multiple hosts in the platform.yml' + '\n\n\n\n\n' \ - f'INFO: This message was auto generated by Autosubmit, '\ - f'remember that you can disable these messages on Autosubmit config file. \n' \ No newline at end of file + return f'''Autosubmit notification: Remote Platform issues\n-------------------------\n + Experiment id: {str(exp_id)} + Logs and errors: {BasicConfig.expid_aslog_dir(exp_id)} + Attached to this message you will find the related _run.log files. + + Platform affected: {str(platform.name)} using as host: {str(platform.host)}\n\n[WARN] Autosubmit encountered an issue with a remote platform.\n It will resume itself, whenever is possible\n If this issue persists, you can change the host IP or put multiple hosts in the platform.yml file' + '\n\n\n\n\nINFO: This message was auto generated by Autosubmit, + remember that you can disable these messages on Autosubmit config file.\n''' diff --git a/test/unit/conftest.py b/test/unit/conftest.py index b83345b03..c981ddd20 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -8,6 +8,7 @@ from ruamel.yaml import YAML from shutil import rmtree from tempfile import TemporaryDirectory from typing import Any, Dict, Callable, List, Protocol, Optional +import os from autosubmit.autosubmit import Autosubmit from autosubmit.platforms.slurmplatform import SlurmPlatform, ParamikoPlatform @@ -27,7 +28,6 @@ class AutosubmitExperiment: status_dir: Path platform: ParamikoPlatform - @pytest.fixture(scope='function') def autosubmit_exp(autosubmit: Autosubmit, request: pytest.FixtureRequest) -> Callable: """Create an instance of ``Autosubmit`` with an experiment.""" @@ -36,17 +36,21 @@ def autosubmit_exp(autosubmit: Autosubmit, request: pytest.FixtureRequest) -> Ca tmp_dir = TemporaryDirectory() tmp_path = Path(tmp_dir.name) + def _create_autosubmit_exp(expid: str): - # directories used when searching for logs to cat root_dir = tmp_path BasicConfig.LOCAL_ROOT_DIR = str(root_dir) - exp_path = root_dir / expid - exp_tmp_dir = exp_path / BasicConfig.LOCAL_TMP_DIR - aslogs_dir = exp_tmp_dir / BasicConfig.LOCAL_ASLOG_DIR - status_dir = exp_path / 'status' - aslogs_dir.mkdir(parents=True, exist_ok=True) - status_dir.mkdir(parents=True, exist_ok=True) - + exp_path = BasicConfig.expid_dir(expid) + + # directories used when searching for logs to cat + exp_tmp_dir = BasicConfig.expid_tmp_dir(expid) + aslogs_dir = BasicConfig.expid_aslog_dir(expid) + status_dir =exp_path / 'status' + if not os.path.exists(aslogs_dir): + os.makedirs(aslogs_dir) + if not os.path.exists(status_dir): + os.makedirs(status_dir) + platform_config = { "LOCAL_ROOT_DIR": BasicConfig.LOCAL_ROOT_DIR, "LOCAL_TMP_DIR": str(exp_tmp_dir), @@ -59,7 +63,7 @@ def autosubmit_exp(autosubmit: Autosubmit, request: pytest.FixtureRequest) -> Ca 'QUEUING': [], 'FAILED': [] } - submit_platform_script = aslogs_dir / 'submit_local.sh' + submit_platform_script = aslogs_dir.joinpath('submit_local.sh') submit_platform_script.touch(exist_ok=True) return AutosubmitExperiment( @@ -94,7 +98,7 @@ def autosubmit() -> Autosubmit: @pytest.fixture(scope='function') def create_as_conf() -> Callable: # May need to be changed to use the autosubmit_config one def _create_as_conf(autosubmit_exp: AutosubmitExperiment, yaml_files: List[Path], experiment_data: Dict[str, Any]): - conf_dir = autosubmit_exp.exp_path / 'conf' + conf_dir = autosubmit_exp.exp_path.joinpath('conf') conf_dir.mkdir(parents=False, exist_ok=False) basic_config = BasicConfig parser_factory = YAMLParserFactory() @@ -117,7 +121,6 @@ def create_as_conf() -> Callable: # May need to be changed to use the autosubmi return _create_as_conf - class AutosubmitConfigFactory(Protocol): # Copied from the autosubmit config parser, that I believe is a revised one from the create_as_conf def __call__(self, expid: str, experiment_data: Optional[Dict], *args: Any, **kwargs: Any) -> AutosubmitConfig: ... @@ -177,4 +180,4 @@ def autosubmit_config( request.addfinalizer(finalizer) - return _create_autosubmit_config \ No newline at end of file + return _create_autosubmit_config diff --git a/test/unit/test_catlog.py b/test/unit/test_catlog.py index f7b2be15e..f9c68a6a6 100644 --- a/test/unit/test_catlog.py +++ b/test/unit/test_catlog.py @@ -6,25 +6,33 @@ from contextlib import suppress, redirect_stdout from pathlib import Path from tempfile import TemporaryDirectory from unittest.mock import patch +import pytest from autosubmit.autosubmit import Autosubmit, AutosubmitCritical from autosubmitconfigparser.config.basicconfig import BasicConfig - class TestJob(TestCase): def setUp(self): - self.autosubmit = Autosubmit() - # directories used when searching for logs to cat self.original_root_dir = BasicConfig.LOCAL_ROOT_DIR self.root_dir = TemporaryDirectory() BasicConfig.LOCAL_ROOT_DIR = self.root_dir.name - self.exp_path = Path(self.root_dir.name, 'a000') - self.tmp_dir = self.exp_path / BasicConfig.LOCAL_TMP_DIR - self.aslogs_dir = self.tmp_dir / BasicConfig.LOCAL_ASLOG_DIR - self.status_path = self.exp_path / 'status' - self.aslogs_dir.mkdir(parents=True) - self.status_path.mkdir() + + self.exp_path = BasicConfig.expid_dir('a000') + self.tmp_dir = BasicConfig.expid_tmp_dir('a000') + self.log_dir = BasicConfig.expid_log_dir('a000') + self.aslogs_dir = BasicConfig.expid_aslog_dir('a000') + + self.autosubmit = Autosubmit() + # directories used when searching for logs to cat + + self.status_path = self.exp_path / 'status' + if not self.aslogs_dir.exists(): + self.aslogs_dir.mkdir(parents = True, exist_ok = True) + if not self.status_path.exists(): + self.status_path.mkdir(parents = True, exist_ok = True) + if not self.log_dir.exists(): + self.log_dir.mkdir(parents=True, exist_ok=True) def tearDown(self) -> None: BasicConfig.LOCAL_ROOT_DIR = self.original_root_dir @@ -55,15 +63,17 @@ class TestJob(TestCase): assert Log.info.call_args[0][0] == 'No logs found.' def test_is_workflow_log_is_dir(self): - log_file_actually_dir = Path(self.aslogs_dir, 'log_run.log') - log_file_actually_dir.mkdir() + log_file_actually_dir = self.aslogs_dir / 'log_run.log' + log_file_actually_dir.mkdir(parents=True) def _fn(): self.autosubmit.cat_log('a000', 'o', 'c') self.assertRaises(AutosubmitCritical, _fn) @patch('subprocess.Popen') def test_is_workflow_out_cat(self, popen): - log_file = Path(self.aslogs_dir, 'log_run.log') + log_file = self.aslogs_dir / 'log_run.log' + if log_file.is_dir(): # dir is created in previous test + log_file.rmdir() with open(log_file, 'w') as f: f.write('as test') f.flush() @@ -75,7 +85,7 @@ class TestJob(TestCase): @patch('subprocess.Popen') def test_is_workflow_status_tail(self, popen): - log_file = Path(self.status_path, 'a000_anything.txt') + log_file = self.status_path / 'a000_anything.txt' with open(log_file, 'w') as f: f.write('as test') f.flush() @@ -93,9 +103,9 @@ class TestJob(TestCase): self.autosubmit.cat_log('a000_INI', file=file, mode='c') assert Log.info.called assert Log.info.call_args[0][0] == 'No logs found.' - + def test_is_jobs_log_is_dir(self): - log_file_actually_dir = Path(self.tmp_dir, 'LOG_a000/a000_INI.20000101.out') + log_file_actually_dir = self.log_dir / 'a000_INI.20000101.out' log_file_actually_dir.mkdir(parents=True) def _fn(): self.autosubmit.cat_log('a000_INI', 'o', 'c') @@ -103,9 +113,9 @@ class TestJob(TestCase): @patch('subprocess.Popen') def test_is_jobs_out_tail(self, popen): - log_dir = self.tmp_dir / 'LOG_a000' - log_dir.mkdir() - log_file = log_dir / 'a000_INI.20200101.out' + log_file = self.log_dir / 'a000_INI.20200101.out' + if log_file.is_dir(): # dir is created in previous test + log_file.rmdir() with open(log_file, 'w') as f: f.write('as test') f.flush() diff --git a/test/unit/test_describe.py b/test/unit/test_describe.py index 97f6946f2..f63077d47 100644 --- a/test/unit/test_describe.py +++ b/test/unit/test_describe.py @@ -33,14 +33,18 @@ def test_describe( if expid not in _EXPIDS: continue exp = autosubmit_exp(expid) - + basic_config = mocker.MagicMock() + # TODO: Whenever autosubmitconfigparser gets released with BasicConfig.expid_dir() and similar functions, the following line and a lot of mocks need to be removed. This line is especially delicate because it "overmocks" the basic_config mock, thus making the last assertion of this file "assert f'Location: {exp.exp_path}' in log_result_output" a dummy assertion. The rest of the test is still useful. + basic_config.expid_dir.side_effect = lambda expid: exp.exp_path config_values = { 'LOCAL_ROOT_DIR': str(exp.exp_path.parent), 'LOCAL_ASLOG_DIR': str(exp.aslogs_dir) } + for key, value in config_values.items(): basic_config.__setattr__(key, value) + basic_config.get.side_effect = lambda key, default='': config_values.get(key, default) for basic_config_location in [ 'autosubmit.autosubmit.BasicConfig', @@ -48,14 +52,16 @@ def test_describe( ]: # TODO: Better design how ``BasicConfig`` is used/injected/IOC/etc.. mocker.patch(basic_config_location, basic_config) + mocked_get_submitter = mocker.patch.object(Autosubmit, '_get_submitter') submitter = mocker.MagicMock() + mocked_get_submitter.return_value = submitter submitter.platforms = [1, 2] get_experiment_descrip = mocker.patch('autosubmit.autosubmit.get_experiment_descrip') get_experiment_descrip.return_value = [[f'{expid} description']] - + create_as_conf( autosubmit_exp=exp, yaml_files=[ @@ -69,6 +75,7 @@ def test_describe( } ) + Autosubmit.describe( input_experiment_list=input_experiment_list, get_from_user=get_from_user @@ -84,4 +91,4 @@ def test_describe( ] root_dir = exp.exp_path.parent for expid in expids: - assert f'Location: {str(root_dir / expid)}' in log_result_output + assert f'Location: {exp.exp_path}' in log_result_output diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py index 39215d6d8..d1d08f79c 100644 --- a/test/unit/test_scheduler_general.py +++ b/test/unit/test_scheduler_general.py @@ -201,6 +201,7 @@ def generate_cmds(prepare_scheduler): ('slurm', 'horizontal_vertical'), ('slurm', 'vertical_horizontal') ]) + def test_scheduler_job_types(scheduler, job_type, generate_cmds): # Test code that uses scheduler and job_typedef test_default_parameters(scheduler: str, job_type: str, generate_cmds): """ -- GitLab