From 2b33745ed261d99cb2d68ca0575f9b27a5c4ab40 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 10 Dec 2024 14:52:09 +0100 Subject: [PATCH 1/6] re-added notify_on --- autosubmit/job/job.py | 11 +++++++++++ autosubmit/notifications/mail_notifier.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 507f3fb75..4c5e7a4eb 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -568,6 +568,16 @@ class Job(object): def splits(self, value): self._splits = value + @property + @autosubmit_parameter(name='notify_on') + def notify_on(self): + """Send mail notification on job status change.""" + return self._notify_on + + @notify_on.setter + def notify_on(self, value): + self._notify_on = value + def __getstate__(self): return {k: v for k, v in self.__dict__.items() if k not in ["_platform", "_children", "_parents", "submitter"]} @@ -1904,6 +1914,7 @@ class Job(object): self.shape = as_conf.jobs_data[self.section].get("SHAPE", "") self.script = as_conf.jobs_data[self.section].get("SCRIPT", "") self.x11 = False if str(as_conf.jobs_data[self.section].get("X11", False)).lower() == "false" else True + self.notify_on = as_conf.jobs_data[self.section].get("NOTIFY_ON", "") if self.wrapper_type != "vertical" and self.packed: self.stat_file = f"{self.script_name[:-4]}_STAT_{self.fail_count}" else: diff --git a/autosubmit/notifications/mail_notifier.py b/autosubmit/notifications/mail_notifier.py index 45c01c1c9..632ec5282 100644 --- a/autosubmit/notifications/mail_notifier.py +++ b/autosubmit/notifications/mail_notifier.py @@ -44,7 +44,7 @@ class MailNotifier: message['From'] = email.utils.formataddr(('Autosubmit', self.config.MAIL_FROM)) message['Subject'] = f'[Autosubmit] The job {job_name} status has changed to {str(status)}' message['Date'] = email.utils.formatdate(localtime=True) - for mail in mail_to: + for mail in mail_to: # expects a list message['To'] = email.utils.formataddr((mail, mail)) try: self._send_mail(self.config.MAIL_FROM, mail, message) @@ -74,4 +74,4 @@ class MailNotifier: + f'Platform affected:{str(platform.name)} using as host:{str(platform.host)} \n\n' \ f'[WARN] Autosubmit encountered an issue with an remote_platform.\n It will resume itself, whenever is possible\n If issue persist, you can change the host IP or put multiple hosts in the platform.yml' + '\n\n\n\n\n' \ f'INFO: This message was auto generated by Autosubmit, '\ - f'remember that you can disable these messages on Autosubmit config file. \n' \ No newline at end of file + f'remember that you can disable these messages on Autosubmit config file. \n' -- GitLab From 38b7407e6663d7620009cb07d06756c1e5e575d2 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 10 Dec 2024 14:52:51 +0100 Subject: [PATCH 2/6] re-added notify_on --- autosubmit/notifications/mail_notifier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autosubmit/notifications/mail_notifier.py b/autosubmit/notifications/mail_notifier.py index 632ec5282..e6a5f2c43 100644 --- a/autosubmit/notifications/mail_notifier.py +++ b/autosubmit/notifications/mail_notifier.py @@ -38,13 +38,14 @@ class MailNotifier: self._send_mail(self.config.MAIL_FROM, mail, message) except BaseException as e: Log.printlog('An error has occurred while sending a mail for warn about remote_platform', 6011) + def notify_status_change(self, exp_id, job_name, prev_status, status, mail_to): message_text = self._generate_message_text(exp_id, job_name, prev_status, status) message = MIMEText(message_text) message['From'] = email.utils.formataddr(('Autosubmit', self.config.MAIL_FROM)) message['Subject'] = f'[Autosubmit] The job {job_name} status has changed to {str(status)}' message['Date'] = email.utils.formatdate(localtime=True) - for mail in mail_to: # expects a list + for mail in mail_to: # expects a list message['To'] = email.utils.formataddr((mail, mail)) try: self._send_mail(self.config.MAIL_FROM, mail, message) -- GitLab From 99c00dcd5fbb82e16a81c75b32e4eda5123fc68e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 10 Dec 2024 14:57:06 +0100 Subject: [PATCH 3/6] changed default value to [] --- autosubmit/job/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 4c5e7a4eb..51889d8e6 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1914,7 +1914,7 @@ class Job(object): self.shape = as_conf.jobs_data[self.section].get("SHAPE", "") self.script = as_conf.jobs_data[self.section].get("SCRIPT", "") self.x11 = False if str(as_conf.jobs_data[self.section].get("X11", False)).lower() == "false" else True - self.notify_on = as_conf.jobs_data[self.section].get("NOTIFY_ON", "") + self.notify_on = as_conf.jobs_data[self.section].get("NOTIFY_ON", []) if self.wrapper_type != "vertical" and self.packed: self.stat_file = f"{self.script_name[:-4]}_STAT_{self.fail_count}" else: -- GitLab From 5888664205e29ee724e1c0e4311f6ffd14976a13 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 11 Dec 2024 15:46:55 +0100 Subject: [PATCH 4/6] Added test to check attr value ( notify_on for now) --- setup.py | 2 +- test/unit/test_job_pytest.py | 58 ++++++++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 807ed719a..3cee16425 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ install_requires = [ 'py3dotplus==1.1.0', 'numpy<2', 'rocrate==0.*', - 'autosubmitconfigparser==1.0.75', + 'autosubmitconfigparser==1.0.76', 'configparser', 'setproctitle', 'invoke>=2.0', diff --git a/test/unit/test_job_pytest.py b/test/unit/test_job_pytest.py index 421429fbc..5efd58a0b 100644 --- a/test/unit/test_job_pytest.py +++ b/test/unit/test_job_pytest.py @@ -4,13 +4,29 @@ from autosubmit.job.job import Job from autosubmit.platforms.psplatform import PsPlatform +def create_job_and_update_parameters(autosubmit_config, experiment_data): + as_conf = autosubmit_config("test-expid", experiment_data) + as_conf.experiment_data = as_conf.deep_normalize(as_conf.experiment_data) + as_conf.experiment_data = as_conf.normalize_variables(as_conf.experiment_data, must_exists=True) + as_conf.experiment_data = as_conf.deep_read_loops(as_conf.experiment_data) + as_conf.experiment_data = as_conf.substitute_dynamic_variables(as_conf.experiment_data) + as_conf.experiment_data = as_conf.parse_data_loops(as_conf.experiment_data) + # Create some jobs + job = Job('A', '1', 0, 1) + platform = PsPlatform(expid='a000', name='DUMMY_PLATFORM', config=as_conf.experiment_data) + job.section = 'RANDOM-SECTION' + job.platform = platform + job.update_parameters(as_conf, {}) + return job + + @pytest.mark.parametrize('experiment_data, expected_data', [( { 'JOBS': { 'RANDOM-SECTION': { 'FILE': "test.sh", 'PLATFORM': 'DUMMY_PLATFORM', - 'TEST': "%other%" + 'TEST': "%other%", }, }, 'PLATFORMS': { @@ -36,17 +52,33 @@ from autosubmit.platforms.psplatform import PsPlatform } )]) def test_update_parameters_current_variables(autosubmit_config, experiment_data, expected_data): - as_conf = autosubmit_config("test-expid", experiment_data) - as_conf.experiment_data = as_conf.deep_normalize(as_conf.experiment_data) - as_conf.experiment_data = as_conf.normalize_variables(as_conf.experiment_data, must_exists=True) - as_conf.experiment_data = as_conf.deep_read_loops(as_conf.experiment_data) - as_conf.experiment_data = as_conf.substitute_dynamic_variables(as_conf.experiment_data) - as_conf.experiment_data = as_conf.parse_data_loops(as_conf.experiment_data) - # Create some jobs - job = Job('A', '1', 0, 1) - platform = PsPlatform(expid='a000', name='DUMMY_PLATFORM', config=as_conf.experiment_data) - job.section = 'RANDOM-SECTION' - job.platform = platform - job.update_parameters(as_conf, {}) + job = create_job_and_update_parameters(autosubmit_config, experiment_data) for key, value in expected_data.items(): assert job.parameters[key] == value + + +@pytest.mark.parametrize('experiment_data, attributes_to_check', [( + { + 'JOBS': { + 'RANDOM-SECTION': { + 'FILE': "test.sh", + 'PLATFORM': 'DUMMY_PLATFORM', + 'NOTIFY_ON': 'COMPLETED', + }, + }, + 'PLATFORMS': { + 'dummy_platform': { + 'type': 'ps', + }, + }, + 'ROOTDIR': 'dummy_rootdir', + 'LOCAL_TMP_DIR': 'dummy_tmpdir', + 'LOCAL_ROOT_DIR': 'dummy_rootdir', + }, + {'notify_on': ['COMPLETED']} +)]) +def test_update_parameters_attributes(autosubmit_config, experiment_data, attributes_to_check): + job = create_job_and_update_parameters(autosubmit_config, experiment_data) + for attr in attributes_to_check: + assert hasattr(job, attr) + assert getattr(job, attr) == attributes_to_check[attr] -- GitLab From 394e7264b140daec3a3bc3901b6038478d13685c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 11 Dec 2024 15:50:16 +0100 Subject: [PATCH 5/6] Update doc --- docs/source/userguide/configure/index.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index ecbcce89d..b755eb89b 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -232,7 +232,7 @@ Edit ``jobs_cxxx.yml`` in the ``conf`` folder of the experiment. defined on the parameter ``NOTIFY_ON`` .. hint:: - Remember that you can define more than one job status divided by a whitespace. +Remember that you can define more than one job status separated by a whitespace, a comma (`,`), or using a list. Example: :: @@ -246,6 +246,16 @@ Example: FILE: LOCAL_SETUP.sh PLATFORM: LOCAL NOTIFY_ON: FAILED COMPLETED + EXAMPLE_JOB: + FILE: EXAMPLE_JOB.sh + PLATFORM: LOCAL + NOTIFY_ON: FAILED, COMPLETED + EXAMPLE_JOB_2: + FILE: EXAMPLE_JOB_2.sh + PLATFORM: LOCAL + NOTIFY_ON: + - FAILED + - COMPLETED How to add a new platform ------------------------- -- GitLab From 066bc39e2a537159bdd91845b47a3b947f6f2aca Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 11 Dec 2024 15:54:58 +0100 Subject: [PATCH 6/6] Update doc for mail_to --- docs/source/userguide/configure/index.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index b755eb89b..37f1c76c5 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -221,7 +221,10 @@ Example: # Default: False NOTIFICATIONS: True # Mail address where notifications will be received - TO: jsmith@example.com rlewis@example.com + TO: + - jsmith@example.com + - rlewis@example.com + 2. Then you have to define for which jobs you want to be notified. -- GitLab