From 7600a8d9f49a274683996a75e26693f8403f962a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Wed, 28 Jun 2023 12:36:34 +0200 Subject: [PATCH 1/9] include tailer and header to parameters dict --- autosubmit/job/job.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 09253c49a..2d816a877 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -149,6 +149,8 @@ class Job(object): self.export = "none" self.dependencies = [] self.start_time = None + self.ext_header_path = '' + self.ext_tailer_path = '' def __getstate__(self): odict = self.__dict__ @@ -1098,7 +1100,37 @@ class Job(object): parameters['SCRATCH_FREE_SPACE'] = self.scratch_free_space parameters['CUSTOM_DIRECTIVES'] = self.custom_directives parameters['HYPERTHREADING'] = self.hyperthreading + # we open the files and offload the whole script as a string + # memory issues if the script is too long? Add a check to avoid problems... + if self.ext_header_path != '': + try: + header_script = open(self.ext_tailer_path, 'r').read() + except Exception as e: # add the file not found exception + AutosubmitError("Couldn't fetch extended header script") + # log it! + Log.debug( + "PARAMETER update: Extended Header: {0} doesn't exist".format(e.message)) + # ignore it! + header_script = '' + parameters['EXTENDED_HEADER'] = header_script + else: + # we have no script to include + parameters['EXTENDED_HEADER'] = '' + if self.ext_tailer_path != '': + try: + tailer_script = open(self.ext_tailer_path, 'r').read() + except Exception as e: # add the file not found exception + AutosubmitError("Couldn't fetch extended tailer script") + # log it! + Log.debug( + "PARAMETER update: Extended Tailer: {0} doesn't exist".format(e.message)) + # ignore it! + tailer_script = '' + parameters['EXTENDED_TAILER'] = tailer_script + else: + # we have no script to include + parameters['EXTENDED_TAILER'] = '' parameters['CURRENT_ARCH'] = job_platform.name parameters['CURRENT_HOST'] = job_platform.host -- GitLab From a0d55adf173df21cd7140153e1cf89598f4ab416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 30 Jun 2023 10:49:36 +0200 Subject: [PATCH 2/9] add tailer and header --- autosubmit/job/job.py | 54 +++++++++++++++++------------------- autosubmit/job/job_common.py | 15 ++++++++-- autosubmit/job/job_dict.py | 3 ++ 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 2d816a877..13221c72f 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -60,6 +60,29 @@ def threaded(fn): return wrapper +def read_header_tailer_script(script_path): + """ + Opens and reads a BASH script. + + Will ignore lines starting with # + + :param script_path: absolute path to the script + :type script_path: string + """ + script = '' + if script_path == '': + return script + try: + for line in open(script_path, 'r'): + if line[0] != "#": + script += line + except Exception as e: # add the file not found exception + Log.debug( + "PARAMETER update: Extended script: {0} doesn't exist".format(e.message)) + raise AutosubmitError("Couldn't fetch extended script") + return script + + class Job(object): """ Class to handle all the tasks with Jobs at HPC. @@ -1102,35 +1125,8 @@ class Job(object): parameters['HYPERTHREADING'] = self.hyperthreading # we open the files and offload the whole script as a string # memory issues if the script is too long? Add a check to avoid problems... - if self.ext_header_path != '': - try: - header_script = open(self.ext_tailer_path, 'r').read() - except Exception as e: # add the file not found exception - AutosubmitError("Couldn't fetch extended header script") - # log it! - Log.debug( - "PARAMETER update: Extended Header: {0} doesn't exist".format(e.message)) - # ignore it! - header_script = '' - parameters['EXTENDED_HEADER'] = header_script - else: - # we have no script to include - parameters['EXTENDED_HEADER'] = '' - - if self.ext_tailer_path != '': - try: - tailer_script = open(self.ext_tailer_path, 'r').read() - except Exception as e: # add the file not found exception - AutosubmitError("Couldn't fetch extended tailer script") - # log it! - Log.debug( - "PARAMETER update: Extended Tailer: {0} doesn't exist".format(e.message)) - # ignore it! - tailer_script = '' - parameters['EXTENDED_TAILER'] = tailer_script - else: - # we have no script to include - parameters['EXTENDED_TAILER'] = '' + parameters['EXTENDED_HEADER'] = read_header_tailer_script(self.ext_header_path) + parameters['EXTENDED_TAILER'] = read_header_tailer_script(self.ext_tailer_path) parameters['CURRENT_ARCH'] = job_platform.name parameters['CURRENT_HOST'] = job_platform.host diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 6a81f64cb..bfbe2cbac 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -110,6 +110,7 @@ class StatisticsSnippetBash: ################### # Autosubmit header ################### + locale_to_set=$(locale -a | grep ^C.) if [ -z "$locale_to_set" ] ; then # locale installed... @@ -127,7 +128,12 @@ class StatisticsSnippetBash: set -xuve job_name_ptrn='%CURRENT_LOGDIR%/%JOBNAME%' echo $(date +%s) > ${job_name_ptrn}_STAT - + + ################### + # Extended header + ################### + %EXTENDED_HEADER% + ################### # Autosubmit job ################### @@ -137,7 +143,11 @@ class StatisticsSnippetBash: @staticmethod def as_tailer(): return textwrap.dedent("""\ - + + ################### + # Extended tailer + ################### + %EXTENDED_TAILER% ################### # Autosubmit tailer ################### @@ -201,7 +211,6 @@ class StatisticsSnippetPython: # expand tailer to use python3 def as_tailer(self): return textwrap.dedent("""\ - ################### # Autosubmit tailer ################### diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index ef98ee576..f2494ee33 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -519,6 +519,9 @@ class DicJobs: job.running = self.get_option(section, 'RUNNING', 'once').lower() job.x11 = bool(self.get_option(section, 'X11', False )) + job.ext_tailer_path = self.get_option(section, 'EXTENDED_TAILER_PATH', '') + job.ext_header_path = self.get_option(section, 'EXTENDED_HEADER_PATH', '') + if self.get_option(section, "SKIPPABLE", "False").lower() == "true": job.skippable = True else: -- GitLab From d1a4f3eb5072ff0fe2168c02c1abe4b288e5fbb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 30 Jun 2023 11:23:19 +0200 Subject: [PATCH 3/9] update template file --- autosubmit/config/files/jobs.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/autosubmit/config/files/jobs.conf b/autosubmit/config/files/jobs.conf index e57b8c819..4744113bd 100644 --- a/autosubmit/config/files/jobs.conf +++ b/autosubmit/config/files/jobs.conf @@ -56,6 +56,9 @@ ## Optional. Custom directives for the resource manager of the platform used for that job. ## Put as many as you wish in json formatted array. # CUSTOM_DIRECTIVE = ["#PBS -v myvar=value, "#PBS -v othervar=value"] +## Optional. Custom directive to add a custom script at the beginning ir end of the autosubmit cmd +# EXTENDED_HEADER_PATH = /path/to/script.sh +# EXTENDED_TAILER_PATH = /path/to/script.sh [LOCAL_SETUP] FILE = LOCAL_SETUP.sh -- GitLab From c9e4ae28151bd6bc1cb6a0f07afc501732a88aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 30 Jun 2023 11:40:21 +0200 Subject: [PATCH 4/9] add error code to input/output issues --- autosubmit/job/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 13221c72f..dcbb09615 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -79,7 +79,7 @@ def read_header_tailer_script(script_path): except Exception as e: # add the file not found exception Log.debug( "PARAMETER update: Extended script: {0} doesn't exist".format(e.message)) - raise AutosubmitError("Couldn't fetch extended script") + raise AutosubmitError("Couldn't fetch extended script", 6004) return script -- GitLab From a127c03e63a8314f25e1eff556814fe0fef72984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 30 Jun 2023 11:40:31 +0200 Subject: [PATCH 5/9] add documentation on header and tailer --- docs/source/userguide/configure/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index c57ecf29d..cadaf925e 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -176,6 +176,10 @@ There are also other, less used features that you can use: * QUEUE: queue to add the job to. If not specified, uses PLATFORM default. +* EXTENDED_HEADER_PATH: path to a script to be appended at the begging of the .cmd script that Autosubmit generates. Only supports job type BASH. + +* EXTENDED_TAILER_PATH: path to a script to be appended at the end of the .cmd script that Autosubmit generates. Only supports job type BASH. + How to configure email notifications ------------------------------------ -- GitLab From cb8e42e75d6b1a82d04a8c2820943f80ae54ebc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 30 Jun 2023 12:29:38 +0200 Subject: [PATCH 6/9] made the path relative to the project dir --- autosubmit/job/job.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index dcbb09615..e3cbf4e92 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -60,7 +60,7 @@ def threaded(fn): return wrapper -def read_header_tailer_script(script_path): +def read_header_tailer_script(script_path, as_conf): """ Opens and reads a BASH script. @@ -68,12 +68,14 @@ def read_header_tailer_script(script_path): :param script_path: absolute path to the script :type script_path: string + :param as_conf: Autosubmit configuration file + :type as_conf: config """ script = '' if script_path == '': return script try: - for line in open(script_path, 'r'): + for line in open(os.path.join(as_conf.get_project_dir(), script_path), 'r'): if line[0] != "#": script += line except Exception as e: # add the file not found exception @@ -1125,8 +1127,8 @@ class Job(object): parameters['HYPERTHREADING'] = self.hyperthreading # we open the files and offload the whole script as a string # memory issues if the script is too long? Add a check to avoid problems... - parameters['EXTENDED_HEADER'] = read_header_tailer_script(self.ext_header_path) - parameters['EXTENDED_TAILER'] = read_header_tailer_script(self.ext_tailer_path) + parameters['EXTENDED_HEADER'] = read_header_tailer_script(self.ext_header_path, as_conf) + parameters['EXTENDED_TAILER'] = read_header_tailer_script(self.ext_tailer_path, as_conf) parameters['CURRENT_ARCH'] = job_platform.name parameters['CURRENT_HOST'] = job_platform.host -- GitLab From d307d0992cb6ddf167293d54d576c9fe9b952164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 30 Jun 2023 15:23:04 +0200 Subject: [PATCH 7/9] test if tailer and header are properly substituted onto the script --- test/unit/test_job.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index fae76dccb..59e1f51fc 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -179,6 +179,36 @@ class TestJob(TestCase): write_mock.write.assert_called_with('some-content: 999, 777, 666 % %') chmod_mock.assert_called_with(os.path.join(self.job._tmp_path, self.job.name + '.cmd'), 0o755) + def test_create_header_tailer_script(self): + tailer_script = '#!/usr/bin/bash\necho "Header test"\n' + header_script = '#!/usr/bin/bash\necho "Tailer test"\n' + # arrange + self.job.parameters = dict() + self.job.type = 0 # Type.BASH + self.job.parameters["EXTENDED_HEADER"] = header_script + self.job.parameters["EXTENDED_TAILER"] = tailer_script + + self.job._tmp_path = '/tmp/' + + update_content_mock = Mock(return_value='%EXTENDED_HEADER%\nsome-content\n%EXTENDED_TAILER%') + self.job.update_content = update_content_mock + + # fill the rest of the values on the job with something + update_parameters_mock = Mock(return_value=self.job.parameters) + self.job.update_parameters = update_parameters_mock + + # create an autosubmit config + config = Mock(spec=AutosubmitConfig) + + # will create a file on /tmp + self.job.create_script(config) + + with open(os.path.join(self.job._tmp_path, self.job.name + '.cmd')) as script_file: + full_script = script_file.read() + assert header_script in full_script + assert tailer_script in full_script + + def test_that_check_script_returns_false_when_there_is_an_unbound_template_variable(self): # arrange update_content_mock = Mock(return_value='some-content: %UNBOUND%') -- GitLab From 9c6c9d4f4f21a755b57aaabc6218ae25a7352cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Thu, 6 Jul 2023 11:24:00 +0200 Subject: [PATCH 8/9] move to classmethod and added check for the type --- autosubmit/job/job.py | 72 +++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index e3cbf4e92..46a68b62f 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -60,31 +60,6 @@ def threaded(fn): return wrapper -def read_header_tailer_script(script_path, as_conf): - """ - Opens and reads a BASH script. - - Will ignore lines starting with # - - :param script_path: absolute path to the script - :type script_path: string - :param as_conf: Autosubmit configuration file - :type as_conf: config - """ - script = '' - if script_path == '': - return script - try: - for line in open(os.path.join(as_conf.get_project_dir(), script_path), 'r'): - if line[0] != "#": - script += line - except Exception as e: # add the file not found exception - Log.debug( - "PARAMETER update: Extended script: {0} doesn't exist".format(e.message)) - raise AutosubmitError("Couldn't fetch extended script", 6004) - return script - - class Job(object): """ Class to handle all the tasks with Jobs at HPC. @@ -637,6 +612,49 @@ class Job(object): str(e), self.name), 6001) return + def read_header_tailer_script(self, script_path, as_conf): + """ + Opens and reads a BASH script. + + Will strip away the line with the hash bang (#!) + + :param script_path: relative to the experiment directory path to the script + :type script_path: string + :param as_conf: Autosubmit configuration file + :type as_conf: config + """ + script = '' + if script_path == '': + return script + try: + for line in open(os.path.join(as_conf.get_project_dir(), script_path), 'r'): + if "!#" not in line: + script += line + else: + # check if the type of the script matches the one in the extended + if "bash" in line: + if self.type != Type.BASH: + Log.error("PARAMETER update: Extended script: script seems BASH but job isn't") + # We stop Autosubmit if we don't find the script + raise AutosubmitCritical("Extended script: script seems BASH but job isn't\n", 7011) + elif "Rscript" in line: + if self.type != Type.R: + Log.error("PARAMETER update: Extended script: script seems Rscript but job isn't") + # We stop Autosubmit if we don't find the script + raise AutosubmitCritical("Extended script: script seems Rscript but job isn't\n", 7011) + elif "python" in line: + if self.type not in (Type.PYTHON, Type.PYTHON2, Type.PYTHON3): + Log.error( + "PARAMETER update: Extended script: script seems Python but job isn't") + # We stop Autosubmit if we don't find the script + raise AutosubmitCritical("Extended script: script seems Python but job isn't\n", 7011) + except Exception as e: # log + Log.error( + "PARAMETER update: Extended script: {0} doesn't exist".format(e.message)) + # We stop Autosubmit if we don't find the script + raise AutosubmitCritical("Extended script: failed to fetch {0} \n".format(str(e)), 7014) + return script + @threaded def retrieve_logfiles(self, copy_remote_logs, local_logs, remote_logs, expid, platform_name,fail_count = 0,job_id=""): max_logs = 0 @@ -1127,8 +1145,8 @@ class Job(object): parameters['HYPERTHREADING'] = self.hyperthreading # we open the files and offload the whole script as a string # memory issues if the script is too long? Add a check to avoid problems... - parameters['EXTENDED_HEADER'] = read_header_tailer_script(self.ext_header_path, as_conf) - parameters['EXTENDED_TAILER'] = read_header_tailer_script(self.ext_tailer_path, as_conf) + parameters['EXTENDED_HEADER'] = self.read_header_tailer_script(self.ext_header_path, as_conf) + parameters['EXTENDED_TAILER'] = self.read_header_tailer_script(self.ext_tailer_path, as_conf) parameters['CURRENT_ARCH'] = job_platform.name parameters['CURRENT_HOST'] = job_platform.host -- GitLab From cf5ad01cfd3d95d335b0fb7a90a42508e3ddea5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Thu, 6 Jul 2023 11:43:29 +0200 Subject: [PATCH 9/9] Remove error message when critical error is raised --- autosubmit/job/job.py | 44 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 46a68b62f..6565b8c31 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -614,7 +614,7 @@ class Job(object): def read_header_tailer_script(self, script_path, as_conf): """ - Opens and reads a BASH script. + Opens and reads a script. If it is not a BASH script it will fail :( Will strip away the line with the hash bang (#!) @@ -623,36 +623,32 @@ class Job(object): :param as_conf: Autosubmit configuration file :type as_conf: config """ + script_name = script_path.rsplit("/")[-1] # pick the name of the script for a more verbose error script = '' if script_path == '': return script + try: - for line in open(os.path.join(as_conf.get_project_dir(), script_path), 'r'): - if "!#" not in line: - script += line - else: - # check if the type of the script matches the one in the extended - if "bash" in line: - if self.type != Type.BASH: - Log.error("PARAMETER update: Extended script: script seems BASH but job isn't") - # We stop Autosubmit if we don't find the script - raise AutosubmitCritical("Extended script: script seems BASH but job isn't\n", 7011) - elif "Rscript" in line: - if self.type != Type.R: - Log.error("PARAMETER update: Extended script: script seems Rscript but job isn't") - # We stop Autosubmit if we don't find the script - raise AutosubmitCritical("Extended script: script seems Rscript but job isn't\n", 7011) - elif "python" in line: - if self.type not in (Type.PYTHON, Type.PYTHON2, Type.PYTHON3): - Log.error( - "PARAMETER update: Extended script: script seems Python but job isn't") - # We stop Autosubmit if we don't find the script - raise AutosubmitCritical("Extended script: script seems Python but job isn't\n", 7011) + script_file = open(os.path.join(as_conf.get_project_dir(), script_path), 'r') except Exception as e: # log - Log.error( - "PARAMETER update: Extended script: {0} doesn't exist".format(e.message)) # We stop Autosubmit if we don't find the script raise AutosubmitCritical("Extended script: failed to fetch {0} \n".format(str(e)), 7014) + + for line in script_file: + if "#!" not in line: + script += line + else: + # check if the type of the script matches the one in the extended + if "bash" in line: + if self.type != Type.BASH: + raise AutosubmitCritical("Extended script: script {0} seems BASH but job {1} isn't\n".format(script_name, self.script_name), 7011) + elif "Rscript" in line: + if self.type != Type.R: + raise AutosubmitCritical("Extended script: script {0} seems Rscript but job {1} isn't\n".format(script_name, self.script_name), 7011) + elif "python" in line: + if self.type not in (Type.PYTHON, Type.PYTHON2, Type.PYTHON3): + raise AutosubmitCritical("Extended script: script {0} seems Python but job {1} isn't\n".format(script_name, self.script_name), 7011) + return script @threaded -- GitLab