diff --git a/autosubmit/config/files/jobs.conf b/autosubmit/config/files/jobs.conf index e57b8c81977d251ef0ec1b36f46831a765c674b3..4744113bd0125651c2eae6f26d3ecedc9049335a 100644 --- a/autosubmit/config/files/jobs.conf +++ b/autosubmit/config/files/jobs.conf @@ -56,6 +56,9 @@ ## Optional. Custom directives for the resource manager of the platform used for that job. ## Put as many as you wish in json formatted array. # CUSTOM_DIRECTIVE = ["#PBS -v myvar=value, "#PBS -v othervar=value"] +## Optional. Custom directive to add a custom script at the beginning ir end of the autosubmit cmd +# EXTENDED_HEADER_PATH = /path/to/script.sh +# EXTENDED_TAILER_PATH = /path/to/script.sh [LOCAL_SETUP] FILE = LOCAL_SETUP.sh diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 09253c49af08cfef66771f8b62fb41c176d22f61..6565b8c3112bde7da2be3faae7940db503287b99 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -149,6 +149,8 @@ class Job(object): self.export = "none" self.dependencies = [] self.start_time = None + self.ext_header_path = '' + self.ext_tailer_path = '' def __getstate__(self): odict = self.__dict__ @@ -610,6 +612,45 @@ class Job(object): str(e), self.name), 6001) return + def read_header_tailer_script(self, script_path, as_conf): + """ + Opens and reads a script. If it is not a BASH script it will fail :( + + Will strip away the line with the hash bang (#!) + + :param script_path: relative to the experiment directory path to the script + :type script_path: string + :param as_conf: Autosubmit configuration file + :type as_conf: config + """ + script_name = script_path.rsplit("/")[-1] # pick the name of the script for a more verbose error + script = '' + if script_path == '': + return script + + try: + script_file = open(os.path.join(as_conf.get_project_dir(), script_path), 'r') + except Exception as e: # log + # We stop Autosubmit if we don't find the script + raise AutosubmitCritical("Extended script: failed to fetch {0} \n".format(str(e)), 7014) + + for line in script_file: + if "#!" not in line: + script += line + else: + # check if the type of the script matches the one in the extended + if "bash" in line: + if self.type != Type.BASH: + raise AutosubmitCritical("Extended script: script {0} seems BASH but job {1} isn't\n".format(script_name, self.script_name), 7011) + elif "Rscript" in line: + if self.type != Type.R: + raise AutosubmitCritical("Extended script: script {0} seems Rscript but job {1} isn't\n".format(script_name, self.script_name), 7011) + elif "python" in line: + if self.type not in (Type.PYTHON, Type.PYTHON2, Type.PYTHON3): + raise AutosubmitCritical("Extended script: script {0} seems Python but job {1} isn't\n".format(script_name, self.script_name), 7011) + + return script + @threaded def retrieve_logfiles(self, copy_remote_logs, local_logs, remote_logs, expid, platform_name,fail_count = 0,job_id=""): max_logs = 0 @@ -1098,7 +1139,10 @@ class Job(object): parameters['SCRATCH_FREE_SPACE'] = self.scratch_free_space parameters['CUSTOM_DIRECTIVES'] = self.custom_directives parameters['HYPERTHREADING'] = self.hyperthreading - + # we open the files and offload the whole script as a string + # memory issues if the script is too long? Add a check to avoid problems... + parameters['EXTENDED_HEADER'] = self.read_header_tailer_script(self.ext_header_path, as_conf) + parameters['EXTENDED_TAILER'] = self.read_header_tailer_script(self.ext_tailer_path, as_conf) parameters['CURRENT_ARCH'] = job_platform.name parameters['CURRENT_HOST'] = job_platform.host diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 6a81f64cbd5f7ddfcab7b7f397a10e291553d89a..bfbe2cbac97749123c298cb5b3f71a18d9092513 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -110,6 +110,7 @@ class StatisticsSnippetBash: ################### # Autosubmit header ################### + locale_to_set=$(locale -a | grep ^C.) if [ -z "$locale_to_set" ] ; then # locale installed... @@ -127,7 +128,12 @@ class StatisticsSnippetBash: set -xuve job_name_ptrn='%CURRENT_LOGDIR%/%JOBNAME%' echo $(date +%s) > ${job_name_ptrn}_STAT - + + ################### + # Extended header + ################### + %EXTENDED_HEADER% + ################### # Autosubmit job ################### @@ -137,7 +143,11 @@ class StatisticsSnippetBash: @staticmethod def as_tailer(): return textwrap.dedent("""\ - + + ################### + # Extended tailer + ################### + %EXTENDED_TAILER% ################### # Autosubmit tailer ################### @@ -201,7 +211,6 @@ class StatisticsSnippetPython: # expand tailer to use python3 def as_tailer(self): return textwrap.dedent("""\ - ################### # Autosubmit tailer ################### diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index ef98ee5761508a4242031accfe3bec01d8cb42ab..f2494ee33699503aebfa9002305a7fcfa862d531 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -519,6 +519,9 @@ class DicJobs: job.running = self.get_option(section, 'RUNNING', 'once').lower() job.x11 = bool(self.get_option(section, 'X11', False )) + job.ext_tailer_path = self.get_option(section, 'EXTENDED_TAILER_PATH', '') + job.ext_header_path = self.get_option(section, 'EXTENDED_HEADER_PATH', '') + if self.get_option(section, "SKIPPABLE", "False").lower() == "true": job.skippable = True else: diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index c57ecf29d9d66f83f91aa2bb49284383acf8d463..cadaf925e87847a078b95c71f785a5142b3fe4ce 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -176,6 +176,10 @@ There are also other, less used features that you can use: * QUEUE: queue to add the job to. If not specified, uses PLATFORM default. +* EXTENDED_HEADER_PATH: path to a script to be appended at the begging of the .cmd script that Autosubmit generates. Only supports job type BASH. + +* EXTENDED_TAILER_PATH: path to a script to be appended at the end of the .cmd script that Autosubmit generates. Only supports job type BASH. + How to configure email notifications ------------------------------------ diff --git a/test/unit/test_job.py b/test/unit/test_job.py index fae76dccb110ff31d84373f732753b8dc4647b22..59e1f51fc89670e28e0c2cfe12538fad4a5c4d8e 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -179,6 +179,36 @@ class TestJob(TestCase): write_mock.write.assert_called_with('some-content: 999, 777, 666 % %') chmod_mock.assert_called_with(os.path.join(self.job._tmp_path, self.job.name + '.cmd'), 0o755) + def test_create_header_tailer_script(self): + tailer_script = '#!/usr/bin/bash\necho "Header test"\n' + header_script = '#!/usr/bin/bash\necho "Tailer test"\n' + # arrange + self.job.parameters = dict() + self.job.type = 0 # Type.BASH + self.job.parameters["EXTENDED_HEADER"] = header_script + self.job.parameters["EXTENDED_TAILER"] = tailer_script + + self.job._tmp_path = '/tmp/' + + update_content_mock = Mock(return_value='%EXTENDED_HEADER%\nsome-content\n%EXTENDED_TAILER%') + self.job.update_content = update_content_mock + + # fill the rest of the values on the job with something + update_parameters_mock = Mock(return_value=self.job.parameters) + self.job.update_parameters = update_parameters_mock + + # create an autosubmit config + config = Mock(spec=AutosubmitConfig) + + # will create a file on /tmp + self.job.create_script(config) + + with open(os.path.join(self.job._tmp_path, self.job.name + '.cmd')) as script_file: + full_script = script_file.read() + assert header_script in full_script + assert tailer_script in full_script + + def test_that_check_script_returns_false_when_there_is_an_unbound_template_variable(self): # arrange update_content_mock = Mock(return_value='some-content: %UNBOUND%')