diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 47565d3b9ab1b408b94840c831cf18d7899a3524..7b38eb744854faf229e441a4855b71ce8920b801 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -211,6 +211,8 @@ class Job(object): self._export = "none" self._dependencies = [] self.running = "once" + self.ext_header_path = "" + self.ext_tailer_path = "" self.start_time = None self.edge_info = dict() self.total_jobs = None @@ -837,6 +839,43 @@ class Job(object): retrials_list.insert(0, retrial_dates) return retrials_list + def read_header_tailer_script(self, script_path: str, as_conf: AutosubmitConfig): + """ + Opens and reads a script. If it is not a BASH script it will fail :( + + Will strip away the line with the hash bang (#!) + + :param script_path: relative to the experiment directory path to the script + :param as_conf: Autosubmit configuration file + """ + script_name = script_path.rsplit("/")[-1] # pick the name of the script for a more verbose error + script = '' + if script_path == '': + return script + + try: + script_file = open(os.path.join(as_conf.get_project_dir(), script_path), 'r') + except Exception as e: # log + # We stop Autosubmit if we don't find the script + raise AutosubmitCritical("Extended script: failed to fetch {0} \n".format(str(e)), 7014) + + for line in script_file: + if "#!" not in line: + script += line + else: + # check if the type of the script matches the one in the extended + if "bash" in line: + if self.type != Type.BASH: + raise AutosubmitCritical("Extended script: script {0} seems BASH but job {1} isn't\n".format(script_name, self.script_name), 7011) + elif "Rscript" in line: + if self.type != Type.R: + raise AutosubmitCritical("Extended script: script {0} seems Rscript but job {1} isn't\n".format(script_name, self.script_name), 7011) + elif "python" in line: + if self.type not in (Type.PYTHON, Type.PYTHON2, Type.PYTHON3): + raise AutosubmitCritical("Extended script: script {0} seems Python but job {1} isn't\n".format(script_name, self.script_name), 7011) + + return script + def retrieve_logfiles_unthreaded(self, copy_remote_logs, local_logs): remote_logs = (self.script_name + ".out."+str(self.fail_count), self.script_name + ".err."+str(self.fail_count)) out_exist = False @@ -1251,7 +1290,7 @@ class Job(object): parameters['CURRENT_LOGDIR'] = job_platform.get_files_path() return parameters - def update_platform_associated_parameters(self,as_conf, parameters, job_platform, chunk): + def update_platform_associated_parameters(self, as_conf, parameters, job_platform, chunk): self.executable = str(as_conf.jobs_data[self.section].get("EXECUTABLE", as_conf.platforms_data.get(job_platform.name,{}).get("EXECUTABLE",""))) self.total_jobs = int(as_conf.jobs_data[self.section].get("TOTALJOBS", job_platform.total_jobs)) self.max_waiting_jobs = int(as_conf.jobs_data[self.section].get("MAXWAITINGJOBS", job_platform.max_waiting_jobs)) @@ -1313,6 +1352,10 @@ class Job(object): parameters['SCRATCH_FREE_SPACE'] = self.scratch_free_space parameters['CUSTOM_DIRECTIVES'] = self.custom_directives parameters['HYPERTHREADING'] = self.hyperthreading + # memory issues? We are storing the whole extended script as a string + if as_conf.get_project_type() != "none": + parameters['EXTENDED_HEADER'] = self.read_header_tailer_script(self.ext_header_path, as_conf) + parameters['EXTENDED_TAILER'] = self.read_header_tailer_script(self.ext_tailer_path, as_conf) parameters['CURRENT_QUEUE'] = self.queue return parameters diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 4d05d985ca2dc28415a35746cd6b1c225cc3d6b0..e5294a1b67aea97e685f75ed462d7362212dc640 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -127,6 +127,11 @@ class StatisticsSnippetBash: set -xuve job_name_ptrn='%CURRENT_LOGDIR%/%JOBNAME%' echo $(date +%s) > ${job_name_ptrn}_STAT + + ################## + # Extended header + ################## + %EXTENDED_HEADER% ################### # Autosubmit job @@ -138,6 +143,11 @@ class StatisticsSnippetBash: def as_tailer(): return textwrap.dedent("""\ + ################### + # Extended tailer + ################### + %EXTENDED_TAILER% + ################### # Autosubmit tailer ################### diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index e2f673563efe1b6859cdfdf37350b05b8c5a08f7..51c34023c9b6f7a031f255e7f29da74ddd4d88ba 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -425,6 +425,9 @@ class DicJobs: job.running = str(parameters[section].get( 'RUNNING', 'once')) job.x11 = str(parameters[section].get( 'X11', False )).lower() job.skippable = str(parameters[section].get( "SKIPPABLE", False)).lower() + # we make the empty string as default value, case is not present in the config + job.ext_header_path = str(parameters[section].get("EXTENDED_HEADER_PATH", "")) + job.ext_tailer_path = str(parameters[section].get("EXTENDED_TAILER_PATH", "")) self._jobs_list.get_job_list().append(job) return job diff --git a/docs/source/userguide/defining_workflows/fig/dashed.png b/docs/source/userguide/defining_workflows/fig/dashed.png deleted file mode 100644 index 6986dc0f8f592ae69614f0fccb4f706c0a521938..0000000000000000000000000000000000000000 Binary files a/docs/source/userguide/defining_workflows/fig/dashed.png and /dev/null differ diff --git a/docs/source/userguide/defining_workflows/fig/select_chunks.png b/docs/source/userguide/defining_workflows/fig/select_chunks.png deleted file mode 100644 index bd994cc9edaa5dc941752e0674ebdf2972a1aa58..0000000000000000000000000000000000000000 Binary files a/docs/source/userguide/defining_workflows/fig/select_chunks.png and /dev/null differ diff --git a/docs/source/userguide/defining_workflows/fig/select_members.png b/docs/source/userguide/defining_workflows/fig/select_members.png deleted file mode 100644 index c908d45e986ea4e03013db3b0ead96a382f2b16b..0000000000000000000000000000000000000000 Binary files a/docs/source/userguide/defining_workflows/fig/select_members.png and /dev/null differ diff --git a/docs/source/userguide/defining_workflows/fig/skip.png b/docs/source/userguide/defining_workflows/fig/skip.png deleted file mode 100644 index 6083dad8b3c16a5667a366adf2f210d3896c50bf..0000000000000000000000000000000000000000 Binary files a/docs/source/userguide/defining_workflows/fig/skip.png and /dev/null differ diff --git a/test/regression/tests_runner.py b/test/regression/tests_runner.py index 54d38624e40ceb13a7ea11fd624324c87dca0519..13cb8fc056ca3daea2b2f6cbc3ac3af96fb036c3 100644 --- a/test/regression/tests_runner.py +++ b/test/regression/tests_runner.py @@ -79,6 +79,7 @@ def run(current_experiment_id, only_list=None, exclude_list=None, max_threads=5) tests_parser.optionxform = str tests_parser.read(tests_parser_file) + # Resetting the database clean_database(db_path) create_database() diff --git a/test/unit/test_job.py b/test/unit/test_job.py index caaf9c60a2da9478839ee8cc400f03422ad731f5..02d906983ff3ea05a92a4fe192b1ecd6f4805e17 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -191,6 +191,40 @@ class TestJob(TestCase): write_mock.write.assert_called_with(b'some-content: 999, 777, 666 % %') chmod_mock.assert_called_with(os.path.join(self.job._tmp_path, self.job.name + '.cmd'), 0o755) + def test_create_header_tailer_script(self): + """ + Here we are testing if the parameters from the dictionary are properly read and then regex acts correctly. + + The internal logic to avoid header/tailer of a different type on a R or Python script is not tested. We skip + all of that when we do the content_update mock. + """ + # arrange + header_script = '#!/usr/bin/bash\necho "Header test"\n' + tailer_script = '#!/usr/bin/bash\necho "Tailer test"\n' + # the full script in its binary form + expected_script = b'#!/usr/bin/bash\necho "Header test"\n\nsome-content\n#!/usr/bin/bash\necho "Tailer test"\n' + self.job.parameters = dict() + self.job.parameters['EXTENDED_HEADER'] = header_script + self.job.parameters['EXTENDED_TAILER'] = tailer_script + # We mock the function that returns the template script it is tuple because that is what the + # function outputs. We don't care about additional templates, hence the empty list :D + self.job.update_content = Mock(return_value=('%EXTENDED_HEADER%\nsome-content\n%EXTENDED_TAILER%', [])) + # self.job.update_parameters = Mock(return_value=self.job.parameters) + # create an autosubmit config + config = Mock(spec=AutosubmitConfig) + + # mock parts to write the file + sys.modules['os'].chmod = Mock() # needed cuz AS changes the permission on file + write_mock = Mock().write = Mock() + open_mock = Mock(return_value=write_mock) # so that we don't try to open a file that does not exist + # here we replace (patch) the "open" function with our mocked one >:) + with patch.object(builtins, "open", open_mock): + # act + self.job.create_script(config) + + # assert + write_mock.write.assert_called_with(expected_script) + def test_that_check_script_returns_false_when_there_is_an_unbound_template_variable(self): # arrange update_content_mock = Mock(return_value=('some-content: %UNBOUND%','some-content: %UNBOUND%'))