From 684c457e67868cc8a68ee7f801699f687d6e3a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Tue, 29 Aug 2023 16:09:39 +0200 Subject: [PATCH 1/9] Add R, Bash, and python extended scripts --- autosubmit/job/job.py | 83 ++++++++++++++++++++++++++++++++++++ autosubmit/job/job_common.py | 11 +++-- autosubmit/job/job_dict.py | 3 ++ 3 files changed, 93 insertions(+), 4 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index e5b921de2..9073dc94f 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -217,6 +217,8 @@ class Job(object): self._dependencies = [] self.running = "once" self.start_time = None + self.ext_header_path = '' + self.ext_tailer_path = '' self.edge_info = dict() self.total_jobs = None self.max_waiting_jobs = None @@ -484,6 +486,83 @@ class Job(object): del odict['_platform'] # remove filehandle entry return odict + def read_header_tailer_script(self, script_path: str, as_conf: AutosubmitConfig, is_header: bool): + """ + Opens and reads a script. If it is not a BASH script it will fail :( + + Will strip away the line with the hash bang (#!) + + :param script_path: relative to the experiment directory path to the script + :param as_conf: Autosubmit configuration file + :param is_header: boolean indicating if it is header extended script + """ + + found_hashbang = False + script_name = script_path.rsplit("/")[-1] # pick the name of the script for a more verbose error + script = '' + if script_path == '': + return script + + # adjusts the error message to the type of the script + if is_header: + error_message_type = "header" + else: + error_message_type = "tailer" + + try: + # find the absolute path + script_file = open(os.path.join(as_conf.get_project_dir(), script_path), 'r') + except Exception as e: # log + # We stop Autosubmit if we don't find the script + raise AutosubmitCritical("Extended {1} script: failed to fetch {0} \n".format(str(e), + error_message_type), 7014) + + for line in script_file: + if "#!" not in line: + script += line + else: + found_hashbang = True + # check if the type of the script matches the one in the extended + if "bash" in line: + if self.type != Type.BASH: + raise AutosubmitCritical( + "Extended {2} script: script {0} seems BASH but job {1} isn't\n".format(script_name, + self.script_name, + error_message_type), + 7011) + elif "Rscript" in line: + if self.type != Type.R: + raise AutosubmitCritical( + "Extended {2} script: script {0} seems Rscript but job {1} isn't\n".format(script_name, + self.script_name, + error_message_type), + 7011) + elif "python" in line: + if self.type not in (Type.PYTHON, Type.PYTHON2, Type.PYTHON3): + raise AutosubmitCritical( + "Extended {2} script: script {0} seems Python but job {1} isn't\n".format(script_name, + self.script_name, + error_message_type), + 7011) + else: + raise AutosubmitCritical( + "Extended {2} script: couldn't figure out script {0} type ".format(script_name, + self.script_name, + error_message_type), 7011) + + if not found_hashbang: + raise AutosubmitCritical( + "Extended {2} script: couldn't figure out script {0} type ".format(script_name, + self.script_name, + error_message_type), 7011) + + if is_header: + script = "\n###############\n# Header script\n###############\n" + script + else: + script = "\n###############\n# Tailer script\n###############\n" + script + + return script + @property def parents(self): """ @@ -1341,6 +1420,10 @@ class Job(object): parameters['SCRATCH_FREE_SPACE'] = self.scratch_free_space parameters['CUSTOM_DIRECTIVES'] = self.custom_directives parameters['HYPERTHREADING'] = self.hyperthreading + # we open the files and offload the whole script as a string + # memory issues if the script is too long? Add a check to avoid problems... + parameters['EXTENDED_HEADER'] = self.read_header_tailer_script(self.ext_header_path, as_conf, True) + parameters['EXTENDED_TAILER'] = self.read_header_tailer_script(self.ext_tailer_path, as_conf, False) parameters['CURRENT_QUEUE'] = self.queue parameters['RESERVATION'] = self.reservation return parameters diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index d705b1d1d..69d541352 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -138,6 +138,7 @@ class StatisticsSnippetBash: AS_CHECKPOINT_CALLS=$((AS_CHECKPOINT_CALLS+1)) touch ${job_name_ptrn}_CHECKPOINT_${AS_CHECKPOINT_CALLS} } + %EXTENDED_HEADER% ################### # Autosubmit job ################### @@ -147,7 +148,7 @@ class StatisticsSnippetBash: @staticmethod def as_tailer(): return textwrap.dedent("""\ - + %EXTENDED_TAILER% ################### # Autosubmit tailer ################### @@ -209,7 +210,8 @@ class StatisticsSnippetPython: global AS_CHECKPOINT_CALLS global job_name_ptrn AS_CHECKPOINT_CALLS = AS_CHECKPOINT_CALLS + 1 - open(job_name_ptrn + '_CHECKPOINT_' + str(AS_CHECKPOINT_CALLS), 'w').close() + open(job_name_ptrn + '_CHECKPOINT_' + str(AS_CHECKPOINT_CALLS), 'w').close() + %EXTENDED_HEADER% ################### # Autosubmit job ################### @@ -220,7 +222,7 @@ class StatisticsSnippetPython: # expand tailer to use python3 def as_tailer(self): return textwrap.dedent("""\ - + %EXTENDED_TAILER% ################### # Autosubmit tailer ################### @@ -283,6 +285,7 @@ class StatisticsSnippetR: fileConn<-file(paste(job_name_ptrn,"_CHECKPOINT_",AS_CHECKPOINT_CALLS, sep = ''),"w") close(fileConn) } + %EXTENDED_HEADER% ################### # Autosubmit job ################### @@ -292,7 +295,7 @@ class StatisticsSnippetR: @staticmethod def as_tailer(): return textwrap.dedent("""\ - + %EXTENDED_TAILER% ################### # Autosubmit tailer ################### diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index e2f673563..0520785c5 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -425,6 +425,9 @@ class DicJobs: job.running = str(parameters[section].get( 'RUNNING', 'once')) job.x11 = str(parameters[section].get( 'X11', False )).lower() job.skippable = str(parameters[section].get( "SKIPPABLE", False)).lower() + # store from within the relative path to the project + job.ext_header_path = str(parameters[section].get('EXTENDED_HEADER_PATH', '')) + job.ext_tailer_path = str(parameters[section].get('EXTENDED_TAILER_PATH', '')) self._jobs_list.get_job_list().append(job) return job -- GitLab From e2fef0cfc2d4d80e10548e1430783a81924630d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Wed, 30 Aug 2023 11:38:16 +0200 Subject: [PATCH 2/9] Handle if user sets value with empty key --- autosubmit/job/job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 9073dc94f..e100f871c 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -500,7 +500,8 @@ class Job(object): found_hashbang = False script_name = script_path.rsplit("/")[-1] # pick the name of the script for a more verbose error script = '' - if script_path == '': + # the value might be None string if the key has been set, but with no value + if script_path == '' or script_path == "None": return script # adjusts the error message to the type of the script -- GitLab From 0c75e681a6b52eb2a9cd2c8d1aa6fb439eeabe12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Thu, 31 Aug 2023 12:56:09 +0200 Subject: [PATCH 3/9] change the check of hashbang to the first two characters --- autosubmit/job/job.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index e100f871c..c8b5e99eb 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -519,7 +519,7 @@ class Job(object): error_message_type), 7014) for line in script_file: - if "#!" not in line: + if line[:2] != "#!": script += line else: found_hashbang = True @@ -527,7 +527,7 @@ class Job(object): if "bash" in line: if self.type != Type.BASH: raise AutosubmitCritical( - "Extended {2} script: script {0} seems BASH but job {1} isn't\n".format(script_name, + "Extended {2} script: script {0} seems Bash but job {1} isn't\n".format(script_name, self.script_name, error_message_type), 7011) @@ -547,13 +547,13 @@ class Job(object): 7011) else: raise AutosubmitCritical( - "Extended {2} script: couldn't figure out script {0} type ".format(script_name, + "Extended {2} script: couldn't figure out script {0} type\n".format(script_name, self.script_name, error_message_type), 7011) if not found_hashbang: raise AutosubmitCritical( - "Extended {2} script: couldn't figure out script {0} type ".format(script_name, + "Extended {2} script: couldn't figure out script {0} type\n".format(script_name, self.script_name, error_message_type), 7011) -- GitLab From bc59282d797b87098efc0641ea22bf46bb172b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Thu, 31 Aug 2023 12:56:58 +0200 Subject: [PATCH 4/9] test all the routes from extended tailer and header except fetching the file --- test/unit/test_job.py | 231 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 91d7fbe24..ce5500a0e 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -14,6 +14,7 @@ from autosubmitconfigparser.config.configcommon import BasicConfig, YAMLParserFa from mock import Mock, MagicMock from mock import patch +import log.log from autosubmit.autosubmit import Autosubmit from autosubmit.job.job import Job from autosubmit.job.job_common import Status @@ -25,6 +26,9 @@ if version_info.major == 2: else: import builtins +# import the exception. Three dots means two folders up the hierarchy +# reference: https://peps.python.org/pep-0328/ +from log.log import AutosubmitCritical class TestJob(TestCase): def setUp(self): @@ -244,6 +248,233 @@ class TestJob(TestCase): update_content_mock.assert_called_with(config) self.assertTrue(checked) + @patch('autosubmitconfigparser.config.basicconfig.BasicConfig') + def test_header_tailer(self, mocked_global_basic_config: Mock): + """Test if header and tailer are being properly substituted onto the final .cmd file without + a bunch of mocks + + Copied from Aina's and Bruno's test for the reservation key. Hence, the following code still + applies: "Actually one mock, but that's for something in the AutosubmitConfigParser that can + be modified to remove the need of that mock." + """ + + # set up + + expid = 'zzyy' + + # this is the actual job's type + for script_type in ["Bash", "Python", "Rscript"]: + # we will test the substitution on the header, tailer, and in both + for extended_position in ["header", "tailer", "header tailer", "neither"]: + # we will test every type of the extended script. It should fail in all but the correct script type + for extended_type in ["Bash", "Python", "Rscript", "Bad"]: + with tempfile.TemporaryDirectory() as temp_dir: + BasicConfig.LOCAL_ROOT_DIR = str(temp_dir) + Path(temp_dir, expid).mkdir() + # FIXME: (Copied from Bruno) Not sure why but the submitted and Slurm were using the $expid/tmp/ASLOGS folder? + for path in [f'{expid}/tmp', f'{expid}/tmp/ASLOGS', f'{expid}/tmp/ASLOGS_{expid}', f'{expid}/proj', + f'{expid}/conf', f'{expid}/proj/project_files']: + Path(temp_dir, path).mkdir() + + header_file_name = "" + # this is the part of the script that executes + header_content = "" + tailer_file_name = "" + tailer_content = "" + + # create the extended header and tailer scripts + if "header" in extended_position: + if extended_type == "Bash": + header_content = 'echo "header bash"' + full_header_content = dedent(f'''\ + #!/usr/bin/bash + {header_content} + ''') + header_file_name = "header.sh" + elif extended_type == "Python": + header_content = 'print("header python")' + full_header_content = dedent(f'''\ + #!/usr/bin/python + {header_content} + ''') + header_file_name = "header.py" + elif extended_type == "Rscript": + header_content = 'print("header R")' + full_header_content = dedent(f'''\ + #!/usr/bin/env Rscript + {header_content} + ''') + header_file_name = "header.R" + else: # a badly formed script + header_content = 'this is a script without #!' + full_header_content = dedent(f'''\ + {header_content} + ''') + header_file_name = "header.bad" + + # build the header script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{header_file_name}'), 'w+') as header: + header.write(full_header_content) + header.flush() + + if "tailer" in extended_position: + if extended_type == "Bash": + tailer_content = 'echo "tailer bash"' + full_tailer_content = dedent(f'''\ + #!/usr/bin/bash + {tailer_content} + ''') + tailer_file_name = "tailer.sh" + elif extended_type == "Python": + tailer_content = 'print("tailer python")' + full_tailer_content = dedent(f'''\ + #!/usr/bin/python + {tailer_content} + ''') + tailer_file_name = "tailer.py" + elif extended_type == "Rscript": + tailer_content = 'print("header R")' + full_tailer_content = dedent(f'''\ + #!/usr/bin/env Rscript + {tailer_content} + ''') + tailer_file_name = "tailer.R" + else: # a badly formed script + tailer_content = 'this is a script without #!' + full_tailer_content = dedent(f'''\ + {tailer_content} + ''') + tailer_file_name = "tailer.bad" + + # build the tailer script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{tailer_file_name}'), 'w+') as tailer: + tailer.write(full_tailer_content) + tailer.flush() + + # configuration file + + with open(Path(temp_dir, f'{expid}/conf/minimal.yml'), 'w+') as minimal: + minimal.write(dedent(f'''\ + DEFAULT: + EXPID: {expid} + HPCARCH: local + JOBS: + A: + FILE: a + TYPE: {script_type if script_type != "Rscript" else "R"} + PLATFORM: local + RUNNING: once + EXTENDED_HEADER_PATH: {header_file_name} + EXTENDED_TAILER_PATH: {tailer_file_name} + PLATFORMS: + test: + TYPE: slurm + HOST: localhost + PROJECT: abc + QUEUE: debug + USER: me + SCRATCH_DIR: /anything/ + ADD_PROJECT_TO_HOST: False + MAX_WALLCLOCK: '000:55' + TEMP_DIR: '' + ''')) + minimal.flush() + + mocked_basic_config = Mock(spec=BasicConfig) + mocked_basic_config.LOCAL_ROOT_DIR = str(temp_dir) + mocked_global_basic_config.LOCAL_ROOT_DIR.return_value = str(temp_dir) + + config = AutosubmitConfig(expid, basic_config=mocked_basic_config, parser_factory=YAMLParserFactory()) + config.reload(True) + + # act + + parameters = config.load_parameters() + + job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(), + Autosubmit._get_job_list_persistence(expid, config), config) + job_list_obj.generate( + date_list=[], + member_list=[], + num_chunks=1, + chunk_ini=1, + parameters=parameters, + date_format='M', + default_retrials=config.get_retrials(), + default_job_type=config.get_default_job_type(), + wrapper_type=config.get_wrapper_type(), + wrapper_jobs={}, + notransitive=True, + update_structure=True, + run_only_members=config.get_member_list(run_only=True), + jobs_data=config.experiment_data, + as_conf=config + ) + + job_list = job_list_obj.get_job_list() + + submitter = Autosubmit._get_submitter(config) + submitter.load_platforms(config) + + hpcarch = config.get_platform() + for job in job_list: + if job.platform_name == "" or job.platform_name is None: + job.platform_name = hpcarch + job.platform = submitter.platforms[job.platform_name] + + # pick ur single job + job = job_list[0] + + # assert + if extended_position != "neither": + # we have either a tailer or header + if extended_type == script_type: + # load the parameters + job.check_script(config, parameters) + # create the script + job.create_script(config) + with open(Path(temp_dir, f'{expid}/tmp/zzyy_A.cmd'), 'r') as file: + full_script = file.read() + if "header" in extended_position: + self.assertTrue(header_content in full_script) + if "tailer" in extended_position: + self.assertTrue(tailer_content in full_script) + else: # the extended script is not the same as the host + if extended_type == "Bad": # we check if a script without hash bang fails + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: couldn't figure out script {header_file_name} type\n") + else: + self.assertEqual(context.exception.message, + f"Extended tailer script: couldn't figure out script {tailer_file_name} type\n") + else: # if the script is properly done + # Asserts that an exception is raised if there is a mismatch between job and extension types + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + # if we have both header and tailer, it will fail at the header first + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: script {header_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") + else: # extended_position == "tailer" + self.assertEqual(context.exception.message, + f"Extended tailer script: script {tailer_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") + else: # we don't have either a tailer or header + # load the parameters + job.check_script(config, parameters) + # create the script + job.create_script(config) + # finally, if we don't have scripts, check if the placeholders have been removed + with open(Path(temp_dir, f'{expid}/tmp/zzyy_A.cmd'), 'r') as file: + final_script = file.read() + self.assertFalse("%EXTENDED_HEADER%" in final_script) + self.assertFalse("%EXTENDED_TAILER%" in final_script) + @patch('autosubmitconfigparser.config.basicconfig.BasicConfig') def test_job_parameters(self, mocked_global_basic_config: Mock): """Test job platforms with a platform. Builds job and platform using YAML data, without mocks. -- GitLab From e3cc9362afd6c83424a8581b95ce98bb4a33dabe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 1 Sep 2023 09:37:13 +0200 Subject: [PATCH 5/9] test if the file does not exist, it throws an exception --- test/unit/test_job.py | 107 +++++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 39 deletions(-) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index ce5500a0e..a37838fbc 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -267,7 +267,7 @@ class TestJob(TestCase): # we will test the substitution on the header, tailer, and in both for extended_position in ["header", "tailer", "header tailer", "neither"]: # we will test every type of the extended script. It should fail in all but the correct script type - for extended_type in ["Bash", "Python", "Rscript", "Bad"]: + for extended_type in ["Bash", "Python", "Rscript", "Bad1", "Bad2", "FileNotFound"]: with tempfile.TemporaryDirectory() as temp_dir: BasicConfig.LOCAL_ROOT_DIR = str(temp_dir) Path(temp_dir, expid).mkdir() @@ -305,17 +305,27 @@ class TestJob(TestCase): {header_content} ''') header_file_name = "header.R" - else: # a badly formed script + elif extended_type == "Bad1": header_content = 'this is a script without #!' full_header_content = dedent(f'''\ {header_content} ''') - header_file_name = "header.bad" + header_file_name = "header.bad1" + elif extended_type == "Bad2": + header_content = 'this is a header with a bath executable' + full_header_content = dedent(f'''\ + #!/does/not/exist + {header_content} + ''') + header_file_name = "header.bad2" + else: # file not found case + header_file_name = "non_existent_header" - # build the header script if we need to - with open(Path(temp_dir, f'{expid}/proj/project_files/{header_file_name}'), 'w+') as header: - header.write(full_header_content) - header.flush() + if extended_type != "FileNotFound": + # build the header script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{header_file_name}'), 'w+') as header: + header.write(full_header_content) + header.flush() if "tailer" in extended_position: if extended_type == "Bash": @@ -339,17 +349,27 @@ class TestJob(TestCase): {tailer_content} ''') tailer_file_name = "tailer.R" - else: # a badly formed script + elif extended_type == "Bad1": tailer_content = 'this is a script without #!' full_tailer_content = dedent(f'''\ {tailer_content} ''') - tailer_file_name = "tailer.bad" + tailer_file_name = "tailer.bad1" + elif extended_type == "Bad2": + tailer_content = 'this is a tailer with a bath executable' + full_tailer_content = dedent(f'''\ + #!/does/not/exist + {tailer_content} + ''') + tailer_file_name = "tailer.bad2" + else: # file not found case + tailer_file_name = "non_existent_tailer" - # build the tailer script if we need to - with open(Path(temp_dir, f'{expid}/proj/project_files/{tailer_file_name}'), 'w+') as tailer: - tailer.write(full_tailer_content) - tailer.flush() + if extended_type != "FileNotFound": + # build the tailer script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{tailer_file_name}'), 'w+') as tailer: + tailer.write(full_tailer_content) + tailer.flush() # configuration file @@ -427,7 +447,6 @@ class TestJob(TestCase): # assert if extended_position != "neither": - # we have either a tailer or header if extended_type == script_type: # load the parameters job.check_script(config, parameters) @@ -439,31 +458,41 @@ class TestJob(TestCase): self.assertTrue(header_content in full_script) if "tailer" in extended_position: self.assertTrue(tailer_content in full_script) - else: # the extended script is not the same as the host - if extended_type == "Bad": # we check if a script without hash bang fails - with self.assertRaises(AutosubmitCritical) as context: - job.check_script(config, parameters) - self.assertEqual(context.exception.code, 7011) - if extended_position == "header tailer" or extended_position == "header": - self.assertEqual(context.exception.message, - f"Extended header script: couldn't figure out script {header_file_name} type\n") - else: - self.assertEqual(context.exception.message, - f"Extended tailer script: couldn't figure out script {tailer_file_name} type\n") - else: # if the script is properly done - # Asserts that an exception is raised if there is a mismatch between job and extension types - with self.assertRaises(AutosubmitCritical) as context: - job.check_script(config, parameters) - self.assertEqual(context.exception.code, 7011) - # if we have both header and tailer, it will fail at the header first - if extended_position == "header tailer" or extended_position == "header": - self.assertEqual(context.exception.message, - f"Extended header script: script {header_file_name} seems " - f"{extended_type} but job zzyy_A.cmd isn't\n") - else: # extended_position == "tailer" - self.assertEqual(context.exception.message, - f"Extended tailer script: script {tailer_file_name} seems " - f"{extended_type} but job zzyy_A.cmd isn't\n") + elif extended_type == "FileNotFound": + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7014) + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{header_file_name}' \n") + else: # extended_position == "tailer": + self.assertEqual(context.exception.message, + f"Extended tailer script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{tailer_file_name}' \n") + elif extended_type == "Bad1" or extended_type == "Bad2": + # we check if a script without hash bang fails or with a bad executable + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: couldn't figure out script {header_file_name} type\n") + else: + self.assertEqual(context.exception.message, + f"Extended tailer script: couldn't figure out script {tailer_file_name} type\n") + else: # extended_type != script_type + # Asserts that an exception is raised if there is a mismatch between job and extension types + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + # if we have both header and tailer, it will fail at the header first + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: script {header_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") + else: # extended_position == "tailer" + self.assertEqual(context.exception.message, + f"Extended tailer script: script {tailer_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") else: # we don't have either a tailer or header # load the parameters job.check_script(config, parameters) -- GitLab From ba253cc8a359c15f309dbeb0a03b442185a3608d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 1 Sep 2023 09:37:37 +0200 Subject: [PATCH 6/9] add extended header and tailer documentation --- docs/source/userguide/configure/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index b0a000bc0..8532531e6 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -147,6 +147,10 @@ There are also other, less used features that you can use: * QUEUE: queue to add the job to. If not specified, uses PLATFORM default. +* EXTENDED_HEADER_PATH: specify the path relative to the project folder where the extension to the autosubmit's header is + +* EXTENDED_TAILER_PATH: specify the path relative to the project folder where the extension to the autosubmit's tailer is + How to configure email notifications ------------------------------------ -- GitLab From ea99210a11a7ba6b695fcd0c9ff32e4f5e85ab67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Fri, 1 Sep 2023 09:42:41 +0200 Subject: [PATCH 7/9] Add dani's check so that it doesnt complain with file not found when proj type is none --- autosubmit/job/job.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index c8b5e99eb..e4f57501c 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1423,8 +1423,9 @@ class Job(object): parameters['HYPERTHREADING'] = self.hyperthreading # we open the files and offload the whole script as a string # memory issues if the script is too long? Add a check to avoid problems... - parameters['EXTENDED_HEADER'] = self.read_header_tailer_script(self.ext_header_path, as_conf, True) - parameters['EXTENDED_TAILER'] = self.read_header_tailer_script(self.ext_tailer_path, as_conf, False) + if as_conf.get_project_type() != "none": + parameters['EXTENDED_HEADER'] = self.read_header_tailer_script(self.ext_header_path, as_conf, True) + parameters['EXTENDED_TAILER'] = self.read_header_tailer_script(self.ext_tailer_path, as_conf, False) parameters['CURRENT_QUEUE'] = self.queue parameters['RESERVATION'] = self.reservation return parameters -- GitLab From 3ba130c536428e489584ea9c0b8ce6c1634a768b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Gim=C3=A9nez=20de=20Castro?= Date: Tue, 31 Oct 2023 10:00:09 +0100 Subject: [PATCH 8/9] Move temp folder to the outside of for loops to reduce file creation. Rewrite the assertion part --- test/unit/test_job.py | 480 +++++++++++++++++++++--------------------- 1 file changed, 244 insertions(+), 236 deletions(-) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index a37838fbc..7fcfc3589 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -262,247 +262,255 @@ class TestJob(TestCase): expid = 'zzyy' - # this is the actual job's type - for script_type in ["Bash", "Python", "Rscript"]: - # we will test the substitution on the header, tailer, and in both - for extended_position in ["header", "tailer", "header tailer", "neither"]: - # we will test every type of the extended script. It should fail in all but the correct script type - for extended_type in ["Bash", "Python", "Rscript", "Bad1", "Bad2", "FileNotFound"]: - with tempfile.TemporaryDirectory() as temp_dir: - BasicConfig.LOCAL_ROOT_DIR = str(temp_dir) - Path(temp_dir, expid).mkdir() - # FIXME: (Copied from Bruno) Not sure why but the submitted and Slurm were using the $expid/tmp/ASLOGS folder? - for path in [f'{expid}/tmp', f'{expid}/tmp/ASLOGS', f'{expid}/tmp/ASLOGS_{expid}', f'{expid}/proj', - f'{expid}/conf', f'{expid}/proj/project_files']: - Path(temp_dir, path).mkdir() - - header_file_name = "" - # this is the part of the script that executes - header_content = "" - tailer_file_name = "" - tailer_content = "" - - # create the extended header and tailer scripts - if "header" in extended_position: - if extended_type == "Bash": - header_content = 'echo "header bash"' - full_header_content = dedent(f'''\ - #!/usr/bin/bash - {header_content} - ''') - header_file_name = "header.sh" - elif extended_type == "Python": - header_content = 'print("header python")' - full_header_content = dedent(f'''\ - #!/usr/bin/python - {header_content} - ''') - header_file_name = "header.py" - elif extended_type == "Rscript": - header_content = 'print("header R")' - full_header_content = dedent(f'''\ - #!/usr/bin/env Rscript - {header_content} - ''') - header_file_name = "header.R" - elif extended_type == "Bad1": - header_content = 'this is a script without #!' - full_header_content = dedent(f'''\ - {header_content} - ''') - header_file_name = "header.bad1" - elif extended_type == "Bad2": - header_content = 'this is a header with a bath executable' - full_header_content = dedent(f'''\ - #!/does/not/exist - {header_content} - ''') - header_file_name = "header.bad2" - else: # file not found case - header_file_name = "non_existent_header" - - if extended_type != "FileNotFound": - # build the header script if we need to - with open(Path(temp_dir, f'{expid}/proj/project_files/{header_file_name}'), 'w+') as header: - header.write(full_header_content) - header.flush() - - if "tailer" in extended_position: - if extended_type == "Bash": - tailer_content = 'echo "tailer bash"' - full_tailer_content = dedent(f'''\ - #!/usr/bin/bash - {tailer_content} - ''') - tailer_file_name = "tailer.sh" - elif extended_type == "Python": - tailer_content = 'print("tailer python")' - full_tailer_content = dedent(f'''\ - #!/usr/bin/python - {tailer_content} - ''') - tailer_file_name = "tailer.py" - elif extended_type == "Rscript": - tailer_content = 'print("header R")' - full_tailer_content = dedent(f'''\ - #!/usr/bin/env Rscript - {tailer_content} - ''') - tailer_file_name = "tailer.R" - elif extended_type == "Bad1": - tailer_content = 'this is a script without #!' - full_tailer_content = dedent(f'''\ - {tailer_content} - ''') - tailer_file_name = "tailer.bad1" - elif extended_type == "Bad2": - tailer_content = 'this is a tailer with a bath executable' - full_tailer_content = dedent(f'''\ - #!/does/not/exist - {tailer_content} - ''') - tailer_file_name = "tailer.bad2" - else: # file not found case - tailer_file_name = "non_existent_tailer" - - if extended_type != "FileNotFound": - # build the tailer script if we need to - with open(Path(temp_dir, f'{expid}/proj/project_files/{tailer_file_name}'), 'w+') as tailer: - tailer.write(full_tailer_content) - tailer.flush() - - # configuration file - - with open(Path(temp_dir, f'{expid}/conf/minimal.yml'), 'w+') as minimal: - minimal.write(dedent(f'''\ - DEFAULT: - EXPID: {expid} - HPCARCH: local - JOBS: - A: - FILE: a - TYPE: {script_type if script_type != "Rscript" else "R"} - PLATFORM: local - RUNNING: once - EXTENDED_HEADER_PATH: {header_file_name} - EXTENDED_TAILER_PATH: {tailer_file_name} - PLATFORMS: - test: - TYPE: slurm - HOST: localhost - PROJECT: abc - QUEUE: debug - USER: me - SCRATCH_DIR: /anything/ - ADD_PROJECT_TO_HOST: False - MAX_WALLCLOCK: '000:55' - TEMP_DIR: '' - ''')) - minimal.flush() - - mocked_basic_config = Mock(spec=BasicConfig) - mocked_basic_config.LOCAL_ROOT_DIR = str(temp_dir) - mocked_global_basic_config.LOCAL_ROOT_DIR.return_value = str(temp_dir) - - config = AutosubmitConfig(expid, basic_config=mocked_basic_config, parser_factory=YAMLParserFactory()) - config.reload(True) - - # act - - parameters = config.load_parameters() - - job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(), - Autosubmit._get_job_list_persistence(expid, config), config) - job_list_obj.generate( - date_list=[], - member_list=[], - num_chunks=1, - chunk_ini=1, - parameters=parameters, - date_format='M', - default_retrials=config.get_retrials(), - default_job_type=config.get_default_job_type(), - wrapper_type=config.get_wrapper_type(), - wrapper_jobs={}, - notransitive=True, - update_structure=True, - run_only_members=config.get_member_list(run_only=True), - jobs_data=config.experiment_data, - as_conf=config - ) - - job_list = job_list_obj.get_job_list() - - submitter = Autosubmit._get_submitter(config) - submitter.load_platforms(config) - - hpcarch = config.get_platform() - for job in job_list: - if job.platform_name == "" or job.platform_name is None: - job.platform_name = hpcarch - job.platform = submitter.platforms[job.platform_name] - - # pick ur single job - job = job_list[0] - - # assert - if extended_position != "neither": - if extended_type == script_type: + with tempfile.TemporaryDirectory() as temp_dir: + Path(temp_dir, expid).mkdir() + # FIXME: (Copied from Bruno) Not sure why but the submitted and Slurm were using the $expid/tmp/ASLOGS folder? + for path in [f'{expid}/tmp', f'{expid}/tmp/ASLOGS', f'{expid}/tmp/ASLOGS_{expid}', f'{expid}/proj', + f'{expid}/conf', f'{expid}/proj/project_files']: + Path(temp_dir, path).mkdir() + # loop over the host script's type + for script_type in ["Bash", "Python", "Rscript"]: + # loop over the position of the extension + for extended_position in ["header", "tailer", "header tailer", "neither"]: + # loop over the extended type + for extended_type in ["Bash", "Python", "Rscript", "Bad1", "Bad2", "FileNotFound"]: + BasicConfig.LOCAL_ROOT_DIR = str(temp_dir) + + header_file_name = "" + # this is the part of the script that executes + header_content = "" + tailer_file_name = "" + tailer_content = "" + + # create the extended header and tailer scripts + if "header" in extended_position: + if extended_type == "Bash": + header_content = 'echo "header bash"' + full_header_content = dedent(f'''\ + #!/usr/bin/bash + {header_content} + ''') + header_file_name = "header.sh" + elif extended_type == "Python": + header_content = 'print("header python")' + full_header_content = dedent(f'''\ + #!/usr/bin/python + {header_content} + ''') + header_file_name = "header.py" + elif extended_type == "Rscript": + header_content = 'print("header R")' + full_header_content = dedent(f'''\ + #!/usr/bin/env Rscript + {header_content} + ''') + header_file_name = "header.R" + elif extended_type == "Bad1": + header_content = 'this is a script without #!' + full_header_content = dedent(f'''\ + {header_content} + ''') + header_file_name = "header.bad1" + elif extended_type == "Bad2": + header_content = 'this is a header with a bath executable' + full_header_content = dedent(f'''\ + #!/does/not/exist + {header_content} + ''') + header_file_name = "header.bad2" + else: # file not found case + header_file_name = "non_existent_header" + + if extended_type != "FileNotFound": + # build the header script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{header_file_name}'), 'w+') as header: + header.write(full_header_content) + header.flush() + else: + # make sure that the file does not exist + for file in os.listdir(Path(temp_dir, f'{expid}/proj/project_files/')): + os.remove(Path(temp_dir, f'{expid}/proj/project_files/{file}')) + + if "tailer" in extended_position: + if extended_type == "Bash": + tailer_content = 'echo "tailer bash"' + full_tailer_content = dedent(f'''\ + #!/usr/bin/bash + {tailer_content} + ''') + tailer_file_name = "tailer.sh" + elif extended_type == "Python": + tailer_content = 'print("tailer python")' + full_tailer_content = dedent(f'''\ + #!/usr/bin/python + {tailer_content} + ''') + tailer_file_name = "tailer.py" + elif extended_type == "Rscript": + tailer_content = 'print("header R")' + full_tailer_content = dedent(f'''\ + #!/usr/bin/env Rscript + {tailer_content} + ''') + tailer_file_name = "tailer.R" + elif extended_type == "Bad1": + tailer_content = 'this is a script without #!' + full_tailer_content = dedent(f'''\ + {tailer_content} + ''') + tailer_file_name = "tailer.bad1" + elif extended_type == "Bad2": + tailer_content = 'this is a tailer with a bath executable' + full_tailer_content = dedent(f'''\ + #!/does/not/exist + {tailer_content} + ''') + tailer_file_name = "tailer.bad2" + else: # file not found case + tailer_file_name = "non_existent_tailer" + + if extended_type != "FileNotFound": + # build the tailer script if we need to + with open(Path(temp_dir, f'{expid}/proj/project_files/{tailer_file_name}'), 'w+') as tailer: + tailer.write(full_tailer_content) + tailer.flush() + else: + # clear the content of the project file + for file in os.listdir(Path(temp_dir, f'{expid}/proj/project_files/')): + os.remove(Path(temp_dir, f'{expid}/proj/project_files/{file}')) + + # configuration file + + with open(Path(temp_dir, f'{expid}/conf/configuration.yml'), 'w+') as configuration: + configuration.write(dedent(f'''\ + DEFAULT: + EXPID: {expid} + HPCARCH: local + JOBS: + A: + FILE: a + TYPE: {script_type if script_type != "Rscript" else "R"} + PLATFORM: local + RUNNING: once + EXTENDED_HEADER_PATH: {header_file_name} + EXTENDED_TAILER_PATH: {tailer_file_name} + PLATFORMS: + test: + TYPE: slurm + HOST: localhost + PROJECT: abc + QUEUE: debug + USER: me + SCRATCH_DIR: /anything/ + ADD_PROJECT_TO_HOST: False + MAX_WALLCLOCK: '00:55' + TEMP_DIR: '' + ''')) + configuration.flush() + + mocked_basic_config = Mock(spec=BasicConfig) + mocked_basic_config.LOCAL_ROOT_DIR = str(temp_dir) + mocked_global_basic_config.LOCAL_ROOT_DIR.return_value = str(temp_dir) + + config = AutosubmitConfig(expid, basic_config=mocked_basic_config, parser_factory=YAMLParserFactory()) + config.reload(True) + + # act + + parameters = config.load_parameters() + + job_list_obj = JobList(expid, mocked_basic_config, YAMLParserFactory(), + Autosubmit._get_job_list_persistence(expid, config), config) + job_list_obj.generate( + date_list=[], + member_list=[], + num_chunks=1, + chunk_ini=1, + parameters=parameters, + date_format='M', + default_retrials=config.get_retrials(), + default_job_type=config.get_default_job_type(), + wrapper_type=config.get_wrapper_type(), + wrapper_jobs={}, + notransitive=True, + update_structure=True, + run_only_members=config.get_member_list(run_only=True), + jobs_data=config.experiment_data, + as_conf=config + ) + + job_list = job_list_obj.get_job_list() + + submitter = Autosubmit._get_submitter(config) + submitter.load_platforms(config) + + hpcarch = config.get_platform() + for job in job_list: + if job.platform_name == "" or job.platform_name is None: + job.platform_name = hpcarch + job.platform = submitter.platforms[job.platform_name] + + # pick ur single job + job = job_list[0] + + if extended_position == "header" or extended_position == "tailer" or extended_position == "header tailer": + if extended_type == script_type: + # load the parameters + job.check_script(config, parameters) + # create the script + job.create_script(config) + with open(Path(temp_dir, f'{expid}/tmp/zzyy_A.cmd'), 'r') as file: + full_script = file.read() + if "header" in extended_position: + self.assertTrue(header_content in full_script) + if "tailer" in extended_position: + self.assertTrue(tailer_content in full_script) + else: # extended_type != script_type + if extended_type == "FileNotFound": + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7014) + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{header_file_name}' \n") + else: # extended_position == "tailer": + self.assertEqual(context.exception.message, + f"Extended tailer script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{tailer_file_name}' \n") + elif extended_type == "Bad1" or extended_type == "Bad2": + # we check if a script without hash bang fails or with a bad executable + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: couldn't figure out script {header_file_name} type\n") + else: + self.assertEqual(context.exception.message, + f"Extended tailer script: couldn't figure out script {tailer_file_name} type\n") + else: # if extended type is any but the script_type and the malformed scripts + with self.assertRaises(AutosubmitCritical) as context: + job.check_script(config, parameters) + self.assertEqual(context.exception.code, 7011) + # if we have both header and tailer, it will fail at the header first + if extended_position == "header tailer" or extended_position == "header": + self.assertEqual(context.exception.message, + f"Extended header script: script {header_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") + else: # extended_position == "tailer" + self.assertEqual(context.exception.message, + f"Extended tailer script: script {tailer_file_name} seems " + f"{extended_type} but job zzyy_A.cmd isn't\n") + else: # extended_position == "neither" + # assert it doesn't exist # load the parameters job.check_script(config, parameters) # create the script job.create_script(config) + # finally, if we don't have scripts, check if the placeholders have been removed with open(Path(temp_dir, f'{expid}/tmp/zzyy_A.cmd'), 'r') as file: - full_script = file.read() - if "header" in extended_position: - self.assertTrue(header_content in full_script) - if "tailer" in extended_position: - self.assertTrue(tailer_content in full_script) - elif extended_type == "FileNotFound": - with self.assertRaises(AutosubmitCritical) as context: - job.check_script(config, parameters) - self.assertEqual(context.exception.code, 7014) - if extended_position == "header tailer" or extended_position == "header": - self.assertEqual(context.exception.message, - f"Extended header script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{header_file_name}' \n") - else: # extended_position == "tailer": - self.assertEqual(context.exception.message, - f"Extended tailer script: failed to fetch [Errno 2] No such file or directory: '{temp_dir}/{expid}/proj/project_files/{tailer_file_name}' \n") - elif extended_type == "Bad1" or extended_type == "Bad2": - # we check if a script without hash bang fails or with a bad executable - with self.assertRaises(AutosubmitCritical) as context: - job.check_script(config, parameters) - self.assertEqual(context.exception.code, 7011) - if extended_position == "header tailer" or extended_position == "header": - self.assertEqual(context.exception.message, - f"Extended header script: couldn't figure out script {header_file_name} type\n") - else: - self.assertEqual(context.exception.message, - f"Extended tailer script: couldn't figure out script {tailer_file_name} type\n") - else: # extended_type != script_type - # Asserts that an exception is raised if there is a mismatch between job and extension types - with self.assertRaises(AutosubmitCritical) as context: - job.check_script(config, parameters) - self.assertEqual(context.exception.code, 7011) - # if we have both header and tailer, it will fail at the header first - if extended_position == "header tailer" or extended_position == "header": - self.assertEqual(context.exception.message, - f"Extended header script: script {header_file_name} seems " - f"{extended_type} but job zzyy_A.cmd isn't\n") - else: # extended_position == "tailer" - self.assertEqual(context.exception.message, - f"Extended tailer script: script {tailer_file_name} seems " - f"{extended_type} but job zzyy_A.cmd isn't\n") - else: # we don't have either a tailer or header - # load the parameters - job.check_script(config, parameters) - # create the script - job.create_script(config) - # finally, if we don't have scripts, check if the placeholders have been removed - with open(Path(temp_dir, f'{expid}/tmp/zzyy_A.cmd'), 'r') as file: - final_script = file.read() - self.assertFalse("%EXTENDED_HEADER%" in final_script) - self.assertFalse("%EXTENDED_TAILER%" in final_script) + final_script = file.read() + self.assertFalse("%EXTENDED_HEADER%" in final_script) + self.assertFalse("%EXTENDED_TAILER%" in final_script) @patch('autosubmitconfigparser.config.basicconfig.BasicConfig') def test_job_parameters(self, mocked_global_basic_config: Mock): -- GitLab From 1887b2a3681d31434ca97b98a80a54aa07d4b08e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 6 Nov 2023 16:06:27 +0100 Subject: [PATCH 9/9] added retrial key --- test/unit/test_job.py | 47 +++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 3e6a09a4e..218da278f 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -383,29 +383,32 @@ class TestJob(TestCase): with open(Path(temp_dir, f'{expid}/conf/configuration.yml'), 'w+') as configuration: configuration.write(dedent(f'''\ - DEFAULT: - EXPID: {expid} - HPCARCH: local - JOBS: - A: - FILE: a - TYPE: {script_type if script_type != "Rscript" else "R"} - PLATFORM: local - RUNNING: once - EXTENDED_HEADER_PATH: {header_file_name} - EXTENDED_TAILER_PATH: {tailer_file_name} - PLATFORMS: - test: - TYPE: slurm - HOST: localhost - PROJECT: abc - QUEUE: debug - USER: me - SCRATCH_DIR: /anything/ - ADD_PROJECT_TO_HOST: False - MAX_WALLCLOCK: '00:55' - TEMP_DIR: '' +DEFAULT: + EXPID: {expid} + HPCARCH: local +JOBS: + A: + FILE: a + TYPE: {script_type if script_type != "Rscript" else "R"} + PLATFORM: local + RUNNING: once + EXTENDED_HEADER_PATH: {header_file_name} + EXTENDED_TAILER_PATH: {tailer_file_name} +PLATFORMS: + test: + TYPE: slurm + HOST: localhost + PROJECT: abc + QUEUE: debug + USER: me + SCRATCH_DIR: /anything/ + ADD_PROJECT_TO_HOST: False + MAX_WALLCLOCK: '00:55' + TEMP_DIR: '' +CONFIG: + RETRIALS: 0 ''')) + configuration.flush() mocked_basic_config = Mock(spec=BasicConfig) -- GitLab