From 69a43bd29fa7bacba007802b0b7e0a35785fb679 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 13 Feb 2023 09:53:58 +0100 Subject: [PATCH 1/7] Additional files fixes --- autosubmit/job/job.py | 10 +++++++--- autosubmit/job/job_packages.py | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 57bdca689..ef2a682aa 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1198,10 +1198,10 @@ class Job(object): def update_content_extra(self,as_conf,files): additional_templates = [] for file in files: - if as_conf.get_project_type().lower() != "none": + if as_conf.get_project_type().lower() == "none": template = "%DEFAULT.EXPID%" else: - template = open(os.path.join(as_conf.get_project_dir(), file), 'r') + template = open(os.path.join(as_conf.get_project_dir(), file), 'r').read() additional_templates += [template] return additional_templates def update_content(self, as_conf): @@ -1336,8 +1336,12 @@ class Job(object): additional_template_content = re.sub('%(? Date: Mon, 13 Feb 2023 10:36:05 +0100 Subject: [PATCH 2/7] submodule --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 76c06bbd6..26eeaaf28 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "autosubmit4-config-parser"] - path = autosubmit/autosubmit4-config-parser + path = autosubmit4-config-parser url = ../../ces/autosubmit4-config-parser -- GitLab From 9215c05c28aabcce8f1acca878102de0f184a9f1 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 13 Feb 2023 12:51:52 +0100 Subject: [PATCH 3/7] docs --- .../userguide/configure/develop_a_project.rst | 104 +++++++++--------- docs/source/userguide/configure/index.rst | 2 +- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/docs/source/userguide/configure/develop_a_project.rst b/docs/source/userguide/configure/develop_a_project.rst index a2aa8ecb6..2e0377b5a 100644 --- a/docs/source/userguide/configure/develop_a_project.rst +++ b/docs/source/userguide/configure/develop_a_project.rst @@ -146,58 +146,58 @@ Jobs configuration JOBS: ## Job name # JOBNAME: - ## Script to execute. If not specified, job will be omitted from workflow. - ## Path relative to the project directory - # FILE: - ## Platform to execute the job. If not specified, defaults to HPCARCH in expedf file. - ## LOCAL is always defined and refers to current machine - # PLATFORM: - ## Queue to add the job to. If not specified, uses PLATFORM default. - # QUEUE: - ## Defines dependencies from job as a list of parents jobs separated by spaces. - ## Dependencies to jobs in previous chunk, member o startdate, use -(DISTANCE) - # DEPENDENCIES:INI SIM-1 CLEAN-2 - ## Define if jobs runs once, once per stardate, once per member or once per chunk. Options: once, date, member, chunk. - ## If not specified, defaults to once - # RUNNING:once - ## Specifies that job has only to be run after X dates, members or chunk. A job will always be created for the last - ## If not specified, defaults to 1 - # FREQUENCY:3 - ## On a job with FREQUENCY > 1, if True, the dependencies are evaluated against all - ## jobs in the frequency interval, otherwise only evaluate dependencies against current - ## iteration. - ## If not specified, defaults to True - # WAIT:False - ## Defines if job is only to be executed in reruns. If not specified, defaults to false. - # RERUN_ONLY:False - ## Wallclock to be submitted to the HPC queue in format HH:MM - # WALLCLOCK:00:05 - - ## Processors number to be submitted to the HPC. If not specified, defaults to 1. - ## Wallclock chunk increase (WALLCLOCK will be increased according to the formula WALLCLOCK + WCHUNKINC * (chunk - 1)). - ## Ideal for sequences of jobs that change their expected running time according to the current chunk. - # WCHUNKINC: 00:01 - # PROCESSORS: 1 - ## Threads number to be submitted to the HPC. If not specified, defaults to 1. - # THREADS: 1 - ## Enables hyper-threading. If not specified, defaults to false. - # HYPERTHREADING: false - ## Tasks number to be submitted to the HPC. If not specified, defaults to 1. - # Tasks: 1 - ## Memory requirements for the job in MB - # MEMORY: 4096 - ## Number of retrials if a job fails. If not specified, defaults to the value given on experiment's autosubmit.yml - # RETRIALS: 4 - ## Allows to put a delay between retries, of retrials if a job fails. If not specified, it will be static - # DELAY_RETRY_TIME: 11 - # DELAY_RETRY_TIME: +11 # will wait 11,22,33,44... - # DELAY_RETRY_TIME: *11 # will wait 11,110,1110,11110... - ## Some jobs can not be checked before running previous jobs. Set this option to false if that is the case - # CHECK: False - ## Select the interpreter that will run the job. Options: bash, python, r Default: bash - # TYPE: bash - ## Specify the path to the interpreter. If empty, use system default based on job type . Default: empty - # EXECUTABLE: /my_python_env/python3 + ## Script to execute. If not specified, job will be omitted from workflow. You can also specify non-executable additional files. + ## Path relative to the project directory + # FILE: + ## Platform to execute the job. If not specified, defaults to HPCARCH in expedf file. + ## LOCAL is always defined and refers to current machine + # PLATFORM: + ## Queue to add the job to. If not specified, uses PLATFORM default. + # QUEUE: + ## Defines dependencies from job as a list of parents jobs separated by spaces. + ## Dependencies to jobs in previous chunk, member o startdate, use -(DISTANCE) + # DEPENDENCIES:INI SIM-1 CLEAN-2 + ## Define if jobs runs once, once per stardate, once per member or once per chunk. Options: once, date, member, chunk. + ## If not specified, defaults to once + # RUNNING:once + ## Specifies that job has only to be run after X dates, members or chunk. A job will always be created for the last + ## If not specified, defaults to 1 + # FREQUENCY:3 + ## On a job with FREQUENCY > 1, if True, the dependencies are evaluated against all + ## jobs in the frequency interval, otherwise only evaluate dependencies against current + ## iteration. + ## If not specified, defaults to True + # WAIT:False + ## Defines if job is only to be executed in reruns. If not specified, defaults to false. + # RERUN_ONLY:False + ## Wallclock to be submitted to the HPC queue in format HH:MM + # WALLCLOCK:00:05 + + ## Processors number to be submitted to the HPC. If not specified, defaults to 1. + ## Wallclock chunk increase (WALLCLOCK will be increased according to the formula WALLCLOCK + WCHUNKINC * (chunk - 1)). + ## Ideal for sequences of jobs that change their expected running time according to the current chunk. + # WCHUNKINC: 00:01 + # PROCESSORS: 1 + ## Threads number to be submitted to the HPC. If not specified, defaults to 1. + # THREADS: 1 + ## Enables hyper-threading. If not specified, defaults to false. + # HYPERTHREADING: false + ## Tasks number to be submitted to the HPC. If not specified, defaults to 1. + # Tasks: 1 + ## Memory requirements for the job in MB + # MEMORY: 4096 + ## Number of retrials if a job fails. If not specified, defaults to the value given on experiment's autosubmit.yml + # RETRIALS: 4 + ## Allows to put a delay between retries, of retrials if a job fails. If not specified, it will be static + # DELAY_RETRY_TIME: 11 + # DELAY_RETRY_TIME: +11 # will wait 11,22,33,44... + # DELAY_RETRY_TIME: *11 # will wait 11,110,1110,11110... + ## Some jobs can not be checked before running previous jobs. Set this option to false if that is the case + # CHECK: False + ## Select the interpreter that will run the job. Options: bash, python, r Default: bash + # TYPE: bash + ## Specify the path to the interpreter. If empty, use system default based on job type . Default: empty + # EXECUTABLE: /my_python_env/python3 LOCAL_SETUP: diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index ad6414e92..78badef6c 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -344,7 +344,7 @@ In the file: ## Job name # JOBNAME: - ## Script to execute. If not specified, job will be omitted from workflow. + ## Script to execute. If not specified, job will be omitted from workflow. You can also specify additional files separated by a ",". ## Path relative to the project directory # FILE : ## Platform to execute the job. If not specified, defaults to HPCARCH in expedf file. -- GitLab From 2e8f6129dd288c7957f2d67f5a21914020c4697c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 16 Feb 2023 09:52:09 +0100 Subject: [PATCH 4/7] adressed the changes --- autosubmit/job/job.py | 4 ++-- autosubmit/job/job_packages.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index ef2a682aa..264e5d2c7 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1328,7 +1328,7 @@ class Job(object): #enumerate and get value #todo revise pipeline that check this, additional templates value is not Mocked well - for file_n,additional_template_content in enumerate(additional_templates): + for additional_file, additional_template_content in zip(self.additional_files, additional_templates): template_content += additional_template_content for key, value in parameters.items(): additional_template_content = re.sub('%(? Date: Thu, 16 Feb 2023 10:00:50 +0100 Subject: [PATCH 5/7] forgot len() --- autosubmit/job/job_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index 01a2e0b80..79275cc1f 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -220,7 +220,7 @@ class JobPackageSimple(JobPackageBase): def _send_files(self): for job in self.jobs: self.platform.send_file(self._job_scripts[job.name]) - for file_n in range(job.additional_files): + for file_n in range(len(job.additional_files)): filename = os.path.basename(os.path.splitext(job.additional_files[file_n])[0]) full_path = os.path.join(self._tmp_path,filename ) + "_" + job.name[5:] self.platform.send_file(os.path.join(self._tmp_path, full_path)) -- GitLab From 9ca1673d01458552033c7802edc55b3d36364f47 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 16 Feb 2023 11:55:39 +0100 Subject: [PATCH 6/7] Fixed additional_content in template_content --- autosubmit/job/job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 264e5d2c7..25cd4b151 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1329,7 +1329,6 @@ class Job(object): #todo revise pipeline that check this, additional templates value is not Mocked well for additional_file, additional_template_content in zip(self.additional_files, additional_templates): - template_content += additional_template_content for key, value in parameters.items(): additional_template_content = re.sub('%(? Date: Thu, 16 Feb 2023 16:26:20 +0100 Subject: [PATCH 7/7] doc update --- docs/source/userguide/configure/develop_a_project.rst | 3 ++- docs/source/userguide/configure/index.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/userguide/configure/develop_a_project.rst b/docs/source/userguide/configure/develop_a_project.rst index 2e0377b5a..f2b92e6b3 100644 --- a/docs/source/userguide/configure/develop_a_project.rst +++ b/docs/source/userguide/configure/develop_a_project.rst @@ -146,7 +146,8 @@ Jobs configuration JOBS: ## Job name # JOBNAME: - ## Script to execute. If not specified, job will be omitted from workflow. You can also specify non-executable additional files. + ## Script to execute. If not specified, job will be omitted from workflow. "You can also specify additional files separated by a ",". + # Note: The post-processed additional_files will be sent to %HPCROOT%/LOG_%EXPID% ## Path relative to the project directory # FILE: ## Platform to execute the job. If not specified, defaults to HPCARCH in expedf file. diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index 78badef6c..5e0ea58b4 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -345,6 +345,7 @@ In the file: ## Job name # JOBNAME: ## Script to execute. If not specified, job will be omitted from workflow. You can also specify additional files separated by a ",". + # Note: The post processed additional_files will be sent to %HPCROOT%/LOG_%EXPID% ## Path relative to the project directory # FILE : ## Platform to execute the job. If not specified, defaults to HPCARCH in expedf file. -- GitLab