From 854e788a1ba4fc5001073958f34fbbe6d8ff947d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 20 Jan 2023 09:17:58 +0100 Subject: [PATCH 1/6] wip placeholder changes --- autosubmit/job/job.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 495288ee3..0ef5b6798 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -993,6 +993,8 @@ class Job(object): """ chunk = 1 as_conf.reload() + parameters = as_conf.sustitute_dynamic_variables(parameters,25) + parameters = parameters.copy() parameters.update(default_parameters) parameters['JOBNAME'] = self.name @@ -1188,24 +1190,9 @@ class Job(object): parameters['EXPORT'] = self.export parameters['PROJECT_TYPE'] = as_conf.get_project_type() - substituted = False - max_deep = 25 - dynamic_variables = [] - backup_variables = as_conf.dynamic_variables - while len(as_conf.dynamic_variables) > 0 and max_deep > 0: - dynamic_variables = [] - for dynamic_var in as_conf.dynamic_variables: - substituted,new_param = as_conf.sustitute_placeholder_variables(dynamic_var[0],dynamic_var[1],parameters) - if not substituted: - dynamic_variables.append(dynamic_var) - else: - parameters= new_param - as_conf.dynamic_variables = dynamic_variables - max_deep = max_deep - 1 - as_conf.dynamic_variables = backup_variables - self.parameters = parameters + self.parameters = as_conf.sustitute_dynamic_variables(parameters,25) - return parameters + return self.parameters def update_content_extra(self,as_conf,files): additional_templates = [] for file in files: -- GitLab From 67840afa54f79549ab993c1b3deaabdfa0a375d5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 30 Jan 2023 17:09:56 +0100 Subject: [PATCH 2/6] placeholders working, need to clean some code --- autosubmit/job/job.py | 46 +++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 0ef5b6798..c8beed9dd 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1017,10 +1017,10 @@ class Job(object): chunk = self.chunk parameters['CHUNK'] = chunk - total_chunk = int(parameters.get('EXPERIMENT.NUMCHUNKS')) - chunk_length = int(parameters['EXPERIMENT.CHUNKSIZE']) - chunk_unit = str(parameters['EXPERIMENT.CHUNKSIZEUNIT']).lower() - cal = str(parameters['EXPERIMENT.CALENDAR']).lower() + total_chunk = int(parameters.get('EXPERIMENT.NUMCHUNKS',1)) + chunk_length = int(parameters.get('EXPERIMENT.CHUNKSIZE',1)) + chunk_unit = str(parameters.get('EXPERIMENT.CHUNKSIZEUNIT',"")).lower() + cal = str(parameters.get('EXPERIMENT.CALENDAR',"")).lower() chunk_start = chunk_start_date( self.date, chunk, chunk_length, chunk_unit, cal) chunk_end = chunk_end_date( @@ -1035,36 +1035,30 @@ class Job(object): parameters['RUN_DAYS'] = str( subs_dates(chunk_start, chunk_end, cal)) - parameters['Chunk_End_IN_DAYS'] = str( + parameters['CHUNK_END_IN_DAYS'] = str( subs_dates(self.date, chunk_end, cal)) - #parameters['Chunk_START_DATE'] = date2str( - # chunk_start, self.date_format) - #parameters['Chunk_START_YEAR'] = str(chunk_start.year) - #parameters['Chunk_START_MONTH'] = str(chunk_start.month).zfill(2) - #parameters['Chunk_START_DAY'] = str(chunk_start.day).zfill(2) - #parameters['Chunk_START_HOUR'] = str(chunk_start.hour).zfill(2) - parameters['Chunk_START_DATE'] = date2str( + parameters['CHUNK_START_DATE'] = date2str( chunk_start, self.date_format) - parameters['Chunk_START_YEAR'] = str(chunk_start.year) - parameters['Chunk_START_MONTH'] = str(chunk_start.month).zfill(2) - parameters['Chunk_START_DAY'] = str(chunk_start.day).zfill(2) - parameters['Chunk_START_HOUR'] = str(chunk_start.hour).zfill(2) + parameters['CHUNK_START_YEAR'] = str(chunk_start.year) + parameters['CHUNK_START_MONTH'] = str(chunk_start.month).zfill(2) + parameters['CHUNK_START_DAY'] = str(chunk_start.day).zfill(2) + parameters['CHUNK_START_HOUR'] = str(chunk_start.hour).zfill(2) - parameters['Chunk_SECOND_TO_LAST_DATE'] = date2str( + parameters['CHUNK_SECOND_TO_LAST_DATE'] = date2str( chunk_end_1, self.date_format) - parameters['Chunk_SECOND_TO_LAST_YEAR'] = str(chunk_end_1.year) - parameters['Chunk_SECOND_TO_LAST_MONTH'] = str(chunk_end_1.month).zfill(2) - parameters['Chunk_SECOND_TO_LAST_DAY'] = str(chunk_end_1.day).zfill(2) - parameters['Chunk_SECOND_TO_LAST_HOUR'] = str(chunk_end_1.hour).zfill(2) + parameters['CHUNK_SECOND_TO_LAST_YEAR'] = str(chunk_end_1.year) + parameters['CHUNK_SECOND_TO_LAST_MONTH'] = str(chunk_end_1.month).zfill(2) + parameters['CHUNK_SECOND_TO_LAST_DAY'] = str(chunk_end_1.day).zfill(2) + parameters['CHUNK_SECOND_TO_LAST_HOUR'] = str(chunk_end_1.hour).zfill(2) - parameters['Chunk_END_DATE'] = date2str( + parameters['CHUNK_END_DATE'] = date2str( chunk_end_1, self.date_format) - parameters['Chunk_END_YEAR'] = str(chunk_end.year) - parameters['Chunk_END_MONTH'] = str(chunk_end.month).zfill(2) - parameters['Chunk_END_DAY'] = str(chunk_end.day).zfill(2) - parameters['Chunk_END_HOUR'] = str(chunk_end.hour).zfill(2) + parameters['CHUNK_END_YEAR'] = str(chunk_end.year) + parameters['CHUNK_END_MONTH'] = str(chunk_end.month).zfill(2) + parameters['CHUNK_END_DAY'] = str(chunk_end.day).zfill(2) + parameters['CHUNK_END_HOUR'] = str(chunk_end.hour).zfill(2) parameters['PREV'] = str(subs_dates(self.date, chunk_start, cal)) -- GitLab From 90492d064e0d3b213a913e01d661be3b169704ba Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 31 Jan 2023 10:24:15 +0100 Subject: [PATCH 3/6] placeholders supported, req configparser 0.75 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 44481bf04..47ddd81a2 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=['packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib==3.4.3','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl==1.4.0','Pygments','autosubmitconfigparser==0.0.74'], + install_requires=['packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib==3.4.3','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl==1.4.0','Pygments','autosubmitconfigparser==0.0.75'], classifiers=[ "Programming Language :: Python :: 3.7", "License :: OSI Approved :: GNU General Public License (GPL)", -- GitLab From acead2025daf3daf9b85b48ef15173ea723846b6 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 31 Jan 2023 11:24:30 +0100 Subject: [PATCH 4/6] Changed few code as the substitution is now taking place in another step --- autosubmit/job/job.py | 9 +++++---- autosubmit/job/job_common.py | 1 + test/unit/test_job.py | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index c8beed9dd..bfdadc95d 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -993,7 +993,7 @@ class Job(object): """ chunk = 1 as_conf.reload() - parameters = as_conf.sustitute_dynamic_variables(parameters,25) + #parameters = as_conf.sustitute_dynamic_variables(parameters,25) parameters = parameters.copy() parameters.update(default_parameters) @@ -1168,6 +1168,7 @@ class Job(object): parameters[wrapper_section+"_EXTENSIBLE"] = int(as_conf.get_extensible_wallclock(as_conf.experiment_data["WRAPPERS"].get(wrapper_section))) self.dependencies = parameters['DEPENDENCIES'] + # This shouldn't be necesary anymore as now all sub is done in the as_conf.reload() if len(self.export) > 0: variables = re.findall('%(? 0: @@ -1184,9 +1185,9 @@ class Job(object): parameters['EXPORT'] = self.export parameters['PROJECT_TYPE'] = as_conf.get_project_type() - self.parameters = as_conf.sustitute_dynamic_variables(parameters,25) - - return self.parameters + # For some reason, there is return but the assigne is also neccesary + self.parameters = parameters + return parameters def update_content_extra(self,as_conf,files): additional_templates = [] for file in files: diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index 16b9b473a..bc87eaaf3 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -347,6 +347,7 @@ def increase_wallclock_by_chunk(current, increase, chunk): :return: HH:MM wallclock :rtype: str """ + # Pipeline is not testing this since mock is not well made try: if current and increase and chunk and chunk > 0: wallclock = current.split(":") diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 091386753..9a0cdddbf 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -166,8 +166,9 @@ class TestJob(TestCase): self.job.parameters['NUMTASK'] = 666 self.job._tmp_path = '/dummy/tmp/path' + self.job.additional_files = '/dummy/tmp/path_additional_file' - update_content_mock = Mock(return_value=('some-content: %NUMPROC%, %NUMTHREADS%, %NUMTASK% %% %%','some-content: %NUMPROC%, %NUMTHREADS%, %NUMTASK% %% %%')) + update_content_mock = Mock(return_value=('some-content: %NUMPROC%, %NUMTHREADS%, %NUMTASK% %% %%',['some-content: %NUMPROC%, %NUMTHREADS%, %NUMTASK% %% %%'])) self.job.update_content = update_content_mock config = Mock(spec=AutosubmitConfig) -- GitLab From f9b3cac382087f92fd1b6a2177e99a9cac486a7f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 31 Jan 2023 11:47:26 +0100 Subject: [PATCH 5/6] Fix an issue with create not ignoring errors like in the past --- autosubmit/autosubmit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d852e8fdd..bd2a8d303 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4436,14 +4436,17 @@ class Autosubmit: "Preparing .lock file to avoid multiple instances with same expid.") as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(True) - + # Get original configuration + as_conf.check_conf_files(False) project_type = as_conf.get_project_type() # Getting output type provided by the user in config, 'pdf' as default output_type = as_conf.get_output_type() if not Autosubmit._copy_code(as_conf, expid, project_type, False): return False + # Update configuration with the new config in the dist ( if any ) + as_conf.check_conf_files(False) + if not os.path.exists(os.path.join(exp_path, "pkl")): raise AutosubmitCritical( "The pkl folder doesn't exists. Make sure that the 'pkl' folder exists in the following path: {}".format( -- GitLab From c0b492390629f3b21b55971531b42ef8c252142c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 31 Jan 2023 13:11:50 +0100 Subject: [PATCH 6/6] Placeholders in build_job fix --- autosubmit/job/job_dict.py | 6 +++++- bin/autosubmit | 4 ++-- test/README_PIP.md | 12 ++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 test/README_PIP.md diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index 85b81bbc9..5ef1d7177 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -379,7 +379,11 @@ class DicJobs: elif job_type == 'r': job.type = Type.R job.executable = str(parameters[section].get( "EXECUTABLE", "")) - job.platform_name = str(parameters[section].get( "PLATFORM", "")).upper() + hpcarch = self.experiment_data.get("DEFAULT",{}) + hpcarch = hpcarch.get("HPCARCH","") + job.platform_name = str(parameters[section].get("PLATFORM", hpcarch)).upper() + if self.experiment_data["PLATFORMS"].get(job.platform_name, "") == "" and job.platform_name.upper() != "LOCAL": + raise AutosubmitCritical("Platform does not exists, check the value of %JOBS.{0}.PLATFORM% = {1} parameter".format(job.section,job.platform_name),7000,"List of platforms: {0} ".format(self.experiment_data["PLATFORMS"].keys()) ) job.file = str(parameters[section].get( "FILE", "")) job.additional_files = parameters[section].get( "ADDITIONAL_FILES", []) diff --git a/bin/autosubmit b/bin/autosubmit index b3d084120..bc18d5431 100755 --- a/bin/autosubmit +++ b/bin/autosubmit @@ -49,8 +49,8 @@ def main(): except AutosubmitCritical as e: if os.path.exists(os.path.join(Log.file_path, "autosubmit.lock")): os.remove(os.path.join(Log.file_path, "autosubmit.lock")) - Log.error(str(traceback.print_exc())) - if e.trace is not None: + str(traceback.print_exc()) + if e.trace is not None or e.trace == "": Log.error("Trace: {0}", e.trace) Log.critical("{1} [eCode={0}]", e.code, e.message) diff --git a/test/README_PIP.md b/test/README_PIP.md new file mode 100644 index 000000000..3fce62d72 --- /dev/null +++ b/test/README_PIP.md @@ -0,0 +1,12 @@ + +Autosubmit is a lightweight workflow manager designed to meet climate research necessities. Unlike other workflow solutions in the domain, it integrates the capabilities of an experiment manager, workflow orchestrator and monitor in a self-contained application. The experiment manager allows for defining and configuring experiments, supported by a hierarchical database that ensures reproducibility and traceability. The orchestrator is designed to run complex workflows in research and operational mode by managing their dependencies and interfacing with local and remote hosts. These multi-scale workflows can involve from a few to thousands of steps and from one to multiple platforms. + +Autosubmit facilitates easy and fast integration and relocation on new platforms. On the one hand, users can rapidly execute general scripts and progressively parametrize them by reading Autosubmit variables. On the other hand, it is a self-contained desktop application capable of submitting jobs to remote platforms without any external deployment. + +Due to its robustness, it can handle different eventualities, such as networking or I/O errors. Finally, the monitoring capabilities extend beyond the desktop application through a REST API that allows communication with workflow monitoring tools such as the Autosubmit web GUI. + +Autosubmit is a Python package provided in PyPI. Conda recipes can also be found on the website. A containerized version for testing purposes is also available but not public yet. + +It has contributed to various European research projects and runs different operational systems. During the following years, it will support some of the Earth Digital Twins as the Digital Twin Ocean. + +Concretely, it is currently used at Barcelona Supercomputing Centre (BSC) to run models (EC-Earth, MONARCH, NEMO, CALIOPE, HERMES...), operational toolchains (S2S4E), data-download workflows (ECMWF MARS), and many other. Autosubmit has run these workflows in different supercomputers in BSC, ECMWF, IC3, CESGA, EPCC, PDC, and OLCF. \ No newline at end of file -- GitLab