From b7b5cceeb5633bf0628ea10ab8d45a9df826bf35 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 14 Nov 2024 14:58:20 +0100 Subject: [PATCH 1/6] Added changelog --- CHANGELOG | 60 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6cb3b3b44..38da77d4c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,17 +1,57 @@ -4.1.11 - Not released -===================== +4.1.11 - Enhancements, New Features, Documentation, and Bug Fixes +================================================================= -- #1408: Updated our Docker image to work with Kubernetes. - Included sample helm charts (tested with minikube). -- #1373: Update installation instructions with `rsync`, and fix - our `Dockerfile` adding `rsync` and `subversion`. -- #1207: Add experiment path in `autosubmit describe`. +Enhancements and new features: + +- #1444: Additional files now support YAML format. - #1397: Use `tini` as entrypoint in our Docker image. -- #1428: Use natural sort order for graph entries (adapted from Cylc), - enable doctests to be executed with pytest. +- #1207: Add experiment path in `autosubmit describe`. +- #1130: Now `autosubmit refresh` can clone without submodules. +- #1320: Pytest added to the CI/CD. - #945: Fix portalocker releasing the lock when a portalocker exception is raised. Now it prints the warnings but only releases the lock when - the command finished (successfully or not). + the command finishes (successfully or not). +- #1428: Use natural sort order for graph entries (adapted from Cylc), + enable doctests to be executed with pytest. +- #1408: Updated our Docker image to work with Kubernetes. + Included sample helm charts (tested with minikube). +- #1338: Stats can now be visualized in a PDF format. +- #1337: Autosubmit inspect, and other commands, are now faster. + +Documentation and log improvements: +- #1274: A traceability section has been added. +- #1273: Improved AS logs by removing deprecated messages. +- #1394: Added Easybuild recipes for Autosubmit. +- #1439: Autosubmitconfigparser and ruamel.yaml updated. +- #1400: Fixes an issue with monitor and check experiment RTD section. +- #1382: Improved AS logs, removing paramiko warnings. (Updated paramiko version) +- #1373: Update installation instructions with `rsync`, and fix + our `Dockerfile` by adding `rsync` and `subversion`. +- #1242: Improved warnings when using extended header/tailer. +- #1431: Updated the YAML files to use marenostrum5 instead of marenostrum4. + +Bug fixes: +- #1423, #1421, and #1419: Fixes different issues with the split feature. +- #1407: Solves an issue when using one node. +- #1406: Autosubmit monitor is now able to monitor non-owned experiments again. +- #1317: Autosubmit delete now deletes the metadata and database entries. +- #1105: Autosubmit configure now admits paths that end with "/". +- #1045: Autosubmit now admits placeholders set in lists. +- #1417, #1398, #1287, and #1386: Fixes and improves the wrapper deadlock detection. +- #1436: Dependencies with Status=Running not working properly. +- (also enhancement) #1426: Fixes an issue with as_checkpoints. +- #1393: Better support for boolean YAML configurations. +- #1129: (Custom config) Platforms can now be defined under $expid/conf. +- #1443: Fixes an issue with additional files under bscearth000. + +Others: + +- #1427: Readthedocs works again. +- #1376: Autosubmit 4.1.9 was published to DockerHub. +- #1327: LSF platform has been removed. +- #1322: Autosubmit now has a DockerHub organization. +- #1123: Profiler can now be stopped. + 4.1.10 - Hotfix =============== -- GitLab From a80fba36ab7b03457b324db6e47f905146e0289d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 14 Nov 2024 14:59:02 +0100 Subject: [PATCH 2/6] Update Version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 5d30083e9..152e4522c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.10 +4.1.11 -- GitLab From da24dd81ab5e4883ef88b5e1bb1d9bc678b4a070 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 14 Nov 2024 15:11:19 +0100 Subject: [PATCH 3/6] Update configparser version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90ea5a1ad..91f46b715 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ install_requires = [ 'py3dotplus==1.1.0', 'numpy<2', 'rocrate==0.*', - 'autosubmitconfigparser==1.0.72', + 'autosubmitconfigparser==1.0.73', 'configparser', 'setproctitle', # 'sqlalchemy[mypy]' # TODO: pending Postgres MR -- GitLab From 80cb0f7b034f2c578cd8b70534a82816547ce982 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 18 Nov 2024 09:23:28 +0100 Subject: [PATCH 4/6] Fix when platform doesn't exist --- autosubmit/autosubmit.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index f70000216..733d35328 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4710,6 +4710,9 @@ class Autosubmit: except AutosubmitCritical as e: Log.warning(f"{e.message}\nRemote git cloning is disabled") hpcarch = "local" + except KeyError: + Log.warning(f"Platform {as_conf.get_platform()} not found in configuration file") + hpcarch = "local" return AutosubmitGit.clone_repository(as_conf, force, hpcarch) elif project_type == "svn": svn_project_url = as_conf.get_svn_project_url() @@ -6121,7 +6124,3 @@ class Autosubmit: job.status = Status.KEY_TO_VALUE[status] job_list.save() terminate_child_process(expid) - - - - -- GitLab From 7d3e07d6707b29597c8b5af7b65d0d5bdf07c6ba Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 18 Nov 2024 09:58:00 +0100 Subject: [PATCH 5/6] Now all nodes are added into the graph before adding dependencies --- autosubmit/job/job_list.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index a3d102cb2..d373fb6b2 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -417,8 +417,15 @@ class JobList(object): else: changes = True Log.debug("Changes detected, calculating dependencies") - sections_gen = (section for section in jobs_data.keys()) - for job_section in sections_gen: + # Generate all graph before adding dependencies. + for job_section in (section for section in jobs_data.keys()): + for job in (job for job in dic_jobs.get_jobs(job_section, sort_string=True)): + if job.name not in self.graph.nodes: + self.graph.add_node(job.name, job=job) + elif job.name in self.graph.nodes and self.graph.nodes.get(job.name).get("job", None) is None: # Old versions of autosubmit needs re-adding the job to the graph + self.graph.nodes.get(job.name)["job"] = job + + for job_section in (section for section in jobs_data.keys()): # Changes when all jobs of a section are added self.depends_on_previous_chunk = dict() self.depends_on_previous_split = dict() @@ -432,14 +439,8 @@ class JobList(object): # call function if dependencies_key is not None dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs) if dependencies_keys else {} self.job_names = set() - - jobs_gen = (job for job in dic_jobs.get_jobs(job_section,sort_string=True)) - for job in jobs_gen: + for job in (job for job in dic_jobs.get_jobs(job_section, sort_string=True)): self.actual_job_depends_on_special_chunk = False - if job.name not in self.graph.nodes: - self.graph.add_node(job.name, job=job) - elif job.name in self.graph.nodes and self.graph.nodes.get(job.name).get("job", None) is None: # Old versions of autosubmit needs re-adding the job to the graph - self.graph.nodes.get(job.name)["job"] = job if dependencies and changes: job = self.graph.nodes.get(job.name)['job'] ## Adds the dependencies to the job, and if not possible, adds the job to the problematic_dependencies -- GitLab From 6c125e19601225501836e9b17e3ba8bccb908856 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 18 Nov 2024 10:45:09 +0100 Subject: [PATCH 6/6] added invoke dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 91f46b715..230582c20 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ install_requires = [ 'autosubmitconfigparser==1.0.73', 'configparser', 'setproctitle', + 'invoke>=2.0', # 'sqlalchemy[mypy]' # TODO: pending Postgres MR ] -- GitLab