diff --git a/CHANGELOG b/CHANGELOG index 6cb3b3b44838633da973f910248457e7d11ea8b2..38da77d4cc0a998579835b79d2a446cfc156b89d 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,17 +1,57 @@ -4.1.11 - Not released -===================== +4.1.11 - Enhancements, New Features, Documentation, and Bug Fixes +================================================================= -- #1408: Updated our Docker image to work with Kubernetes. - Included sample helm charts (tested with minikube). -- #1373: Update installation instructions with `rsync`, and fix - our `Dockerfile` adding `rsync` and `subversion`. -- #1207: Add experiment path in `autosubmit describe`. +Enhancements and new features: + +- #1444: Additional files now support YAML format. - #1397: Use `tini` as entrypoint in our Docker image. -- #1428: Use natural sort order for graph entries (adapted from Cylc), - enable doctests to be executed with pytest. +- #1207: Add experiment path in `autosubmit describe`. +- #1130: Now `autosubmit refresh` can clone without submodules. +- #1320: Pytest added to the CI/CD. - #945: Fix portalocker releasing the lock when a portalocker exception is raised. Now it prints the warnings but only releases the lock when - the command finished (successfully or not). + the command finishes (successfully or not). +- #1428: Use natural sort order for graph entries (adapted from Cylc), + enable doctests to be executed with pytest. +- #1408: Updated our Docker image to work with Kubernetes. + Included sample helm charts (tested with minikube). +- #1338: Stats can now be visualized in a PDF format. +- #1337: Autosubmit inspect, and other commands, are now faster. + +Documentation and log improvements: +- #1274: A traceability section has been added. +- #1273: Improved AS logs by removing deprecated messages. +- #1394: Added Easybuild recipes for Autosubmit. +- #1439: Autosubmitconfigparser and ruamel.yaml updated. +- #1400: Fixes an issue with monitor and check experiment RTD section. +- #1382: Improved AS logs, removing paramiko warnings. (Updated paramiko version) +- #1373: Update installation instructions with `rsync`, and fix + our `Dockerfile` by adding `rsync` and `subversion`. +- #1242: Improved warnings when using extended header/tailer. +- #1431: Updated the YAML files to use marenostrum5 instead of marenostrum4. + +Bug fixes: +- #1423, #1421, and #1419: Fixes different issues with the split feature. +- #1407: Solves an issue when using one node. +- #1406: Autosubmit monitor is now able to monitor non-owned experiments again. +- #1317: Autosubmit delete now deletes the metadata and database entries. +- #1105: Autosubmit configure now admits paths that end with "/". +- #1045: Autosubmit now admits placeholders set in lists. +- #1417, #1398, #1287, and #1386: Fixes and improves the wrapper deadlock detection. +- #1436: Dependencies with Status=Running not working properly. +- (also enhancement) #1426: Fixes an issue with as_checkpoints. +- #1393: Better support for boolean YAML configurations. +- #1129: (Custom config) Platforms can now be defined under $expid/conf. +- #1443: Fixes an issue with additional files under bscearth000. + +Others: + +- #1427: Readthedocs works again. +- #1376: Autosubmit 4.1.9 was published to DockerHub. +- #1327: LSF platform has been removed. +- #1322: Autosubmit now has a DockerHub organization. +- #1123: Profiler can now be stopped. + 4.1.10 - Hotfix =============== diff --git a/VERSION b/VERSION index 5d30083e95d9e90fe531994a0a63f7e5a92c4f1f..152e4522ca7dd6b968334e3822eaf386d418fd66 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.10 +4.1.11 diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index f70000216032b97329e88464fa5054a15fbdc6ae..733d35328b9cdb0b0ba1109923dfb2ba62260b11 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4710,6 +4710,9 @@ class Autosubmit: except AutosubmitCritical as e: Log.warning(f"{e.message}\nRemote git cloning is disabled") hpcarch = "local" + except KeyError: + Log.warning(f"Platform {as_conf.get_platform()} not found in configuration file") + hpcarch = "local" return AutosubmitGit.clone_repository(as_conf, force, hpcarch) elif project_type == "svn": svn_project_url = as_conf.get_svn_project_url() @@ -6121,7 +6124,3 @@ class Autosubmit: job.status = Status.KEY_TO_VALUE[status] job_list.save() terminate_child_process(expid) - - - - diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index a3d102cb2003e196c2beb83c739887651771cb99..d373fb6b2861aa081ba793c610e84de3a4114bc6 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -417,8 +417,15 @@ class JobList(object): else: changes = True Log.debug("Changes detected, calculating dependencies") - sections_gen = (section for section in jobs_data.keys()) - for job_section in sections_gen: + # Generate all graph before adding dependencies. + for job_section in (section for section in jobs_data.keys()): + for job in (job for job in dic_jobs.get_jobs(job_section, sort_string=True)): + if job.name not in self.graph.nodes: + self.graph.add_node(job.name, job=job) + elif job.name in self.graph.nodes and self.graph.nodes.get(job.name).get("job", None) is None: # Old versions of autosubmit needs re-adding the job to the graph + self.graph.nodes.get(job.name)["job"] = job + + for job_section in (section for section in jobs_data.keys()): # Changes when all jobs of a section are added self.depends_on_previous_chunk = dict() self.depends_on_previous_split = dict() @@ -432,14 +439,8 @@ class JobList(object): # call function if dependencies_key is not None dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs) if dependencies_keys else {} self.job_names = set() - - jobs_gen = (job for job in dic_jobs.get_jobs(job_section,sort_string=True)) - for job in jobs_gen: + for job in (job for job in dic_jobs.get_jobs(job_section, sort_string=True)): self.actual_job_depends_on_special_chunk = False - if job.name not in self.graph.nodes: - self.graph.add_node(job.name, job=job) - elif job.name in self.graph.nodes and self.graph.nodes.get(job.name).get("job", None) is None: # Old versions of autosubmit needs re-adding the job to the graph - self.graph.nodes.get(job.name)["job"] = job if dependencies and changes: job = self.graph.nodes.get(job.name)['job'] ## Adds the dependencies to the job, and if not possible, adds the job to the problematic_dependencies diff --git a/autosubmit/platforms/headers/pjm_header.py b/autosubmit/platforms/headers/pjm_header.py index db0ebed1ff624118edf1351bb2e5faad23b0d2bf..8764541472bfee95003c5638f631f1594d6aa089 100644 --- a/autosubmit/platforms/headers/pjm_header.py +++ b/autosubmit/platforms/headers/pjm_header.py @@ -175,14 +175,12 @@ class PJMHeader(object): #%ACCOUNT_DIRECTIVE% #%MEMORY_DIRECTIVE% %CUSTOM_DIRECTIVES% -%THREADS_PER_TASK_DIRECTIVE% #%SHAPE_DIRECTIVE% #%NODES_DIRECTIVE% - #PJM -o %CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%OUT_LOG_DIRECTIVE% #PJM -e %CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%ERR_LOG_DIRECTIVE% #%X11% -# +%THREADS_PER_TASK_DIRECTIVE% ############################################################################### """) @@ -195,7 +193,6 @@ class PJMHeader(object): #%NODES_DIRECTIVE% #PJM --mpi "proc=%NUMPROC%" #PJM --mpi "%TASKS_PER_NODE_DIRECTIVE%" -%THREADS_PER_TASK_DIRECTIVE% #PJM -L elapse=%WALLCLOCK%:00 #%QUEUE_DIRECTIVE% #%ACCOUNT_DIRECTIVE% @@ -205,6 +202,6 @@ class PJMHeader(object): #PJM -o %CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%OUT_LOG_DIRECTIVE% #PJM -e %CURRENT_SCRATCH_DIR%/%CURRENT_PROJ_DIR%/%CURRENT_USER%/%DEFAULT.EXPID%/LOG_%DEFAULT.EXPID%/%ERR_LOG_DIRECTIVE% %CUSTOM_DIRECTIVES% -# +%THREADS_PER_TASK_DIRECTIVE% ############################################################################### """) diff --git a/setup.py b/setup.py index 90ea5a1ad6fdc2223928bdb8bb310a90f85b3155..230582c2093da57603b4262bbd533f62d9120c70 100644 --- a/setup.py +++ b/setup.py @@ -44,9 +44,10 @@ install_requires = [ 'py3dotplus==1.1.0', 'numpy<2', 'rocrate==0.*', - 'autosubmitconfigparser==1.0.72', + 'autosubmitconfigparser==1.0.73', 'configparser', 'setproctitle', + 'invoke>=2.0', # 'sqlalchemy[mypy]' # TODO: pending Postgres MR ] diff --git a/test/unit/files/base_pjm.cmd b/test/unit/files/base_pjm.cmd index b6b52eb7912a1ce022f4a11535246ce42eb7aa52..044e486ec2c89187b59993f548f75114b7b3768e 100644 --- a/test/unit/files/base_pjm.cmd +++ b/test/unit/files/base_pjm.cmd @@ -10,14 +10,12 @@ #PJM -g whatever # -export OMP_NUM_THREADS=1 # # - #PJM -o /tmp/pytest-of-dbeltran/pytest-0/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_PJM.cmd.out.0 #PJM -e /tmp/pytest-of-dbeltran/pytest-0/scheduler_tests0/scratch/whatever/dbeltran/t000/LOG_t000/t000_BASE_PJM.cmd.err.0 # -# +export OMP_NUM_THREADS=1 ############################################################################### ################### -# Autosubmit header \ No newline at end of file +# Autosubmit header diff --git a/test/unit/files/nodes_pjm.cmd b/test/unit/files/nodes_pjm.cmd index c305f4deee18b57318f8a5889fc7a4c3d9fbaaaf..0f0fe9f6a2d8cac50422e9e0a47ce3d1414d27b8 100644 --- a/test/unit/files/nodes_pjm.cmd +++ b/test/unit/files/nodes_pjm.cmd @@ -10,14 +10,12 @@ #PJM -g whatever # -export OMP_NUM_THREADS=40 # #PJM -L node=1 - #PJM -o /tmp/pytest-of-dbeltran/pytest-2/scheduler_tests1/scratch/whatever/dbeltran/t000/LOG_t000/t000_NODES_PJM.cmd.out.0 #PJM -e /tmp/pytest-of-dbeltran/pytest-2/scheduler_tests1/scratch/whatever/dbeltran/t000/LOG_t000/t000_NODES_PJM.cmd.err.0 # -# +export OMP_NUM_THREADS=40 ############################################################################### ################### -# Autosubmit header \ No newline at end of file +# Autosubmit header diff --git a/test/unit/test_scheduler_general.py b/test/unit/test_scheduler_general.py index 0f7819d9ec09ce88a67d9ad55b0cd11e1247d64b..4dded6f9aef5ee93d33d1d3fed11b3b4089f8844 100644 --- a/test/unit/test_scheduler_general.py +++ b/test/unit/test_scheduler_general.py @@ -247,7 +247,6 @@ def test_scheduler_job_types(scheduler, job_type, generate_cmds): expected_lines = expected_data.split('\n') actual = actual.split('\n')[:len(expected_lines)] actual = '\n'.join(actual) - # Compare line to line for i, (line1, line2) in enumerate(zip(expected_data.split('\n'), actual.split('\n'))): if "PJM -o" in line1 or "PJM -e" in line1 or "#SBATCH --output" in line1 or "#SBATCH --error" in line1 or "#SBATCH -J" in line1: # output error will be different continue @@ -257,6 +256,3 @@ def test_scheduler_job_types(scheduler, job_type, generate_cmds): continue else: assert line1 == line2 - - -