diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 797929a5fe99356af630f6452bc8e526eae3934c..100d2dfa818fe2bb3c0f47097db53e1d48b33ce5 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1590,18 +1590,24 @@ class Autosubmit: jobs_aux.append(job) jobs_cw = jobs_aux del jobs_aux + file_paths = "" + if isinstance(jobs, type([])): for job in jobs: + file_paths += f"{BasicConfig.LOCAL_ROOT_DIR}/{expid}/tmp/{job.name}.cmd\n" job.status = Status.WAITING Autosubmit.generate_scripts_andor_wrappers( as_conf, job_list, jobs, packages_persistence, False) if len(jobs_cw) > 0: for job in jobs_cw: + file_paths += f"{BasicConfig.LOCAL_ROOT_DIR}/{expid}/tmp/{job.name}.cmd\n" job.status = Status.WAITING Autosubmit.generate_scripts_andor_wrappers( as_conf, job_list, jobs_cw, packages_persistence, False) - Log.info("no more scripts to generate, now proceed to check them manually") + Log.info("No more scripts to generate, you can proceed to check them manually") + Log.result(file_paths) + except AutosubmitCritical as e: raise except AutosubmitError as e: diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index f3de7d3fb451ba8d74bee5d143ac1915d3937dcf..a3d102cb2003e196c2beb83c739887651771cb99 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1228,6 +1228,36 @@ class JobList(object): filters_to_apply = {} return filters_to_apply + def _normalize_auto_keyword(self, job: Job, dependency: Dependency) -> Dependency: + """ + Normalize the 'auto' keyword in the dependency relationships for a job. + + This function adjusts the 'SPLITS_TO' value in the dependency relationships + if it contains the 'auto' keyword. The 'auto' keyword is replaced with the + actual number of splits for the job. + + :param job: The job object containing job details. + :param dependency: The dependency object containing dependency details. + :return: The dependency object with the attribute relationships updated with the correct number of splits. + """ + if job.splits and dependency.distance and dependency.relationships and job.running == "chunk": + job_name_separated = job.name.split("_") + if dependency.sign == "-": + auto_chunk = int(job_name_separated[3]) - int(dependency.distance) + else: + auto_chunk = int(job_name_separated[3]) + int(dependency.distance) + if auto_chunk < 1: + auto_chunk = int(job_name_separated[3]) + auto_chunk = str(auto_chunk) + # Get first split of the given chunk + auto_job_name = "_".join(job_name_separated[:3]) + f"_{auto_chunk}_1_{dependency.section}" + auto_splits = str(self.graph.nodes[auto_job_name]['job'].splits) + for filters_to_keys, filters_to in dependency.relationships.get("SPLITS_FROM", {}).items(): + if "auto" in filters_to.get("SPLITS_TO", "").lower(): + filters_to["SPLITS_TO"] = filters_to["SPLITS_TO"].lower() + filters_to["SPLITS_TO"] = filters_to["SPLITS_TO"].replace("auto", auto_splits) + return dependency + def _manage_job_dependencies(self, dic_jobs, job, date_list, member_list, chunk_list, dependencies_keys, dependencies, graph): @@ -1362,6 +1392,7 @@ class JobList(object): dependency) if skip: continue + self._normalize_auto_keyword(job, dependency) filters_to_apply = self.get_filters_to_apply(job, dependency) if len(filters_to_apply) > 0: diff --git a/autosubmit/job/job_utils.py b/autosubmit/job/job_utils.py index 43ae211e6d640bd7c4e515b442b31f458c022a13..273dedee396e9c9e3063a48788af1c63bbfef618 100644 --- a/autosubmit/job/job_utils.py +++ b/autosubmit/job/job_utils.py @@ -219,9 +219,8 @@ def get_split_size_unit(data, section): def get_split_size(as_conf, section): - job_data = as_conf.get('JOBS',{}).get(section,{}) - exp_data = as_conf.get('EXPERIMENT',{}) - return int(job_data.get("SPLITSIZE", exp_data.get("SPLITSIZE", exp_data.get('CHUNKSIZE', 1)))) + job_data = as_conf.get('JOBS', {}).get(section, {}) + return int(job_data.get("SPLITSIZE", 1)) def transitive_reduction(graph): """ diff --git a/test/resources/exp_recipes/test_splits_auto_1_chunk.yml b/test/resources/exp_recipes/test_splits_auto_1_chunk.yml new file mode 100644 index 0000000000000000000000000000000000000000..ff4024a6667757c2e71b146780763cda8ee4fded --- /dev/null +++ b/test/resources/exp_recipes/test_splits_auto_1_chunk.yml @@ -0,0 +1,79 @@ +CONFIG: + # Current version of Autosubmit. + AUTOSUBMIT_VERSION: "4.1.10" + # Maximum number of jobs permitted in the waiting status. + MAXWAITINGJOBS: 20 + # Total number of jobs in the workflow. + TOTALJOBS: 20 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +MAIL: + NOTIFICATIONS: False + TO: +STORAGE: + TYPE: pkl + COPY_REMOTE_LOGS: true +DEFAULT: + # Job experiment ID. + EXPID: "a00c" + # Default HPC platform name. + HPCARCH: "local" + +EXPERIMENT: + DATELIST: 20221101 + MEMBERS: fc0 + CHUNKSIZEUNIT: month + #SPLITSIZEUNIT: day + CHUNKSIZE: 2 + NUMCHUNKS: 1 + #SPLITSIZE: 1 + SPLITPOLICY: flexible + CHUNKINI: '' + CALENDAR: standard +PROJECT: + PROJECT_TYPE: 'none' +PLATFORMS: + debug: + type: ps + host: localhost +JOBS: + APP: + SCRIPT: | + echo "Chunk start date: %CHUNK_START_DATE%" + echo "Chunk end date: %CHUNK_END_DATE%" + echo "Split start date: %SPLIT_START_DATE%" + echo "Split end date: %SPLIT_END_DATE%" + echo "Split size: %SPLIT_SIZE%" + echo "Split number: %SPLIT_NUMBER%" + echo "Chunk size: %CHUNK_SIZE%" + DEPENDENCIES: + APP: + SPLITS_FROM: + ALL: + SPLITS_TO: previous + running: chunk + SPLITS: auto + DN: + SCRIPT: | + echo "Chunk start date: %CHUNK_START_DATE%" + echo "Chunk end date: %CHUNK_END_DATE%" + echo "Split start date: %SPLIT_START_DATE%" + echo "Split end date: %SPLIT_END_DATE%" + echo "Split size: %SPLIT_SIZE%" + echo "Split number: %SPLIT_NUMBER%" + echo "Chunk size: %CHUNK_SIZE%" + CHECK: True + DEPENDENCIES: + DN: + SPLITS_FROM: + ALL: + SPLITS_TO: previous + APP: + SPLITS_FROM: + ALL: # You want to set all the DN jobs to depend on the last APP split, otherwise the DN would be need to be tuned one by one. + SPLITS_TO: "%JOBS.APP.SPLITS%" + SPLITS: auto + running: chunk + + + diff --git a/test/resources/exp_recipes/test_splits_full.yml b/test/resources/exp_recipes/test_splits_full.yml new file mode 100644 index 0000000000000000000000000000000000000000..4877ef73678ba5d5ddd3a09a0a80dd188b036e10 --- /dev/null +++ b/test/resources/exp_recipes/test_splits_full.yml @@ -0,0 +1,117 @@ +CONFIG: + # Current version of autosubmit. + autoSUBMIT_VERSION: "4.1.10" + # Maximum number of jobs permitted in the waiting status. + MAXWAITINGJOBS: 20 + # Total number of jobs in the workflow. + TOTALJOBS: 20 + SAFETYSLEEPTIME: 10 + RETRIALS: 0 +MAIL: + NOTIFICATIONS: False + TO: +STORAGE: + TYPE: pkl + COPY_REMOTE_LOGS: true +DEFAULT: + # Job experiment ID. + EXPID: "a00c" + # Default HPC platform name. + HPCARCH: "local" + +EXPERIMENT: + DATELIST: 20221101 + MEMBERS: fc0 + CHUNKSIZEUNIT: month + #SPLITSIZEUNIT: day + CHUNKSIZE: 2 + NUMCHUNKS: 2 + #SPLITSIZE: 1 + SPLITPOLICY: flexible + CHUNKINI: '' + CALENDAR: standard +PROJECT: + PROJECT_TYPE: 'none' +PLATFORMS: + debug: + type: ps + host: localhost + +JOBS: + APP: + DEPENDENCIES: + APP: + SPLITS_FROM: + ALL: + SPLITS_TO: previous + OPA: + SPLITS_FROM: + ALL: + SPLITS_TO: '[1:auto]*\1' + SCRIPT: | + echo "Hello world" + RUNNING: chunk + SPLITS: auto + DN: + DEPENDENCIES: + APP-1: + SPLITS_FROM: + ALL: # You want to set all the DN jobs to depend on the last APP split, otherwise the DN would be need to be tuned one by one. + SPLITS_TO: "auto" + DN: + SPLITS_FROM: + ALL: + SPLITS_TO: previous + REMOTE_SETUP: + SIM: + STATUS: RUNNING + SCRIPT: | + echo "Hello world" + RUNNING: chunk + SPLITS: auto + + DQC_BASIC: + DEPENDENCIES: + SIM: {} + SCRIPT: | + echo "Hello world" + RUNNING: chunk + DQC_FULL: + DEPENDENCIES: + DQC_BASIC: {} + SCRIPT: | + echo "Hello world" + RUNNING: chunk + INI: + DEPENDENCIES: + REMOTE_SETUP: {} + SCRIPT: | + echo "Hello world" + RUNNING: member + OPA: + DEPENDENCIES: + DN: + SPLITS_FROM: + ALL: + SPLITS_TO: '[1:auto]*\1' + OPA: + SPLITS_FROM: + ALL: + SPLITS_TO: previous + SCRIPT: | + echo "Hello world" + RUNNING: chunk + SPLITS: auto + REMOTE_SETUP: + SCRIPT: | + echo "Hello world" + RUNNING: once + SIM: + DEPENDENCIES: + DQC_BASIC-10: {} + INI: {} + SIM-1: {} + SCRIPT: | + echo "Hello world" + RUNNING: chunk + diff --git a/test/unit/resources/exp_recipes/wrappers_test.yml b/test/resources/exp_recipes/wrappers_test.yml similarity index 100% rename from test/unit/resources/exp_recipes/wrappers_test.yml rename to test/resources/exp_recipes/wrappers_test.yml diff --git a/test/unit/test_dependencies.py b/test/unit/test_dependencies.py index 054728dcb536082fd37458726a39b7311313917b..e6dc3e8804ee12065200131b45ff2c372935bc20 100644 --- a/test/unit/test_dependencies.py +++ b/test/unit/test_dependencies.py @@ -1,6 +1,5 @@ from unittest.mock import Mock -import copy import inspect import mock import tempfile @@ -10,6 +9,7 @@ from datetime import datetime from mock import patch from mock.mock import MagicMock +from autosubmit.autosubmit import Autosubmit from autosubmit.job.job_dict import DicJobs from autosubmit.job.job import Job from autosubmit.job.job_common import Status @@ -17,6 +17,9 @@ from autosubmit.job.job_list import JobList from autosubmit.job.job_list_persistence import JobListPersistenceDb from autosubmitconfigparser.config.yamlparser import YAMLParserFactory +from networkx import DiGraph +from autosubmit.job.job_utils import Dependency + class FakeBasicConfig: def __init__(self): @@ -548,6 +551,70 @@ class TestJobList(unittest.TestCase): self.assertEqual(expected_output, result) +def test_normalize_auto_keyword(autosubmit_config): + as_conf = autosubmit_config('a000', experiment_data={ + + }) + job_list = JobList( + as_conf.expid, + FakeBasicConfig, + YAMLParserFactory(), + Autosubmit._get_job_list_persistence(as_conf.expid, as_conf), + as_conf + ) + dependency = Dependency("test") + + job = Job("a000_20001010_fc1_2_1_test", 1, Status.READY, 1) + job.running = "chunk" + job.section = "test" + job.date = "20001010" + job.member = "fc1" + job.splits = 5 + + job_minus = Job("a000_20001010_fc1_1_1_minus", 1, Status.READY, 1) + job_minus.running = "chunk" + job_minus.section = "minus" + job_minus.date = "20001010" + job_minus.member = "fc1" + job_minus.splits = 40 + + job_plus = Job("a000_20001010_fc1_3_1_plus", 1, Status.READY, 1) + job_plus.running = "chunk" + job_plus.section = "plus" + job_plus.date = "20001010" + job_plus.member = "fc1" + job_plus.splits = 50 + + job_list.graph = DiGraph() + job_list.graph.add_node(job.name, job=job) + job_list.graph.add_node(job_minus.name, job=job_minus) + job_list.graph.add_node(job_plus.name, job=job_plus) + + dependency.distance = 1 + dependency.relationships = { + "SPLITS_FROM": { + "key": { + "SPLITS_TO": "auto" + } + } + } + dependency.sign = "-" + dependency.section = "minus" + dependency = job_list._normalize_auto_keyword(job, dependency) + assert dependency.relationships["SPLITS_FROM"]["key"]["SPLITS_TO"] == "40" + dependency.relationships = { + "SPLITS_FROM": { + "key": { + "SPLITS_TO": "auto" + } + } + } + dependency.sign = "+" + dependency.section = "plus" + dependency = job_list._normalize_auto_keyword(job, dependency) + assert dependency.relationships["SPLITS_FROM"]["key"]["SPLITS_TO"] == "50" + + if __name__ == '__main__': unittest.main()