diff --git a/.gitmodules b/.gitmodules index 26eeaaf28695d54fd84a8ed49a404c2b7267e912..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "autosubmit4-config-parser"] - path = autosubmit4-config-parser - url = ../../ces/autosubmit4-config-parser diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 63da42450ae710c5c1008ee521bec481f34f5f96..2cdcb629895632b4f9d7ed041e75d0a05f4d4841 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -87,7 +87,7 @@ import signal import datetime import portalocker -from pkg_resources import require, resource_listdir, resource_exists, resource_string +from pkg_resources import require, resource_listdir, resource_exists, resource_string, resource_filename from collections import defaultdict from pyparsing import nestedExpr from .history.experiment_status import ExperimentStatus @@ -199,9 +199,10 @@ class Autosubmit: help='Sets the starting time for this experiment') subparser.add_argument('-sa', '--start_after', required=False, help='Sets a experiment expid which completion will trigger the start of this experiment.') - subparser.add_argument('-rm', '--run_members', required=False, + subparser.add_argument('-rom', '--run_only_members', required=False, help='Sets members allowed on this run.') + # Expid subparser = subparsers.add_parser( 'expid', description="Creates a new experiment") @@ -210,14 +211,21 @@ class Autosubmit: '-y', '--copy', help='makes a copy of the specified experiment') group.add_argument('-dm', '--dummy', action='store_true', help='creates a new experiment with default values, usually for testing') + group.add_argument('-min', '--minimal_configuration', action='store_true', + help='creates a new experiment with minimal configuration, usually combined with -repo') + subparser.add_argument('-repo', '--git_repo', type=str, default="", required=False, + help='sets a git repository for the experiment') + subparser.add_argument('-b', '--git_branch', type=str, default="", required=False, + help='sets a git branch for the experiment') + subparser.add_argument('-conf', '--git_as_conf', type=str, default="", required=False,help='sets the git path to as_conf') + group.add_argument('-op', '--operational', action='store_true', help='creates a new experiment with operational experiment id') - subparser.add_argument('-H', '--HPC', required=True, + subparser.add_argument('-H', '--HPC', required=False, default="local", help='specifies the HPC to use for the experiment') subparser.add_argument('-d', '--description', type=str, required=True, help='sets a description for the experiment to store in the database.') - subparser.add_argument('-c', '--config', type=str, required=False, - help='defines where are located the configuration files.') + # Delete subparser = subparsers.add_parser( 'delete', description="delete specified experiment") @@ -628,10 +636,9 @@ class Autosubmit: if args.command == 'run': return Autosubmit.run_experiment(args.expid, args.notransitive, args.update_version, args.start_time, - args.start_after, args.run_members) + args.start_after, args.run_only_members) elif args.command == 'expid': - return Autosubmit.expid(args.HPC, args.description, args.copy, args.dummy, False, - args.operational, args.config) != '' + return Autosubmit.expid(args.description,args.HPC,args.copy, args.dummy,args.minimal_configuration,args.git_repo,args.git_branch,args.git_as_conf,args.operational) != '' elif args.command == 'delete': return Autosubmit.delete(args.expid, args.force) elif args.command == 'monitor': @@ -766,7 +773,7 @@ class Autosubmit: raise AutosubmitCritical(message, 7071) if expid != 'None' and args.command not in expid_less and args.command not in global_log_command: as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) if len(as_conf.experiment_data) == 0: if args.command not in ["expid", "upgrade"]: raise AutosubmitCritical( @@ -1008,280 +1015,177 @@ class Autosubmit: error_message, 6004) @staticmethod - def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False, root_folder=''): + def copy_as_config(exp_id,copy_id): + for conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id,"conf")): + # Copy only relevant files + if conf_file.endswith((".conf, .yml, .yaml")): + shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file.replace(copy_id,exp_id))) + # if ends with .conf convert it to AS4 yaml file + if conf_file.endswith(".conf"): + try: + AutosubmitConfig.ini_to_yaml(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf"), + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file.replace(copy_id,exp_id))) + except Exception as e: + Log.warning("Error converting {0} to yml: {1}".format(conf_file.replace(copy_id,exp_id),str(e))) + @staticmethod + def generate_as_config(exp_id,dummy=False,minimal_configuration=False): + # obtain from autosubmitconfigparser package + # get all as_conf_files from autosubmitconfigparser package + files = resource_listdir('autosubmitconfigparser.config', 'files') + for as_conf_file in files: + if dummy: + if as_conf_file.endswith("dummy.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) + elif minimal_configuration: + if as_conf_file.endswith("minimal.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf","minimal.yml")) + else: + if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file[:-4]+"_"+exp_id+".yml")) + @staticmethod + def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ - Creates a new experiment for given HPC + Replace default values in as_conf files + :param exp_id: experiment id + :param hpc: platform + :param minimal_configuration: minimal configuration + :param git_repo: path to project git repository + :param git_branch: main branch + :param git_as_conf: path to as_conf file in git repository + :return: None + """ + # open and replace values + for as_conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf")): + if as_conf_file.endswith(".yml") or as_conf_file.endswith(".yaml"): + with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'r') as f: + # Copied files could not have default names. + content = f.read() + search = re.search('AUTOSUBMIT_VERSION: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "AUTOSUBMIT_VERSION: \""+Autosubmit.autosubmit_version+"\"") + search = re.search('NOTIFICATIONS: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"NOTIFICATIONS: False") + search = re.search('TO: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "TO: \"\"") + search = re.search('EXPID: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"EXPID: \""+exp_id+"\"") + search = re.search('HPCARCH: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"HPCARCH: \""+hpc+"\"") + if minimal_configuration: + search = re.search('CUSTOM_CONFIG: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "CUSTOM_CONFIG: \"%ROOTDIR%/proj/git_project/"+git_as_conf+"\"") + search = re.search('PROJECT_ORIGIN: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "PROJECT_ORIGIN: \""+git_repo+"\"") + search = re.search('PROJECT_BRANCH: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "PROJECT_BRANCH: \""+git_branch+"\"") + + with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'w') as f: + f.write(content) - :param operational: if true, creates an operational experiment - :type operational: bool - :type hpc: str - :type description: str - :type copy_id: str - :type dummy: bool - :type test: bool - :type root_folder: str - :param hpc: name of the main HPC for the experiment - :param description: short experiment's description. - :param copy_id: experiment identifier of experiment to copy - :param dummy: if true, writes a default dummy configuration for testing - :param test: if true, creates an experiment for testing - - :return: experiment identifier. If method fails, returns ''. - :rtype: str + @staticmethod + def expid(description,hpc="", copy_id='', dummy=False,minimal_configuration=False,git_repo="",git_branch="",git_as_conf="",operational=False): + """ + Creates a new experiment for given HPC + description: description of the experiment + hpc: HPC where the experiment will be executed + copy_id: if specified, experiment id to copy + dummy: if true, creates a dummy experiment + minimal_configuration: if true, creates a minimal configuration + git_repo: git repository to clone + git_branch: git branch to clone + git_as_conf: path to as_conf file in git repository + operational: if true, creates an operational experiment """ + exp_id = "" + root_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR) + if description is None: + raise AutosubmitCritical( + "Check that the parameters are defined (-d) ", 7011) + if hpc is None and not minimal_configuration: + raise AutosubmitCritical( + "Check that the parameters are defined (-H) ", 7011) + # Register the experiment in the database try: - exp_id = None - if description is None or hpc is None: - raise AutosubmitCritical( - "Check that the parameters are defined (-d and -H) ", 7011) - if not copy_id: - exp_id = new_experiment( - description, Autosubmit.autosubmit_version, test, operational) - if exp_id == '': - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass + # Copy another experiment from the database + if copy_id != '' and copy_id is not None: + copy_id_folder = os.path.join(root_folder, copy_id) + if not os.path.exists(copy_id_folder): raise AutosubmitCritical( - "Couldn't create a new experiment", 7011) - try: - os.mkdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id)) - os.mkdir(os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf')) - Log.info("Copying config files...") - - # autosubmit config and experiment copied from AS. - files = resource_listdir('autosubmitconfigparser.config', 'files') - for filename in files: - if resource_exists('autosubmitconfigparser.config', 'files/' + filename): - index = filename.index('.') - new_filename = filename[:index] + "_" + exp_id + filename[index:] - - if filename == 'platforms.yml' and BasicConfig.DEFAULT_PLATFORMS_CONF != '': - content = open(os.path.join( - BasicConfig.DEFAULT_PLATFORMS_CONF, filename), 'rb').read() - elif filename == 'jobs.yml' and BasicConfig.DEFAULT_JOBS_CONF != '': - content = open(os.path.join( - BasicConfig.DEFAULT_JOBS_CONF, filename), 'rb').read() - else: - content = resource_string( - 'autosubmitconfigparser.config', 'files/' + filename) - - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile(BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').read() - - conf_new_filename = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", new_filename) - Log.debug(conf_new_filename) - open(conf_new_filename, 'wb').write(content) - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy, copy_id) - except (OSError, IOError) as e: - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise AutosubmitCritical( - "Couldn't create a new experiment, permissions?", 7012) - - except AutosubmitCritical as e: - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise - except BaseException as e: - err=str(e) - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise AutosubmitCritical("Couldn't create a new experiment", 7012, err) + "Experiment {0} doesn't exists".format(copy_id), 7011) + exp_id = copy_experiment(copy_id, description, Autosubmit.autosubmit_version, False, operational) else: - try: - if root_folder == '' or root_folder is None: - root_folder = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, copy_id) - if os.path.exists(root_folder): - # List of allowed files from conf - conf_copy_filter_folder = [] - conf_copy_filter_old_format_folder = [] - conf_copy_filter = ["autosubmit_" + str(copy_id) + ".yml", - "expdef_" + str(copy_id) + ".yml", - "jobs_" + str(copy_id) + ".yml", - "platforms_" + str(copy_id) + ".yml", - "proj_" + str(copy_id) + ".yml", - "autosubmit_" + str(copy_id) + ".conf", - "expdef_" + str(copy_id) + ".conf", - "jobs_" + str(copy_id) + ".conf", - "platforms_" + str(copy_id) + ".conf", - "proj_" + str(copy_id) + ".conf"] - if root_folder != os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id): - conf_copy_filter_folder = ["autosubmit.yml", - "expdef.yml", - "jobs.yml", - "platforms.yml", - "proj.yml"] - - exp_id = new_experiment( - description, Autosubmit.autosubmit_version, test, operational) - else: - exp_id = copy_experiment( - copy_id, description, Autosubmit.autosubmit_version, test, operational) - - if exp_id == '': - return '' - dir_exp_id = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id) - os.mkdir(dir_exp_id) - os.mkdir(dir_exp_id + '/conf') - if root_folder == os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id): - Log.info( - "Copying previous experiment config directories") - conf_copy_id = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf") - else: - Log.info("Copying from folder: {0}", root_folder) - conf_copy_id = root_folder - files = os.listdir(conf_copy_id) - for filename in files: - # Allow only those files in the list - Log.info("Copying filename: {0}", filename) - if filename in conf_copy_filter: - if os.path.isfile(os.path.join(conf_copy_id, filename)): - new_filename = filename.replace( - copy_id, exp_id) - if new_filename[-4:] == "conf": - new_filename = new_filename[:-4] + "yml" - # Using readlines for replacement handling - content = open(os.path.join( - conf_copy_id, filename), 'r').readlines() - - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile( - BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').readlines() - # Setting email notifications to false - if filename.startswith("autosubmit") and filename.endswith("conf"): - content = ["NOTIFICATIONS = False\n" if line.startswith( - ("NOTIFICATIONS =", "notifications =")) else line for line in content] - elif filename.startswith("autosubmit") and filename.endswith("yml"): - content = [" NOTIFICATIONS: False\n" if "NOTIFICATIONS" in line else line for - line in content] - # Putting content together before writing - sep = "" - open(os.path.join(dir_exp_id, "conf", - new_filename), 'w').write(sep.join(content)) - if filename.endswith("conf"): - try: - AutosubmitConfig.ini_to_yaml(os.path.join(dir_exp_id, "conf"), - os.path.join(os.path.join(dir_exp_id, "conf"), - new_filename)) - except BaseException as e: - Log.warning("Couldn't convert conf file to yml: {0}", new_filename) - if filename in conf_copy_filter_folder: - if os.path.isfile(os.path.join(conf_copy_id, filename)): - new_filename = filename.split( - ".")[0] + "_" + exp_id + ".yml" - content = open(os.path.join( - conf_copy_id, filename), 'rb').read() - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile( - BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').read() - - open(os.path.join(dir_exp_id, "conf", - new_filename), 'wb').write(content) - - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy, copy_id) - ##### - autosubmit_config = AutosubmitConfig( - exp_id, BasicConfig, YAMLParserFactory()) - try: - autosubmit_config.check_conf_files(False) - project_type = autosubmit_config.get_project_type() - if project_type == "git": - autosubmit_git = AutosubmitGit(copy_id[0]) - Log.info("checking model version...") - if not autosubmit_git.check_commit(autosubmit_config): - raise AutosubmitCritical( - "Uncommitted changes", 7013) - except Exception as error: - Log.warning( - "Trace: {0}\nCouldn't load experiment configuration, check the experiments files before perform a create".format( - str(error))) - else: - raise AutosubmitCritical( - "The experiment directory doesn't exist", 7012) - except (OSError, IOError) as e: - try: - Autosubmit._delete_expid(exp_id, True) - except Exception as e: - pass - raise AutosubmitCritical( - "Can not create experiment", 7012) - except BaseException as e: - try: - Autosubmit._delete_expid(exp_id, True) - except Exception as e: - pass - raise AutosubmitCritical( - "Can not create experiment", 7012) - - Log.debug("Creating temporal directory...") - exp_id_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id) - tmp_path = os.path.join(exp_id_path, "tmp") - os.mkdir(tmp_path) - os.chmod(tmp_path, 0o775) - os.mkdir(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR)) - os.chmod(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR), 0o775) - Log.debug("Creating temporal remote directory...") - remote_tmp_path = os.path.join(tmp_path, "LOG_" + exp_id) - os.mkdir(remote_tmp_path) - os.chmod(remote_tmp_path, 0o755) - - Log.debug("Creating pkl directory...") - os.mkdir(os.path.join(exp_id_path, "pkl")) - - Log.debug("Creating plot directory...") - os.mkdir(os.path.join(exp_id_path, "plot")) - os.chmod(os.path.join(exp_id_path, "plot"), 0o775) - Log.result("Experiment registered successfully") - Log.warning("Remember to MODIFY the config files!") + # Create a new experiment from scratch + exp_id = new_experiment(description, Autosubmit.autosubmit_version, False, operational) + + if exp_id == '': + raise AutosubmitCritical("No expid", 7011) + except Exception as e: + raise AutosubmitCritical("Error while generating a new experiment in the db: {0}".format(str(e)), 7011) + + # Create the experiment structure + Log.info("Generating folder structure...") + + exp_folder = os.path.join(root_folder, exp_id) + try: + os.mkdir(exp_folder) + os.mkdir(os.path.join(exp_folder, "conf")) + os.mkdir(os.path.join(exp_folder, "pkl")) + os.mkdir(os.path.join(exp_folder, "tmp")) + os.mkdir(os.path.join(exp_folder, "tmp", "ASLOGS")) + os.mkdir(os.path.join(exp_folder, "tmp", "LOG_"+exp_id.upper())) + os.mkdir(os.path.join(exp_folder, "plot")) + os.mkdir(os.path.join(exp_folder, "status")) + # Setting permissions + os.chmod(exp_folder, 0o755) + os.chmod(os.path.join(exp_folder, "conf"), 0o755) + os.chmod(os.path.join(exp_folder, "pkl"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "ASLOGS"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "LOG_"+exp_id.upper()), 0o755) + os.chmod(os.path.join(exp_folder, "plot"), 0o755) + os.chmod(os.path.join(exp_folder, "status"), 0o755) + except OSError as e: try: - Log.debug("Setting the right permissions...") - os.chmod(os.path.join(exp_id_path, "conf"), 0o755) - os.chmod(os.path.join(exp_id_path, "pkl"), 0o755) - os.chmod(os.path.join(exp_id_path, "tmp"), 0o775) - os.chmod(os.path.join(exp_id_path, "plot"), 0o775) - os.chmod(os.path.join(exp_id_path, "conf/autosubmit_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/expdef_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/jobs_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/platforms_" + - str(exp_id) + ".yml"), 0o755) - try: - os.chmod(os.path.join(exp_id_path, "tmp/ASLOGS"), 0o755) - except Exception as e: - pass - os.chmod(os.path.join(exp_id_path, "conf/proj_" + - str(exp_id) + ".yml"), 0o755) - except Exception as e: - pass # some folder may no exists, like proj - Log.debug("Finished") + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while creating the experiment structure: {0}".format(str(e)), 7011) - return exp_id - except AutosubmitCritical: - raise + # Create the experiment configuration + Log.info("Generating config files...") + try: + if copy_id != '' and copy_id is not None: + # Copy the configuration from selected experiment + Autosubmit.copy_as_config(exp_id, copy_id) + else: + # Create a new configuration + Autosubmit.generate_as_config(exp_id,dummy, minimal_configuration) except Exception as e: - raise AutosubmitCritical("Couldn't create a new experiment", 7012, str(e)) + try: + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while creating the experiment configuration: {0}".format(str(e)), 7011) + # Change template values by default values specified from the commandline + try: + Autosubmit.as_conf_default_values(exp_id,hpc,minimal_configuration,git_repo,git_branch,git_as_conf) + except Exception as e: + try: + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while setting the default values: {0}".format(str(e)), 7011) + + Log.result("Experiment {0} created".format(exp_id)) + return exp_id @staticmethod def delete(expid, force): @@ -1650,11 +1554,11 @@ class Autosubmit: @staticmethod def run_experiment(expid, notransitive=False, update_version=False, start_time=None, start_after=None, - run_members=None): + run_only_members=None): """ Runs and experiment (submitting all the jobs properly and repeating its execution in case of failure). - :param run_members: + :param run_only_ºmembers: :param start_after: :param start_time: :param update_version: @@ -1673,7 +1577,7 @@ class Autosubmit: raise AutosubmitCritical("Failure during the loading of the experiment configuration, check file paths", 7014, str(e)) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(True, True) + as_conf.check_conf_files(running_time=True, force_load=True) try: # Handling starting time AutosubmitHelper.handle_start_time(start_time) @@ -1681,8 +1585,8 @@ class Autosubmit: # Start after completion trigger block AutosubmitHelper.handle_start_after(start_after, expid, BasicConfig()) - # Handling run_members - allowed_members = AutosubmitHelper.get_allowed_members(run_members, as_conf) + # Handling run_only_members + allowed_members = AutosubmitHelper.get_allowed_members(run_only_members, as_conf) except AutosubmitCritical as e: raise except BaseException as e: @@ -2037,7 +1941,7 @@ class Autosubmit: consecutive_retrials = consecutive_retrials + 1 Log.info("Waiting {0} seconds before continue".format(delay)) try: - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) Log.info("Recovering job_list...") job_list = Autosubmit.load_job_list( expid, as_conf, notransitive=notransitive) @@ -3915,7 +3819,7 @@ class Autosubmit: try: Autosubmit._check_ownership(expid, raise_error=True) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - # as_conf.reload(first_load=True) + # as_conf.reload(force_load=True) as_conf.check_conf_files(refresh=True) except (AutosubmitError, AutosubmitCritical): raise @@ -3945,7 +3849,7 @@ class Autosubmit: Autosubmit._check_ownership(expid, raise_error=True) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) as_conf.check_expdef_conf() Log.info("Changing {0} experiment version from {1} to {2}", @@ -4064,7 +3968,7 @@ class Autosubmit: except Exception as e: Log.warning("Couldn't convert conf file to yml: {0}", Path(f).parent) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) # Load current variables as_conf.check_conf_files() # Load current parameters ( this doesn't read job parameters) @@ -4402,8 +4306,8 @@ class Autosubmit: @staticmethod def _create_project_associated_conf(as_conf, force_model_conf, force_jobs_conf): - project_destiny = as_conf.project_file - jobs_destiny = as_conf.jobs_file + project_destiny = as_conf.get_file_project_conf() + jobs_destiny = as_conf.get_file_jobs_conf() if as_conf.get_project_type() != 'none': if as_conf.get_file_project_conf(): @@ -4474,9 +4378,11 @@ class Autosubmit: project_type = as_conf.get_project_type() # Getting output type provided by the user in config, 'pdf' as default output_type = as_conf.get_output_type() - - if not Autosubmit._copy_code(as_conf, expid, project_type, False): - return False + try: + if not Autosubmit._copy_code(as_conf, expid, project_type, False): + return False + except: + raise AutosubmitCritical("Error obtaining the project data, check the parameters related to PROJECT and GIT/SVN or LOCAL sections", code=7014) # Update configuration with the new config in the dist ( if any ) as_conf.check_conf_files(False) @@ -5452,43 +5358,6 @@ class Autosubmit: except ValueError: sys.stdout.write('Please respond with \'y\' or \'n\'.\n') - @staticmethod - def _prepare_conf_files(exp_id, hpc, autosubmit_version, dummy, copy_id): - """ - Changes default configuration files to match new experiment values - - :param exp_id: experiment identifier - :type exp_id: str - :param hpc: hpc to use - :type hpc: str - :param autosubmit_version: current autosubmit's version - :type autosubmit_version: str - :param dummy: if True, creates a dummy experiment adding some default values - :type dummy: bool - """ - as_conf = AutosubmitConfig(exp_id, BasicConfig, YAMLParserFactory()) - as_conf.set_version(autosubmit_version) - update_experiment_descrip_version(exp_id, version=Autosubmit.autosubmit_version) - as_conf.set_expid(exp_id) - as_conf.set_platform(hpc) - - if dummy: - content = open(as_conf.experiment_file).read() - - # Experiment - content = content.replace(re.search('DATELIST: .*', content, re.MULTILINE).group(0), - "DATELIST: 20000101") - content = content.replace(re.search('MEMBERS: .*', content, re.MULTILINE).group(0), - "MEMBERS: fc0") - content = content.replace(re.search('CHUNKSIZE: .*', content, re.MULTILINE).group(0), - "CHUNKSIZE: 4") - content = content.replace(re.search('NUMCHUNKS: .*', content, re.MULTILINE).group(0), - "NUMCHUNKS: 2") - content = content.replace(re.search('PROJECT_TYPE: .*', content, re.MULTILINE).group(0), - "PROJECT_TYPE: 'none'") - - open(as_conf.experiment_file, 'w').write(content) - @staticmethod def _get_status(s): """ diff --git a/autosubmit/experiment/experiment_common.py b/autosubmit/experiment/experiment_common.py index 52ff8b91b4c58bd97552ca01091440719ad21ba1..17eba23fcee5061461b30406b53367d5a1032e84 100644 --- a/autosubmit/experiment/experiment_common.py +++ b/autosubmit/experiment/experiment_common.py @@ -29,14 +29,14 @@ def new_experiment(description, version, test=False, operational=False): """ Stores a new experiment on the database and generates its identifier - :param version: autosubmit version associated to the experiment + :param description: description of the experiment + :type description: str + :param version: version of the experiment :type version: str - :param test: flag for test experiments + :param test: if True, the experiment is a test experiment :type test: bool - :param operational: flag for operational experiments + :param operational: if True, the experiment is an operational experiment :type operational: bool - :param description: experiment's description - :type description: str :return: experiment id for the new experiment :rtype: str """ diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index b95ed4ce800d1f61684a103caacfac80cc11fa4a..29e16e8b63ddbaa47e8e062551ab51bbcc5a1f63 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . - +import locale from os import path import os from shutil import rmtree @@ -205,7 +205,7 @@ class AutosubmitGit: Log.debug('Clone command: {0}', command_0) try: git_version = subprocess.check_output("git --version",shell=True) - git_version = git_version.split(" ")[2].strip("\n") + git_version = git_version.split(" ")[2].decode(locale.getlocale()[1]).strip("\n") version_int = "" for number in git_version.split("."): version_int += number diff --git a/autosubmit/helpers/autosubmit_helper.py b/autosubmit/helpers/autosubmit_helper.py index 932c07e9ed5d3b43ab3c6344cfdbfdd392be2483..2aef35c49d8ee8a94312add09e5e931df8ebf34d 100644 --- a/autosubmit/helpers/autosubmit_helper.py +++ b/autosubmit/helpers/autosubmit_helper.py @@ -104,6 +104,6 @@ def get_allowed_members(run_members, as_conf): raise AutosubmitCritical("Some of the members ({0}) in the list of allowed members you supplied do not exist in the current list " + "of members specified in the conf files.\nCurrent list of members: {1}".format(str(rmember), str(as_conf.get_member_list()))) if len(allowed_members) == 0: - raise AutosubmitCritical("Not a valid -rm --run_members input: {0}".format(str(run_members))) + raise AutosubmitCritical("Not a valid -rom --run_only_members input: {0}".format(str(run_members))) return allowed_members return [] \ No newline at end of file diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index a966c8fb0937394ff69ce70235a16f9eeac81b52..21d5d1cbbba2d047a59e63d43b98106b2ec85925 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -655,7 +655,7 @@ class Job(object): while (count < retries) or not success: try: as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) max_retrials = as_conf.get_retrials() max_logs = int(as_conf.get_retrials()) - fail_count last_log = int(as_conf.get_retrials()) - fail_count @@ -978,26 +978,101 @@ class Job(object): self.status = default_status else: return default_status + def update_platform_parameters(self,as_conf,parameters,job_platform): + parameters['CURRENT_ARCH'] = job_platform.name + parameters['CURRENT_HOST'] = job_platform.host + parameters['CURRENT_USER'] = job_platform.user + parameters['CURRENT_PROJ'] = job_platform.project + parameters['CURRENT_BUDG'] = job_platform.budget + parameters['CURRENT_RESERVATION'] = job_platform.reservation + parameters['CURRENT_EXCLUSIVITY'] = job_platform.exclusivity + parameters['CURRENT_HYPERTHREADING'] = job_platform.hyperthreading + parameters['CURRENT_TYPE'] = job_platform.type + parameters['CURRENT_SCRATCH_DIR'] = job_platform.scratch + parameters['CURRENT_PROJ_DIR'] = job_platform.project_dir + parameters['CURRENT_ROOTDIR'] = job_platform.root_dir + parameters['CURRENT_LOGDIR'] = job_platform.get_files_path() - def update_parameters(self, as_conf, parameters, - default_parameters={'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', - 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'}): - """ - Refresh parameters value - - :param default_parameters: - :type default_parameters: dict - :param as_conf: - :type as_conf: AutosubmitConfig - :param parameters: - :type parameters: dict - """ - chunk = 1 - as_conf.reload() - #parameters = as_conf.substitute_dynamic_variables(parameters,25) + return parameters + def update_platform_associated_parameters(self,as_conf, parameters, job_platform, chunk): + self.queue = self.queue + self.processors = str(as_conf.jobs_data[self.section].get("PROCESSORS",as_conf.platforms_data.get(job_platform.name,{}).get("PROCESSORS","1"))) + self.threads = str(as_conf.jobs_data[self.section].get("THREADS",as_conf.platforms_data.get(job_platform.name,{}).get("THREADS","1"))) + self.tasks = str(as_conf.jobs_data[self.section].get("TASKS",as_conf.platforms_data.get(job_platform.name,{}).get("TASKS","1"))) + self.nodes = str(as_conf.jobs_data[self.section].get("NODES",as_conf.platforms_data.get(job_platform.name,{}).get("NODES","1"))) + self.hyperthreading = str(as_conf.jobs_data[self.section].get("HYPERTHREADING",as_conf.platforms_data.get(job_platform.name,{}).get("HYPERTHREADING","none"))) + if int(self.tasks) <= 1 and int(job_platform.processors_per_node) > 1 and int(self.processors) > int(job_platform.processors_per_node): + self.tasks = job_platform.processors_per_node + self.memory = str(as_conf.jobs_data[self.section].get("MEMORY",as_conf.platforms_data.get(job_platform.name,{}).get("MEMORY",""))) + self.memory_per_task = str(as_conf.jobs_data[self.section].get("MEMORY_PER_TASK",as_conf.platforms_data.get(job_platform.name,{}).get("MEMORY_PER_TASK",""))) + self.wallclock = as_conf.jobs_data[self.section].get("WALLCLOCK",as_conf.platforms_data.get(self.platform_name,{}).get("MAX_WALLCLOCK",None)) + if self.wallclock is None and job_platform.type not in ['ps',"local","PS","LOCAL"]: + self.wallclock = "01:59" + elif self.wallclock is None and job_platform.type in ['ps','local',"PS","LOCAL"]: + self.wallclock = "00:00" + # Increasing according to chunk + self.wallclock = increase_wallclock_by_chunk( + self.wallclock, self.wchunkinc, chunk) + self.scratch_free_space = int(as_conf.jobs_data[self.section].get("SCRATCH_FREE_SPACE",as_conf.platforms_data.get(job_platform.name,{}).get("SCRATCH_FREE_SPACE",0))) + try: + self.custom_directives = as_conf.jobs_data[self.section].get("CUSTOM_DIRECTIVES","").replace("\'", "\"").strip("[]").strip(", ") + if self.custom_directives == '': + if job_platform.custom_directives is None: + job_platform.custom_directives = '' + self.custom_directives = job_platform.custom_directives.replace("\'", "\"").strip("[]").strip(", ") + if self.custom_directives != '': + if self.custom_directives[0] != "\"": + self.custom_directives = "\""+self.custom_directives + if self.custom_directives[-1] != "\"": + self.custom_directives = self.custom_directives+"\"" + self.custom_directives = "[" + self.custom_directives + "]" + self.custom_directives = json.loads(self.custom_directives) + else: + self.custom_directives = [] + except BaseException as e: + raise AutosubmitCritical(f"Error in CUSTOM_DIRECTIVES({self.custom_directives}) for job {self.section}",7014,str(e)) + parameters['NUMPROC'] = self.processors + parameters['PROCESSORS'] = self.processors + parameters['MEMORY'] = self.memory + parameters['MEMORY_PER_TASK'] = self.memory_per_task + parameters['NUMTHREADS'] = self.threads + parameters['THREADS'] = self.threads + parameters['CPUS_PER_TASK'] = self.threads + parameters['NUMTASK'] = self.tasks + parameters['TASKS'] = self.tasks + parameters['NODES'] = self.nodes + parameters['TASKS_PER_NODE'] = self.tasks + parameters['WALLCLOCK'] = self.wallclock + parameters['TASKTYPE'] = self.section + parameters['SCRATCH_FREE_SPACE'] = self.scratch_free_space + parameters['CUSTOM_DIRECTIVES'] = self.custom_directives + parameters['HYPERTHREADING'] = self.hyperthreading + parameters['CURRENT_QUEUE'] = self.queue + return parameters + def update_wrapper_parameters(self,as_conf, parameters): + wrappers = as_conf.experiment_data.get("WRAPPERS", {}) + if len(wrappers) > 0: + parameters['WRAPPER'] = as_conf.get_wrapper_type() + parameters['WRAPPER' + "_POLICY"] = as_conf.get_wrapper_policy() + parameters['WRAPPER' + "_METHOD"] = as_conf.get_wrapper_method().lower() + parameters['WRAPPER' + "_JOBS"] = as_conf.get_wrapper_jobs() + parameters['WRAPPER' + "_EXTENSIBLE"] = as_conf.get_extensible_wallclock() - parameters = parameters.copy() - parameters.update(default_parameters) + for wrapper_section, wrapper_val in wrappers.items(): + if type(wrapper_val) is not dict: + continue + parameters[wrapper_section] = as_conf.get_wrapper_type( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) + parameters[wrapper_section + "_POLICY"] = as_conf.get_wrapper_policy( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) + parameters[wrapper_section + "_METHOD"] = as_conf.get_wrapper_method( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)).lower() + parameters[wrapper_section + "_JOBS"] = as_conf.get_wrapper_jobs( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) + parameters[wrapper_section + "_EXTENSIBLE"] = int( + as_conf.get_extensible_wallclock(as_conf.experiment_data["WRAPPERS"].get(wrapper_section))) + return parameters + def update_job_parameters(self,as_conf, parameters): parameters['JOBNAME'] = self.name parameters['FAIL_COUNT'] = str(self.fail_count) parameters['SDATE'] = date2str(self.date, self.date_format) @@ -1007,6 +1082,7 @@ class Job(object): parameters['FREQUENCY'] = self.frequency parameters['SYNCHRONIZE'] = self.synchronize parameters['PACKED'] = self.packed + parameters['CHUNK'] = 1 if hasattr(self, 'RETRIALS'): parameters['RETRIALS'] = self.retrials if hasattr(self, 'delay_retrials'): @@ -1018,10 +1094,10 @@ class Job(object): chunk = self.chunk parameters['CHUNK'] = chunk - total_chunk = int(parameters.get('EXPERIMENT.NUMCHUNKS',1)) - chunk_length = int(parameters.get('EXPERIMENT.CHUNKSIZE',1)) - chunk_unit = str(parameters.get('EXPERIMENT.CHUNKSIZEUNIT',"")).lower() - cal = str(parameters.get('EXPERIMENT.CALENDAR',"")).lower() + total_chunk = int(parameters.get('EXPERIMENT.NUMCHUNKS', 1)) + chunk_length = int(parameters.get('EXPERIMENT.CHUNKSIZE', 1)) + chunk_unit = str(parameters.get('EXPERIMENT.CHUNKSIZEUNIT', "")).lower() + cal = str(parameters.get('EXPERIMENT.CALENDAR', "")).lower() chunk_start = chunk_start_date( self.date, chunk, chunk_length, chunk_unit, cal) chunk_end = chunk_end_date( @@ -1046,7 +1122,6 @@ class Job(object): parameters['CHUNK_START_DAY'] = str(chunk_start.day).zfill(2) parameters['CHUNK_START_HOUR'] = str(chunk_start.hour).zfill(2) - parameters['CHUNK_SECOND_TO_LAST_DATE'] = date2str( chunk_end_1, self.date_format) parameters['CHUNK_SECOND_TO_LAST_YEAR'] = str(chunk_end_1.year) @@ -1072,128 +1147,55 @@ class Job(object): parameters['Chunk_LAST'] = 'TRUE' else: parameters['Chunk_LAST'] = 'FALSE' - - job_platform = self._platform - self.queue = self.queue - self.processors = str(as_conf.jobs_data[self.section].get("PROCESSORS","1")) - self.threads = str(as_conf.jobs_data[self.section].get("THREADS","1")) - self.tasks = str(as_conf.jobs_data[self.section].get("TASKS","1")) - self.nodes = str(as_conf.jobs_data[self.section].get("NODES","")) - - self.hyperthreading = str(as_conf.jobs_data[self.section].get("HYPERTHREADING","none")) - if self.hyperthreading == 'none' and len(self.hyperthreading) > 0: - self.hyperthreading = job_platform.hyperthreading - if int(self.tasks) <= 1 and int(job_platform.processors_per_node) > 1 and int(self.processors) > int(job_platform.processors_per_node): - self.tasks = job_platform.processors_per_node - self.memory = str(as_conf.jobs_data[self.section].get("MEMORY","")) - self.memory_per_task = str(as_conf.jobs_data[self.section].get("MEMORY_PER_TASK","")) - remote_max_wallclock = as_conf.experiment_data["PLATFORMS"].get(self.platform_name,{}) - remote_max_wallclock = remote_max_wallclock.get("MAX_WALLCLOCK",None) - self.wallclock = as_conf.jobs_data[self.section].get("WALLCLOCK",remote_max_wallclock) - self.wchunkinc = str(as_conf.jobs_data[self.section].get("WCHUNKINC","")) - if self.wallclock is None and job_platform.type not in ['ps',"local","PS","LOCAL"]: - self.wallclock = "01:59" - elif self.wallclock is None and job_platform.type in ['ps','local',"PS","LOCAL"]: - self.wallclock = "00:00" + parameters['NUMMEMBERS'] = len(as_conf.get_member_list()) + parameters['DEPENDENCIES'] = str(as_conf.jobs_data[self.section].get("DEPENDENCIES","")) + # This shouldn't be necessary anymore as now all sub is done in the as_conf.reload() + # if len(self.export) > 0: + # variables = re.findall('%(? 0: + # variables = [variable[1:-1] for variable in variables] + # for key in variables: + # try: + # self.export = re.sub( + # '%(? 0: - parameters['WRAPPER'] = as_conf.get_wrapper_type() - parameters['WRAPPER' + "_POLICY"] = as_conf.get_wrapper_policy() - parameters['WRAPPER' + "_METHOD"] = as_conf.get_wrapper_method().lower() - parameters['WRAPPER' + "_JOBS"] = as_conf.get_wrapper_jobs() - parameters['WRAPPER' + "_EXTENSIBLE"] = as_conf.get_extensible_wallclock() - - for wrapper_section,wrapper_val in wrappers.items(): - if type(wrapper_val) is not dict: - continue - parameters[wrapper_section] = as_conf.get_wrapper_type(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) - parameters[wrapper_section+"_POLICY"] = as_conf.get_wrapper_policy(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) - parameters[wrapper_section+"_METHOD"] = as_conf.get_wrapper_method(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)).lower() - parameters[wrapper_section+"_JOBS"] = as_conf.get_wrapper_jobs(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) - parameters[wrapper_section+"_EXTENSIBLE"] = int(as_conf.get_extensible_wallclock(as_conf.experiment_data["WRAPPERS"].get(wrapper_section))) - self.dependencies = parameters['DEPENDENCIES'] - - # This shouldn't be necessary anymore as now all sub is done in the as_conf.reload() - if len(self.export) > 0: - variables = re.findall('%(? 0: - variables = [variable[1:-1] for variable in variables] - for key in variables: - try: - self.export = re.sub( - '%(?=1.4.0 - python-dateutil>=2.8.2 - matplotlib>=3.5.1 diff --git a/requeriments.txt b/requeriments.txt index 929e7c7066ac960725358138e93207022f3bcaf1..4b55c25a36ef81d9106e0af5aa0d14278cb1506d 100644 --- a/requeriments.txt +++ b/requeriments.txt @@ -1,4 +1,4 @@ -autosubmitconfigparser==0.0.78 +autosubmitconfigparser==1.0.0 paramiko>=2.9.2 PyNaCl>=1.5.0 configobj>=5.0.6 diff --git a/setup.py b/setup.py index 6a8e9b0dce1f0959d5ca49fcd39f982b2e1c6384..90d21740524fc4606b6f3c2bfae55b85a4ffcf72 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=['autosubmitconfigparser==0.0.78','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl>=1.4.0','Pygments'], + install_requires=['autosubmitconfigparser==1.0.0','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl>=1.4.0','Pygments'], classifiers=[ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.9", diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 08ec3d7811100c9b3c332892cba6132bc710f181..5982ac6be186cce945486c526ec9cdb6b6db671b 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -262,8 +262,9 @@ class TestJob(TestCase): # This test (and feature) was implemented in order to avoid # false positives on the checking process with auto-ecearth3 # Arrange - section = "random-section" - self.job.section = "random-section" + section = "RANDOM-SECTION" + self.job.section = section + self.job.parameters['ROOTDIR'] = "none" self.job.parameters['PROJECT_TYPE'] = "none" processors = 80 threads = 1 @@ -278,7 +279,8 @@ class TestJob(TestCase): 'TASKS': tasks, 'MEMORY': memory, 'WALLCLOCK': wallclock, - 'CUSTOM_DIRECTIVES': custom_directives + 'CUSTOM_DIRECTIVES': custom_directives, + 'SCRATCH_FREE_SPACE': 0 } self.as_conf.jobs_data[section] = options @@ -286,12 +288,15 @@ class TestJob(TestCase): dummy_serial_platform.name = 'serial' dummy_platform = MagicMock() dummy_platform.serial_platform = dummy_serial_platform + self.as_conf.substitute_dynamic_variables = MagicMock() + self.as_conf.substitute_dynamic_variables.return_value = {'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', + 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'} dummy_platform.custom_directives = '["whatever"]' self.as_conf.dynamic_variables = MagicMock() self.job._platform = dummy_platform - + parameters = {} # Act - parameters = self.job.update_parameters(self.as_conf, dict()) + parameters = self.job.update_parameters(self.as_conf, parameters) # Assert self.assertTrue('d' in parameters) self.assertTrue('d_' in parameters)