From 4b8a629c90a5fb50f226bd0bcf0525b87413370a Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 13 Feb 2023 13:29:09 +0100 Subject: [PATCH 01/26] resolves #829, --- autosubmit/autosubmit.py | 3 ++- autosubmit/helpers/autosubmit_helper.py | 2 +- docs/source/userguide/run/index.rst | 14 +++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 55ec20671..1244f981d 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -199,9 +199,10 @@ class Autosubmit: help='Sets the starting time for this experiment') subparser.add_argument('-sa', '--start_after', required=False, help='Sets a experiment expid which completion will trigger the start of this experiment.') - subparser.add_argument('-rm', '--run_members', required=False, + subparser.add_argument('-rom', '--run_only_members', required=False, help='Sets members allowed on this run.') + # Expid subparser = subparsers.add_parser( 'expid', description="Creates a new experiment") diff --git a/autosubmit/helpers/autosubmit_helper.py b/autosubmit/helpers/autosubmit_helper.py index 932c07e9e..2aef35c49 100644 --- a/autosubmit/helpers/autosubmit_helper.py +++ b/autosubmit/helpers/autosubmit_helper.py @@ -104,6 +104,6 @@ def get_allowed_members(run_members, as_conf): raise AutosubmitCritical("Some of the members ({0}) in the list of allowed members you supplied do not exist in the current list " + "of members specified in the conf files.\nCurrent list of members: {1}".format(str(rmember), str(as_conf.get_member_list()))) if len(allowed_members) == 0: - raise AutosubmitCritical("Not a valid -rm --run_members input: {0}".format(str(run_members))) + raise AutosubmitCritical("Not a valid -rom --run_only_members input: {0}".format(str(run_members))) return allowed_members return [] \ No newline at end of file diff --git a/docs/source/userguide/run/index.rst b/docs/source/userguide/run/index.rst index 5cb0ada3b..fbdd34ef9 100644 --- a/docs/source/userguide/run/index.rst +++ b/docs/source/userguide/run/index.rst @@ -28,7 +28,7 @@ Options: Sets the starting time for the experiment. Accepted format: 'yyyy-mm-dd HH:MM:SS' or 'HH:MM:SS' (defaults to current day). -sa --start_after Sets a experiment expid that will be tracked for completion. When this experiment is completed, the current instance of Autosubmit run will start. - -rm --run_members + -rom,--run_only_members --run_members Sets a list of members allowed to run. The list must have the format '### ###' where '###' represents the name of the member as set in the conf files. -h, --help show this help message and exit @@ -120,7 +120,7 @@ To run only a subset of selected members you can execute the command: # Add your key to ssh agent ( if encrypted ) ssh-add ~/.ssh/id_rsa - autosubmit run EXPID -rm MEMBERS + autosubmit run EXPID -rom MEMBERS *EXPID* is the experiment identifier, the experiment you want to run. @@ -128,17 +128,17 @@ To run only a subset of selected members you can execute the command: Then, your experiment will start running jobs belonging to those members only. If the experiment was previously running and autosubmit was stopped when some jobs belonging to other members (not the ones from your input) where running, those jobs will be tracked and finished in the new exclusive run. -Furthermore, if you wish to run a sequence of only members execution; then, instead of running `autosubmit run -rm "member_1"` ... `autosubmit run -rm "member_n"`, you can make a bash file with that sequence and run the bash file. Example: +Furthermore, if you wish to run a sequence of only members execution; then, instead of running `autosubmit run -rom "member_1"` ... `autosubmit run -rom "member_n"`, you can make a bash file with that sequence and run the bash file. Example: .. code-block:: bash # Add your key to ssh agent ( if encrypted ) ssh-add ~/.ssh/id_rsa - autosubmit run EXPID -rm MEMBER_1 - autosubmit run EXPID -rm MEMBER_2 - autosubmit run EXPID -rm MEMBER_3 + autosubmit run EXPID -rom MEMBER_1 + autosubmit run EXPID -rom MEMBER_2 + autosubmit run EXPID -rom MEMBER_3 ... - autosubmit run EXPID -rm MEMBER_N + autosubmit run EXPID -rom MEMBER_N How to start an experiment at a given time ------------------------------------------ -- GitLab From 74f3c6b066817f8a9c717f2bf645a66c0c270a80 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 11:02:24 +0100 Subject: [PATCH 02/26] Expdef refactored. Added minimal, dummy confs TODO add comments to non-dummy/minimal conf --- autosubmit/autosubmit.py | 477 +++++++-------------- autosubmit/experiment/experiment_common.py | 10 +- docs/source/userguide/create/index.rst | 8 +- 3 files changed, 176 insertions(+), 319 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1244f981d..1abb83cf9 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -87,7 +87,7 @@ import signal import datetime import portalocker -from pkg_resources import require, resource_listdir, resource_exists, resource_string +from pkg_resources import require, resource_listdir, resource_exists, resource_string, resource_filename from collections import defaultdict from pyparsing import nestedExpr from .history.experiment_status import ExperimentStatus @@ -211,14 +211,21 @@ class Autosubmit: '-y', '--copy', help='makes a copy of the specified experiment') group.add_argument('-dm', '--dummy', action='store_true', help='creates a new experiment with default values, usually for testing') + group.add_argument('-min', '--minimal_configuration', action='store_true', + help='creates a new experiment with minimal configuration, usually combined with -repo') + subparser.add_argument('-repo', '--git_repo', type=str, required=False, + help='sets a git repository for the experiment') + subparser.add_argument('-b', '--git_branch', type=str, required=False, + help='sets a git branch for the experiment') + subparser.add_argument('-conf', '--git_as_conf', type=str, required=False,help='sets the git path to as_conf') + group.add_argument('-op', '--operational', action='store_true', help='creates a new experiment with operational experiment id') subparser.add_argument('-H', '--HPC', required=True, help='specifies the HPC to use for the experiment') subparser.add_argument('-d', '--description', type=str, required=True, help='sets a description for the experiment to store in the database.') - subparser.add_argument('-c', '--config', type=str, required=False, - help='defines where are located the configuration files.') + # Delete subparser = subparsers.add_parser( 'delete', description="delete specified experiment") @@ -629,10 +636,9 @@ class Autosubmit: if args.command == 'run': return Autosubmit.run_experiment(args.expid, args.notransitive, args.update_version, args.start_time, - args.start_after, args.run_members) + args.start_after, args.run_only_members) elif args.command == 'expid': - return Autosubmit.expid(args.HPC, args.description, args.copy, args.dummy, False, - args.operational, args.config) != '' + return Autosubmit.expid(args.description,args.HPC,args.copy, args.dummy,args.minimal_configuration,args.git_repo,args.git_branch,args.git_as_conf,args.test,args.operational) != '' elif args.command == 'delete': return Autosubmit.delete(args.expid, args.force) elif args.command == 'monitor': @@ -1009,280 +1015,166 @@ class Autosubmit: error_message, 6004) @staticmethod - def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False, root_folder=''): + def copy_as_config(exp_id,copy_id): + as_conf_files = os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id)) + for conf_file in as_conf_files: + # Copy only relevant files + if conf_file.endswith(".conf") or conf_file.endswith(".yml") or conf_file.endswith(".yaml"): + shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file)) + # if ends with .conf convert it to AS4 yaml file + if conf_file.endswith(".conf"): + try: + AutosubmitConfig.ini_to_yaml(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf"), + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file)) + except Exception as e: + Log.warning("Error converting {0} to yml: {1}".format(conf_file,str(e))) + @staticmethod + def generate_as_config(exp_id,dummy=False,minimal_configuration=False): + # obtain from autosubmitconfigparser package + # get all as_conf_files from autosubmitconfigparser package + files = resource_listdir('autosubmitconfigparser.config', 'files') + for as_conf_file in files: + if dummy: + if as_conf_file.endswith("dummy.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + elif minimal_configuration: + if as_conf_file.endswith("minimal.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + else: + if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + @staticmethod + def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): + """ + Replace default values in as_conf files + :param exp_id: experiment id + :param hpc: platform + :param minimal_configuration: minimal configuration + :param git_repo: path to project git repository + :param branch: main branch + :param config_path: path to config file + :return: + """ + # open and replace values + for as_conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf")): + if as_conf_file.endswith(".yml") or as_conf_file.endswith(".yaml"): + with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'r') as f: + # Copied files could not have default names. + content = f.read() + content = content.replace(re.search('AUTOSUBMIT_VERSION: .*', content, re.MULTILINE).group(0), "AUTOSUBMIT_VERSION: \""+Autosubmit.autosubmit_version+"\"") + content = content.replace(re.search('NOTIFICATIONS: .*', content, re.MULTILINE).group(0), + "NOTIFICATIONS: False") + content = content.replace(re.search('TO: .*', content, re.MULTILINE).group(0), "TO: \"\"") + content = content.replace(re.search('EXPID: .*', content, re.MULTILINE).group(0),"EXPID: \""+exp_id+"\"") + content = content.replace(re.search('HPCARCH: .*', content, re.MULTILINE).group(0),"HPCARCH: \""+hpc+"\"") + if minimal_configuration: + content = content.replace(re.search('CUSTOM_CONFIG: .*', content, re.MULTILINE).group(0), "CUSTOM_CONFIG: \"%ROOTDIR%/proj/git_project/"+git_as_conf+"\"") + content = content.replace(re.search('PROJECT_ORIGIN: .*', content, re.MULTILINE).group(0), "PROJECT_ORIGIN: \""+git_repo+"\"") + content = content.replace(re.search('PROJECT_BRANCH: .*', content, re.MULTILINE).group(0), "PROJECT_BRANCH: \""+git_branch+"\"") + + with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'w') as f: + f.write(content) + + @staticmethod + def expid(description,hpc="", copy_id='', dummy=False,minimal_configuration=False,git_repo="",git_branch="",git_as_conf="",test=False, operational=False): """ Creates a new experiment for given HPC + description: description of the experiment + hpc: HPC where the experiment will be executed + copy_id: if specified, experiment id to copy + dummy: if true, creates a dummy experiment + minimal_configuration: if true, creates a minimal configuration + git_repo: git repository to clone + test: if true, creates a test experiment + operational: if true, creates an operational experiment - :param operational: if true, creates an operational experiment - :type operational: bool - :type hpc: str - :type description: str - :type copy_id: str - :type dummy: bool - :type test: bool - :type root_folder: str - :param hpc: name of the main HPC for the experiment - :param description: short experiment's description. - :param copy_id: experiment identifier of experiment to copy - :param dummy: if true, writes a default dummy configuration for testing - :param test: if true, creates an experiment for testing - - :return: experiment identifier. If method fails, returns ''. - :rtype: str """ + exp_id = None + root_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR) + if description is None: + raise AutosubmitCritical( + "Check that the parameters are defined (-d) ", 7011) + if hpc is None and not minimal_configuration: + raise AutosubmitCritical( + "Check that the parameters are defined (-H) ", 7011) + # Register the experiment in the database try: - exp_id = None - if description is None or hpc is None: - raise AutosubmitCritical( - "Check that the parameters are defined (-d and -H) ", 7011) - if not copy_id: - exp_id = new_experiment( - description, Autosubmit.autosubmit_version, test, operational) - if exp_id == '': - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise AutosubmitCritical( - "Couldn't create a new experiment", 7011) - try: - os.mkdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id)) - os.mkdir(os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf')) - Log.info("Copying config files...") - - # autosubmit config and experiment copied from AS. - files = resource_listdir('autosubmitconfigparser.config', 'files') - for filename in files: - if resource_exists('autosubmitconfigparser.config', 'files/' + filename): - index = filename.index('.') - new_filename = filename[:index] + "_" + exp_id + filename[index:] - - if filename == 'platforms.yml' and BasicConfig.DEFAULT_PLATFORMS_CONF != '': - content = open(os.path.join( - BasicConfig.DEFAULT_PLATFORMS_CONF, filename), 'rb').read() - elif filename == 'jobs.yml' and BasicConfig.DEFAULT_JOBS_CONF != '': - content = open(os.path.join( - BasicConfig.DEFAULT_JOBS_CONF, filename), 'rb').read() - else: - content = resource_string( - 'autosubmitconfigparser.config', 'files/' + filename) - - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile(BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').read() - - conf_new_filename = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", new_filename) - Log.debug(conf_new_filename) - open(conf_new_filename, 'wb').write(content) - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy, copy_id) - except (OSError, IOError) as e: - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass + # Copy another experiment from the database + if copy_id != '': + copy_id_folder = os.path.join(root_folder, copy_id) + if not os.path.exists(copy_id_folder): raise AutosubmitCritical( - "Couldn't create a new experiment, permissions?", 7012) - - except AutosubmitCritical as e: - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise - except BaseException as e: - err=str(e) - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise AutosubmitCritical("Couldn't create a new experiment", 7012, err) + "Experiment {0} doesn't exists".format(copy_id), 7011) + exp_id = copy_experiment(copy_id, description, hpc, test, operational, copy_id_folder) else: - try: - if root_folder == '' or root_folder is None: - root_folder = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, copy_id) - if os.path.exists(root_folder): - # List of allowed files from conf - conf_copy_filter_folder = [] - conf_copy_filter_old_format_folder = [] - conf_copy_filter = ["autosubmit_" + str(copy_id) + ".yml", - "expdef_" + str(copy_id) + ".yml", - "jobs_" + str(copy_id) + ".yml", - "platforms_" + str(copy_id) + ".yml", - "proj_" + str(copy_id) + ".yml", - "autosubmit_" + str(copy_id) + ".conf", - "expdef_" + str(copy_id) + ".conf", - "jobs_" + str(copy_id) + ".conf", - "platforms_" + str(copy_id) + ".conf", - "proj_" + str(copy_id) + ".conf"] - if root_folder != os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id): - conf_copy_filter_folder = ["autosubmit.yml", - "expdef.yml", - "jobs.yml", - "platforms.yml", - "proj.yml"] - - exp_id = new_experiment( - description, Autosubmit.autosubmit_version, test, operational) - else: - exp_id = copy_experiment( - copy_id, description, Autosubmit.autosubmit_version, test, operational) - - if exp_id == '': - return '' - dir_exp_id = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id) - os.mkdir(dir_exp_id) - os.mkdir(dir_exp_id + '/conf') - if root_folder == os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id): - Log.info( - "Copying previous experiment config directories") - conf_copy_id = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf") - else: - Log.info("Copying from folder: {0}", root_folder) - conf_copy_id = root_folder - files = os.listdir(conf_copy_id) - for filename in files: - # Allow only those files in the list - Log.info("Copying filename: {0}", filename) - if filename in conf_copy_filter: - if os.path.isfile(os.path.join(conf_copy_id, filename)): - new_filename = filename.replace( - copy_id, exp_id) - if new_filename[-4:] == "conf": - new_filename = new_filename[:-4] + "yml" - # Using readlines for replacement handling - content = open(os.path.join( - conf_copy_id, filename), 'r').readlines() - - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile( - BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').readlines() - # Setting email notifications to false - if filename.startswith("autosubmit") and filename.endswith("conf"): - content = ["NOTIFICATIONS = False\n" if line.startswith( - ("NOTIFICATIONS =", "notifications =")) else line for line in content] - elif filename.startswith("autosubmit") and filename.endswith("yml"): - content = [" NOTIFICATIONS: False\n" if "NOTIFICATIONS" in line else line for - line in content] - # Putting content together before writing - sep = "" - open(os.path.join(dir_exp_id, "conf", - new_filename), 'w').write(sep.join(content)) - if filename.endswith("conf"): - try: - AutosubmitConfig.ini_to_yaml(os.path.join(dir_exp_id, "conf"), - os.path.join(os.path.join(dir_exp_id, "conf"), - new_filename)) - except BaseException as e: - Log.warning("Couldn't convert conf file to yml: {0}", new_filename) - if filename in conf_copy_filter_folder: - if os.path.isfile(os.path.join(conf_copy_id, filename)): - new_filename = filename.split( - ".")[0] + "_" + exp_id + ".yml" - content = open(os.path.join( - conf_copy_id, filename), 'rb').read() - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile( - BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').read() - - open(os.path.join(dir_exp_id, "conf", - new_filename), 'wb').write(content) - - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy, copy_id) - ##### - autosubmit_config = AutosubmitConfig( - exp_id, BasicConfig, YAMLParserFactory()) - try: - autosubmit_config.check_conf_files(False) - project_type = autosubmit_config.get_project_type() - if project_type == "git": - autosubmit_git = AutosubmitGit(copy_id[0]) - Log.info("checking model version...") - if not autosubmit_git.check_commit(autosubmit_config): - raise AutosubmitCritical( - "Uncommitted changes", 7013) - except Exception as error: - Log.warning( - "Trace: {0}\nCouldn't load experiment configuration, check the experiments files before perform a create".format( - str(error))) - else: - raise AutosubmitCritical( - "The experiment directory doesn't exist", 7012) - except (OSError, IOError) as e: - try: - Autosubmit._delete_expid(exp_id, True) - except Exception as e: - pass - raise AutosubmitCritical( - "Can not create experiment", 7012) - except BaseException as e: - try: - Autosubmit._delete_expid(exp_id, True) - except Exception as e: - pass - raise AutosubmitCritical( - "Can not create experiment", 7012) - - Log.debug("Creating temporal directory...") - exp_id_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id) - tmp_path = os.path.join(exp_id_path, "tmp") - os.mkdir(tmp_path) - os.chmod(tmp_path, 0o775) - os.mkdir(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR)) - os.chmod(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR), 0o775) - Log.debug("Creating temporal remote directory...") - remote_tmp_path = os.path.join(tmp_path, "LOG_" + exp_id) - os.mkdir(remote_tmp_path) - os.chmod(remote_tmp_path, 0o755) - - Log.debug("Creating pkl directory...") - os.mkdir(os.path.join(exp_id_path, "pkl")) - - Log.debug("Creating plot directory...") - os.mkdir(os.path.join(exp_id_path, "plot")) - os.chmod(os.path.join(exp_id_path, "plot"), 0o775) - Log.result("Experiment registered successfully") - Log.warning("Remember to MODIFY the config files!") + # Create a new experiment from scratch + exp_id = new_experiment(description, Autosubmit.autosubmit_version, test, operational, minimal_configuration) + + Autosubmit.as_conf_default_values(exp_id,hpc) + if exp_id == '': + raise AutosubmitCritical("No expid", 7011) + except Exception as e: + raise AutosubmitCritical("Error while generating a new experiment in the db: {0}".format(str(e)), 7011) + + # Create the experiment structure + Log.info("Generating folder structure...") + + exp_folder = os.path.join(root_folder, exp_id) + try: + os.mkdir(exp_folder) + os.mkdir(os.path.join(exp_folder, "conf")) + os.mkdir(os.path.join(exp_folder, "pkl")) + os.mkdir(os.path.join(exp_folder, "tmp")) + os.mkdir(os.path.join(exp_folder, "tmp", "ASLOGS")) + os.mkdir(os.path.join(exp_folder, "tmp", "LOG_"+exp_id)) + os.mkdir(os.path.join(exp_folder, "plot")) + os.mkdir(os.path.join(exp_folder, "status")) + os.mkdir(os.path.join(exp_folder, "proj")) + # Setting permissions + os.chmod(exp_folder, 0o755) + os.chmod(os.path.join(exp_folder, "conf"), 0o755) + os.chmod(os.path.join(exp_folder, "pkl"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "ASLOGS"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "LOG_"+exp_id), 0o755) + os.chmod(os.path.join(exp_folder, "plot"), 0o755) + os.chmod(os.path.join(exp_folder, "status"), 0o755) + os.chmod(os.path.join(exp_folder, "proj"), 0o755) + except OSError as e: try: - Log.debug("Setting the right permissions...") - os.chmod(os.path.join(exp_id_path, "conf"), 0o755) - os.chmod(os.path.join(exp_id_path, "pkl"), 0o755) - os.chmod(os.path.join(exp_id_path, "tmp"), 0o775) - os.chmod(os.path.join(exp_id_path, "plot"), 0o775) - os.chmod(os.path.join(exp_id_path, "conf/autosubmit_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/expdef_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/jobs_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/platforms_" + - str(exp_id) + ".yml"), 0o755) - try: - os.chmod(os.path.join(exp_id_path, "tmp/ASLOGS"), 0o755) - except Exception as e: - pass - os.chmod(os.path.join(exp_id_path, "conf/proj_" + - str(exp_id) + ".yml"), 0o755) - except Exception as e: - pass # some folder may no exists, like proj - Log.debug("Finished") + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while creating the experiment structure: {0}".format(str(e)), 7011) - return exp_id - except AutosubmitCritical: - raise + # Create the experiment configuration + Log.info("Generating config files...") + try: + if copy_id != '': + # Copy the configuration from selected experiment + Autosubmit.copy_as_config(exp_id, copy_id) + else: + # Create a new configuration + Autosubmit.generate_as_config(exp_id,dummy, minimal_configuration) + except Exception as e: + try: + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while creating the experiment configuration: {0}".format(str(e)), 7011) + # Change template values by default values specified from the commandline + try: + Autosubmit.as_conf_default_values(exp_id,hpc,minimal_configuration,git_repo,git_branch,git_as_conf) except Exception as e: - raise AutosubmitCritical("Couldn't create a new experiment", 7012, str(e)) + try: + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while setting the default values: {0}".format(str(e)), 7011) + + Log.result("Experiment {0} created".format(exp_id)) + return exp_id @staticmethod def delete(expid, force): @@ -1651,11 +1543,11 @@ class Autosubmit: @staticmethod def run_experiment(expid, notransitive=False, update_version=False, start_time=None, start_after=None, - run_members=None): + run_only_members=None): """ Runs and experiment (submitting all the jobs properly and repeating its execution in case of failure). - :param run_members: + :param run_only_ºmembers: :param start_after: :param start_time: :param update_version: @@ -1682,8 +1574,8 @@ class Autosubmit: # Start after completion trigger block AutosubmitHelper.handle_start_after(start_after, expid, BasicConfig()) - # Handling run_members - allowed_members = AutosubmitHelper.get_allowed_members(run_members, as_conf) + # Handling run_only_members + allowed_members = AutosubmitHelper.get_allowed_members(run_only_members, as_conf) except AutosubmitCritical as e: raise except BaseException as e: @@ -5449,43 +5341,6 @@ class Autosubmit: except ValueError: sys.stdout.write('Please respond with \'y\' or \'n\'.\n') - @staticmethod - def _prepare_conf_files(exp_id, hpc, autosubmit_version, dummy, copy_id): - """ - Changes default configuration files to match new experiment values - - :param exp_id: experiment identifier - :type exp_id: str - :param hpc: hpc to use - :type hpc: str - :param autosubmit_version: current autosubmit's version - :type autosubmit_version: str - :param dummy: if True, creates a dummy experiment adding some default values - :type dummy: bool - """ - as_conf = AutosubmitConfig(exp_id, BasicConfig, YAMLParserFactory()) - as_conf.set_version(autosubmit_version) - update_experiment_descrip_version(exp_id, version=Autosubmit.autosubmit_version) - as_conf.set_expid(exp_id) - as_conf.set_platform(hpc) - - if dummy: - content = open(as_conf.experiment_file).read() - - # Experiment - content = content.replace(re.search('DATELIST: .*', content, re.MULTILINE).group(0), - "DATELIST: 20000101") - content = content.replace(re.search('MEMBERS: .*', content, re.MULTILINE).group(0), - "MEMBERS: fc0") - content = content.replace(re.search('CHUNKSIZE: .*', content, re.MULTILINE).group(0), - "CHUNKSIZE: 4") - content = content.replace(re.search('NUMCHUNKS: .*', content, re.MULTILINE).group(0), - "NUMCHUNKS: 2") - content = content.replace(re.search('PROJECT_TYPE: .*', content, re.MULTILINE).group(0), - "PROJECT_TYPE: 'none'") - - open(as_conf.experiment_file, 'w').write(content) - @staticmethod def _get_status(s): """ diff --git a/autosubmit/experiment/experiment_common.py b/autosubmit/experiment/experiment_common.py index 52ff8b91b..17eba23fc 100644 --- a/autosubmit/experiment/experiment_common.py +++ b/autosubmit/experiment/experiment_common.py @@ -29,14 +29,14 @@ def new_experiment(description, version, test=False, operational=False): """ Stores a new experiment on the database and generates its identifier - :param version: autosubmit version associated to the experiment + :param description: description of the experiment + :type description: str + :param version: version of the experiment :type version: str - :param test: flag for test experiments + :param test: if True, the experiment is a test experiment :type test: bool - :param operational: flag for operational experiments + :param operational: if True, the experiment is an operational experiment :type operational: bool - :param description: experiment's description - :type description: str :return: experiment id for the new experiment :rtype: str """ diff --git a/docs/source/userguide/create/index.rst b/docs/source/userguide/create/index.rst index 23cbf75e6..70763af34 100644 --- a/docs/source/userguide/create/index.rst +++ b/docs/source/userguide/create/index.rst @@ -15,7 +15,7 @@ To create a new experiment, just run the command: Options: :: - usage: autosubmit expid [-h] [-y COPY | -dm] [-p PATH] -H HPC -d DESCRIPTION + usage: autosubmit expid [-h] [-y COPY | -dm | -min [-repo GIT_PATH ] ] [-p PATH] -H HPC -d DESCRIPTION -h, --help show this help message and exit -y COPY, --copy COPY makes a copy of the specified experiment @@ -23,12 +23,14 @@ Options: -H HPC, --HPC HPC specifies the HPC to use for the experiment -d DESCRIPTION, --description DESCRIPTION sets a description for the experiment to store in the database. - -c PATH, --config PATH - if specified, copies config files from a folder + -min,--minimal_configuration creates a minimal configuration file + -repo GIT_PATH, --git_repo GIT_PATH sets the git_repository + Example: :: autosubmit expid --HPC ithaca --description "experiment is about..." + autosubmit expid --minimal_config -repo --description "experiment is about..." If there is an autosubmitrc or .autosubmitrc file in your home directory (cd ~), you can setup a default file from where the contents of platforms_expid.yml should be copied. -- GitLab From 9539b7b4eb3782ac32ac79815fe1aafe0e4cfc7b Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 11:39:39 +0100 Subject: [PATCH 03/26] -Copy working --- autosubmit/autosubmit.py | 45 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1abb83cf9..271e56b8b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -638,7 +638,7 @@ class Autosubmit: return Autosubmit.run_experiment(args.expid, args.notransitive, args.update_version, args.start_time, args.start_after, args.run_only_members) elif args.command == 'expid': - return Autosubmit.expid(args.description,args.HPC,args.copy, args.dummy,args.minimal_configuration,args.git_repo,args.git_branch,args.git_as_conf,args.test,args.operational) != '' + return Autosubmit.expid(args.description,args.HPC,args.copy, args.dummy,args.minimal_configuration,args.git_repo,args.git_branch,args.git_as_conf,args.operational) != '' elif args.command == 'delete': return Autosubmit.delete(args.expid, args.force) elif args.command == 'monitor': @@ -1016,8 +1016,7 @@ class Autosubmit: @staticmethod def copy_as_config(exp_id,copy_id): - as_conf_files = os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id)) - for conf_file in as_conf_files: + for conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id,"conf")): # Copy only relevant files if conf_file.endswith(".conf") or conf_file.endswith(".yml") or conf_file.endswith(".yaml"): shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), @@ -1062,22 +1061,37 @@ class Autosubmit: with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'r') as f: # Copied files could not have default names. content = f.read() - content = content.replace(re.search('AUTOSUBMIT_VERSION: .*', content, re.MULTILINE).group(0), "AUTOSUBMIT_VERSION: \""+Autosubmit.autosubmit_version+"\"") - content = content.replace(re.search('NOTIFICATIONS: .*', content, re.MULTILINE).group(0), - "NOTIFICATIONS: False") - content = content.replace(re.search('TO: .*', content, re.MULTILINE).group(0), "TO: \"\"") - content = content.replace(re.search('EXPID: .*', content, re.MULTILINE).group(0),"EXPID: \""+exp_id+"\"") - content = content.replace(re.search('HPCARCH: .*', content, re.MULTILINE).group(0),"HPCARCH: \""+hpc+"\"") + search = re.search('AUTOSUBMIT_VERSION: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "AUTOSUBMIT_VERSION: \""+Autosubmit.autosubmit_version+"\"") + search = re.search('NOTIFICATIONS: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"NOTIFICATIONS: False") + search = re.search('TO: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "TO: \"\"") + search = re.search('EXPID: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"EXPID: \""+exp_id+"\"") + search = re.search('HPCARCH: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"HPCARCH: \""+hpc+"\"") if minimal_configuration: - content = content.replace(re.search('CUSTOM_CONFIG: .*', content, re.MULTILINE).group(0), "CUSTOM_CONFIG: \"%ROOTDIR%/proj/git_project/"+git_as_conf+"\"") - content = content.replace(re.search('PROJECT_ORIGIN: .*', content, re.MULTILINE).group(0), "PROJECT_ORIGIN: \""+git_repo+"\"") - content = content.replace(re.search('PROJECT_BRANCH: .*', content, re.MULTILINE).group(0), "PROJECT_BRANCH: \""+git_branch+"\"") + search = re.search('CUSTOM_CONFIG: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "CUSTOM_CONFIG: \"%ROOTDIR%/proj/git_project/"+git_as_conf+"\"") + search = re.search('PROJECT_ORIGIN: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "PROJECT_ORIGIN: \""+git_repo+"\"") + search = re.search('PROJECT_BRANCH: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "PROJECT_BRANCH: \""+git_branch+"\"") with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'w') as f: f.write(content) @staticmethod - def expid(description,hpc="", copy_id='', dummy=False,minimal_configuration=False,git_repo="",git_branch="",git_as_conf="",test=False, operational=False): + def expid(description,hpc="", copy_id='', dummy=False,minimal_configuration=False,git_repo="",git_branch="",git_as_conf="",operational=False): """ Creates a new experiment for given HPC description: description of the experiment @@ -1106,12 +1120,11 @@ class Autosubmit: if not os.path.exists(copy_id_folder): raise AutosubmitCritical( "Experiment {0} doesn't exists".format(copy_id), 7011) - exp_id = copy_experiment(copy_id, description, hpc, test, operational, copy_id_folder) + exp_id = copy_experiment(copy_id, description, Autosubmit.autosubmit_version, False, operational) else: # Create a new experiment from scratch - exp_id = new_experiment(description, Autosubmit.autosubmit_version, test, operational, minimal_configuration) + exp_id = new_experiment(description, Autosubmit.autosubmit_version, False, operational, minimal_configuration) - Autosubmit.as_conf_default_values(exp_id,hpc) if exp_id == '': raise AutosubmitCritical("No expid", 7011) except Exception as e: -- GitLab From cdefeaa2e528d97e734d3ce7fe57173fe5672478 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 12:00:57 +0100 Subject: [PATCH 04/26] -min working in expdef --- autosubmit/autosubmit.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 271e56b8b..23cc1cebe 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -213,15 +213,15 @@ class Autosubmit: help='creates a new experiment with default values, usually for testing') group.add_argument('-min', '--minimal_configuration', action='store_true', help='creates a new experiment with minimal configuration, usually combined with -repo') - subparser.add_argument('-repo', '--git_repo', type=str, required=False, + subparser.add_argument('-repo', '--git_repo', type=str, default="", required=False, help='sets a git repository for the experiment') - subparser.add_argument('-b', '--git_branch', type=str, required=False, + subparser.add_argument('-b', '--git_branch', type=str, default="", required=False, help='sets a git branch for the experiment') - subparser.add_argument('-conf', '--git_as_conf', type=str, required=False,help='sets the git path to as_conf') + subparser.add_argument('-conf', '--git_as_conf', type=str, default="", required=False,help='sets the git path to as_conf') group.add_argument('-op', '--operational', action='store_true', help='creates a new experiment with operational experiment id') - subparser.add_argument('-H', '--HPC', required=True, + subparser.add_argument('-H', '--HPC', required=False, default="local", help='specifies the HPC to use for the experiment') subparser.add_argument('-d', '--description', type=str, required=True, help='sets a description for the experiment to store in the database.') @@ -1036,13 +1036,13 @@ class Autosubmit: for as_conf_file in files: if dummy: if as_conf_file.endswith("dummy.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) elif minimal_configuration: if as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) else: if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ @@ -1115,7 +1115,7 @@ class Autosubmit: # Register the experiment in the database try: # Copy another experiment from the database - if copy_id != '': + if copy_id != '' and copy_id is not None: copy_id_folder = os.path.join(root_folder, copy_id) if not os.path.exists(copy_id_folder): raise AutosubmitCritical( @@ -1123,7 +1123,7 @@ class Autosubmit: exp_id = copy_experiment(copy_id, description, Autosubmit.autosubmit_version, False, operational) else: # Create a new experiment from scratch - exp_id = new_experiment(description, Autosubmit.autosubmit_version, False, operational, minimal_configuration) + exp_id = new_experiment(description, Autosubmit.autosubmit_version, False, operational) if exp_id == '': raise AutosubmitCritical("No expid", 7011) @@ -1164,7 +1164,7 @@ class Autosubmit: # Create the experiment configuration Log.info("Generating config files...") try: - if copy_id != '': + if copy_id != '' and copy_id is not None: # Copy the configuration from selected experiment Autosubmit.copy_as_config(exp_id, copy_id) else: -- GitLab From 19a9105934bbf3f43a8d351af8faecf96779dba5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 12:54:19 +0100 Subject: [PATCH 05/26] -min working --- autosubmit/autosubmit.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 23cc1cebe..5e1c26801 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1020,14 +1020,14 @@ class Autosubmit: # Copy only relevant files if conf_file.endswith(".conf") or conf_file.endswith(".yml") or conf_file.endswith(".yaml"): shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), - os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file)) + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file.replace(copy_id,exp_id))) # if ends with .conf convert it to AS4 yaml file if conf_file.endswith(".conf"): try: AutosubmitConfig.ini_to_yaml(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf"), - os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file)) + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file.replace(copy_id,exp_id))) except Exception as e: - Log.warning("Error converting {0} to yml: {1}".format(conf_file,str(e))) + Log.warning("Error converting {0} to yml: {1}".format(conf_file.replace(copy_id,exp_id),str(e))) @staticmethod def generate_as_config(exp_id,dummy=False,minimal_configuration=False): # obtain from autosubmitconfigparser package @@ -1036,13 +1036,13 @@ class Autosubmit: for as_conf_file in files: if dummy: if as_conf_file.endswith("dummy.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) elif minimal_configuration: if as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) else: if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file[:-4]+"_"+exp_id+".yml")) @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ @@ -1140,20 +1140,18 @@ class Autosubmit: os.mkdir(os.path.join(exp_folder, "pkl")) os.mkdir(os.path.join(exp_folder, "tmp")) os.mkdir(os.path.join(exp_folder, "tmp", "ASLOGS")) - os.mkdir(os.path.join(exp_folder, "tmp", "LOG_"+exp_id)) + os.mkdir(os.path.join(exp_folder, "tmp", "LOG_"+exp_id.upper())) os.mkdir(os.path.join(exp_folder, "plot")) os.mkdir(os.path.join(exp_folder, "status")) - os.mkdir(os.path.join(exp_folder, "proj")) # Setting permissions os.chmod(exp_folder, 0o755) os.chmod(os.path.join(exp_folder, "conf"), 0o755) os.chmod(os.path.join(exp_folder, "pkl"), 0o755) os.chmod(os.path.join(exp_folder, "tmp"), 0o755) os.chmod(os.path.join(exp_folder, "tmp", "ASLOGS"), 0o755) - os.chmod(os.path.join(exp_folder, "tmp", "LOG_"+exp_id), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "LOG_"+exp_id.upper()), 0o755) os.chmod(os.path.join(exp_folder, "plot"), 0o755) os.chmod(os.path.join(exp_folder, "status"), 0o755) - os.chmod(os.path.join(exp_folder, "proj"), 0o755) except OSError as e: try: Autosubmit._delete_expid(exp_id, True) -- GitLab From b3b45de68aa2b8837c0b5dd4a7ac77f710972fbc Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 13:00:23 +0100 Subject: [PATCH 06/26] min rst --- docs/source/userguide/create/index.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/source/userguide/create/index.rst b/docs/source/userguide/create/index.rst index 70763af34..84e416797 100644 --- a/docs/source/userguide/create/index.rst +++ b/docs/source/userguide/create/index.rst @@ -19,18 +19,26 @@ Options: -h, --help show this help message and exit -y COPY, --copy COPY makes a copy of the specified experiment + -op, -operational creates a new experiment, starting with "o" -dm, --dummy creates a new experiment with default values, usually for testing + -min, --minimal_config + creates a new experiment with minimal configuration files, usually for using a custom configuration + -repo GIT_PATH, --git_repo GIT_PATH + sets the git_repository + -b BRANCH, --git_branch BRANCH + sets the branch to use for the git repository + -config, --git_as_conf + sets the configuration folder to use for the experiment -H HPC, --HPC HPC specifies the HPC to use for the experiment -d DESCRIPTION, --description DESCRIPTION sets a description for the experiment to store in the database. - -min,--minimal_configuration creates a minimal configuration file -repo GIT_PATH, --git_repo GIT_PATH sets the git_repository Example: :: autosubmit expid --HPC ithaca --description "experiment is about..." - autosubmit expid --minimal_config -repo --description "experiment is about..." + autosubmit expid -min -repo https://earth.bsc.es/gitlab/ces/auto-advanced_config_example -b main -conf as_conf -d "minimal config example" If there is an autosubmitrc or .autosubmitrc file in your home directory (cd ~), you can setup a default file from where the contents of platforms_expid.yml should be copied. -- GitLab From 73f6e1d73119cf68670c04635203dde0d55e03a0 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 16 Feb 2023 09:44:30 +0100 Subject: [PATCH 07/26] [wip] config changes --- docs/source/userguide/create/index.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/userguide/create/index.rst b/docs/source/userguide/create/index.rst index 84e416797..7ed5a54f0 100644 --- a/docs/source/userguide/create/index.rst +++ b/docs/source/userguide/create/index.rst @@ -28,8 +28,8 @@ Options: -b BRANCH, --git_branch BRANCH sets the branch to use for the git repository -config, --git_as_conf - sets the configuration folder to use for the experiment - -H HPC, --HPC HPC specifies the HPC to use for the experiment + sets the configuration folder to use for the experiment, relative to repo root + -H HPC, --HPC HPC specifies the HPC to use for the experiment, default is localhost -d DESCRIPTION, --description DESCRIPTION sets a description for the experiment to store in the database. -repo GIT_PATH, --git_repo GIT_PATH sets the git_repository @@ -37,8 +37,10 @@ Options: Example: :: - autosubmit expid --HPC ithaca --description "experiment is about..." + autosubmit expid --HPC marenostrum4 --description "experiment is about..." autosubmit expid -min -repo https://earth.bsc.es/gitlab/ces/auto-advanced_config_example -b main -conf as_conf -d "minimal config example" + autosubmit expid -dm -d "dummy test" + If there is an autosubmitrc or .autosubmitrc file in your home directory (cd ~), you can setup a default file from where the contents of platforms_expid.yml should be copied. -- GitLab From 9d5a7f591c8364c94f65b0a0b20e1bf0024ce3fa Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 21 Feb 2023 15:35:34 +0100 Subject: [PATCH 08/26] Still some bugs --- autosubmit/autosubmit.py | 22 ++++++++++++---------- autosubmit/git/autosubmit_git.py | 4 ++-- autosubmit/job/job.py | 2 +- autosubmit/job/job_list.py | 2 +- autosubmit/statistics/test.py | 2 +- autosubmit4-config-parser | 2 +- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 5e1c26801..b9cf3b690 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -773,7 +773,7 @@ class Autosubmit: raise AutosubmitCritical(message, 7071) if expid != 'None' and args.command not in expid_less and args.command not in global_log_command: as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) if len(as_conf.experiment_data) == 0: if args.command not in ["expid", "upgrade"]: raise AutosubmitCritical( @@ -1941,7 +1941,7 @@ class Autosubmit: consecutive_retrials = consecutive_retrials + 1 Log.info("Waiting {0} seconds before continue".format(delay)) try: - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) Log.info("Recovering job_list...") job_list = Autosubmit.load_job_list( expid, as_conf, notransitive=notransitive) @@ -3819,7 +3819,7 @@ class Autosubmit: try: Autosubmit._check_ownership(expid, raise_error=True) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - # as_conf.reload(first_load=True) + # as_conf.reload(force_load=True) as_conf.check_conf_files(refresh=True) except (AutosubmitError, AutosubmitCritical): raise @@ -3849,7 +3849,7 @@ class Autosubmit: Autosubmit._check_ownership(expid, raise_error=True) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) as_conf.check_expdef_conf() Log.info("Changing {0} experiment version from {1} to {2}", @@ -3968,7 +3968,7 @@ class Autosubmit: except Exception as e: Log.warning("Couldn't convert conf file to yml: {0}", Path(f).parent) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) # Load current variables as_conf.check_conf_files() # Load current parameters ( this doesn't read job parameters) @@ -4302,8 +4302,8 @@ class Autosubmit: @staticmethod def _create_project_associated_conf(as_conf, force_model_conf, force_jobs_conf): - project_destiny = as_conf.project_file - jobs_destiny = as_conf.jobs_file + project_destiny = as_conf.get_file_project_conf() + jobs_destiny = as_conf.get_file_jobs_conf() if as_conf.get_project_type() != 'none': if as_conf.get_file_project_conf(): @@ -4374,9 +4374,11 @@ class Autosubmit: project_type = as_conf.get_project_type() # Getting output type provided by the user in config, 'pdf' as default output_type = as_conf.get_output_type() - - if not Autosubmit._copy_code(as_conf, expid, project_type, False): - return False + try: + if not Autosubmit._copy_code(as_conf, expid, project_type, False): + return False + except: + raise AutosubmitCritical("Error obtaining the project data, check the parameters related to PROJECT and GIT/SVN or LOCAL sections", code=7014) # Update configuration with the new config in the dist ( if any ) as_conf.check_conf_files(False) diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 34e95df08..7a292ce6f 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . - +import locale from os import path import os from shutil import rmtree @@ -205,7 +205,7 @@ class AutosubmitGit: Log.debug('Clone command: {0}', command_0) try: git_version = subprocess.check_output("git --version",shell=True) - git_version = git_version.split(" ")[2].strip("\n") + git_version = git_version.split(" ")[2].decode(locale.getlocale()[1]).strip("\n") version_int = "" for number in git_version.split("."): version_int += number diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 57bdca689..0759f8d4a 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -655,7 +655,7 @@ class Job(object): while (count < retries) or not success: try: as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) max_retrials = as_conf.get_retrials() max_logs = int(as_conf.get_retrials()) - fail_count last_log = int(as_conf.get_retrials()) - fail_count diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index b92c89260..2eb4d6968 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -2085,7 +2085,7 @@ class JobList(object): :param as_conf: experiment configuration :type as_conf: AutosubmitConfig """ - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) out = True for job in self._job_list: show_logs = job.check_warnings diff --git a/autosubmit/statistics/test.py b/autosubmit/statistics/test.py index d337d29d8..ccb6ce54c 100644 --- a/autosubmit/statistics/test.py +++ b/autosubmit/statistics/test.py @@ -27,7 +27,7 @@ class TestStatistics(unittest.TestCase): path_structure = BasicConfig.STRUCTURES_DIR path_local_root = BasicConfig.LOCAL_ROOT_DIR as_conf = AutosubmitConfig(expid) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) job_list = Autosubmit.load_job_list(expid, as_conf, False) jobs_considered = [job for job in job_list.get_job_list() if job.status not in [ Status.READY, Status.WAITING]] diff --git a/autosubmit4-config-parser b/autosubmit4-config-parser index 072d997cb..34523d8ad 160000 --- a/autosubmit4-config-parser +++ b/autosubmit4-config-parser @@ -1 +1 @@ -Subproject commit 072d997cb4822cc3958e12282a16181dac9a2db0 +Subproject commit 34523d8ad39db15e8986c7ab74db63e4fbc8e68c -- GitLab From f81d19d367743218d3d28c42232ebd8527c13746 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 22 Feb 2023 14:51:19 +0100 Subject: [PATCH 09/26] Recursive config working --- autosubmit/job/job.py | 2 +- autosubmit/job/job_list.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 0759f8d4a..4e113bbb2 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -994,7 +994,7 @@ class Job(object): """ chunk = 1 as_conf.reload() - #parameters = as_conf.sustitute_dynamic_variables(parameters,25) + parameters = as_conf.sustitute_dynamic_variables(parameters,25) parameters = parameters.copy() parameters.update(default_parameters) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 2eb4d6968..f7f8b8f96 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -285,9 +285,15 @@ class JobList(object): dependencies_keys = jobs_data[job_section].get(option,{}) if type(dependencies_keys) is str: - dependencies_keys = dependencies_keys.split() + if "," in dependencies_keys: + dependencies_list = dependencies_keys.split(",") + else: + dependencies_list = dependencies_keys.split(" ") + dependencies_keys = {} + for dependency in dependencies_list: + dependencies_keys[dependency] = {} if dependencies_keys is None: - dependencies_keys = [] + dependencies_keys = {} dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs, job_section) for job in dic_jobs.get_jobs(job_section): -- GitLab From a7240015c662f3af252c6179191fc454bf696de7 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 22 Feb 2023 15:47:02 +0100 Subject: [PATCH 10/26] Now autosubmit will ignore -minimal files. Now autosubmit will create a version.yml if AUTOSUBMIT_VERSION is not found --- autosubmit/autosubmit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index b9cf3b690..23719525d 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1039,7 +1039,7 @@ class Autosubmit: shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) elif minimal_configuration: if as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf","minimal.yml")) else: if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file[:-4]+"_"+exp_id+".yml")) @@ -1577,7 +1577,7 @@ class Autosubmit: raise AutosubmitCritical("Failure during the loading of the experiment configuration, check file paths", 7014, str(e)) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(True, True) + as_conf.check_conf_files(running_time=True, force_load=True) try: # Handling starting time AutosubmitHelper.handle_start_time(start_time) -- GitLab From 073889fbbe2dab796376902f0b13c818b3815356 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 13 Feb 2023 13:29:09 +0100 Subject: [PATCH 11/26] resolves #829, --- autosubmit/autosubmit.py | 3 ++- autosubmit/helpers/autosubmit_helper.py | 2 +- docs/source/userguide/run/index.rst | 14 +++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 63da42450..6fad187e8 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -199,9 +199,10 @@ class Autosubmit: help='Sets the starting time for this experiment') subparser.add_argument('-sa', '--start_after', required=False, help='Sets a experiment expid which completion will trigger the start of this experiment.') - subparser.add_argument('-rm', '--run_members', required=False, + subparser.add_argument('-rom', '--run_only_members', required=False, help='Sets members allowed on this run.') + # Expid subparser = subparsers.add_parser( 'expid', description="Creates a new experiment") diff --git a/autosubmit/helpers/autosubmit_helper.py b/autosubmit/helpers/autosubmit_helper.py index 932c07e9e..2aef35c49 100644 --- a/autosubmit/helpers/autosubmit_helper.py +++ b/autosubmit/helpers/autosubmit_helper.py @@ -104,6 +104,6 @@ def get_allowed_members(run_members, as_conf): raise AutosubmitCritical("Some of the members ({0}) in the list of allowed members you supplied do not exist in the current list " + "of members specified in the conf files.\nCurrent list of members: {1}".format(str(rmember), str(as_conf.get_member_list()))) if len(allowed_members) == 0: - raise AutosubmitCritical("Not a valid -rm --run_members input: {0}".format(str(run_members))) + raise AutosubmitCritical("Not a valid -rom --run_only_members input: {0}".format(str(run_members))) return allowed_members return [] \ No newline at end of file diff --git a/docs/source/userguide/run/index.rst b/docs/source/userguide/run/index.rst index b03a1cdac..f38ede902 100644 --- a/docs/source/userguide/run/index.rst +++ b/docs/source/userguide/run/index.rst @@ -28,7 +28,7 @@ Options: Sets the starting time for the experiment. Accepted format: 'yyyy-mm-dd HH:MM:SS' or 'HH:MM:SS' (defaults to current day). -sa --start_after Sets a experiment expid that will be tracked for completion. When this experiment is completed, the current instance of Autosubmit run will start. - -rm --run_members + -rom,--run_only_members --run_members Sets a list of members allowed to run. The list must have the format '### ###' where '###' represents the name of the member as set in the conf files. -h, --help show this help message and exit @@ -120,7 +120,7 @@ To run only a subset of selected members you can execute the command: # Add your key to ssh agent ( if encrypted ) ssh-add ~/.ssh/id_rsa - autosubmit run EXPID -rm MEMBERS + autosubmit run EXPID -rom MEMBERS *EXPID* is the experiment identifier, the experiment you want to run. @@ -128,17 +128,17 @@ To run only a subset of selected members you can execute the command: Then, your experiment will start running jobs belonging to those members only. If the experiment was previously running and autosubmit was stopped when some jobs belonging to other members (not the ones from your input) where running, those jobs will be tracked and finished in the new exclusive run. -Furthermore, if you wish to run a sequence of only members execution; then, instead of running `autosubmit run -rm "member_1"` ... `autosubmit run -rm "member_n"`, you can make a bash file with that sequence and run the bash file. Example: +Furthermore, if you wish to run a sequence of only members execution; then, instead of running `autosubmit run -rom "member_1"` ... `autosubmit run -rom "member_n"`, you can make a bash file with that sequence and run the bash file. Example: .. code-block:: bash # Add your key to ssh agent ( if encrypted ) ssh-add ~/.ssh/id_rsa - autosubmit run EXPID -rm MEMBER_1 - autosubmit run EXPID -rm MEMBER_2 - autosubmit run EXPID -rm MEMBER_3 + autosubmit run EXPID -rom MEMBER_1 + autosubmit run EXPID -rom MEMBER_2 + autosubmit run EXPID -rom MEMBER_3 ... - autosubmit run EXPID -rm MEMBER_N + autosubmit run EXPID -rom MEMBER_N How to start an experiment at a given time ------------------------------------------ -- GitLab From 64885968fc9eb2f808efd187162eafa896dc95a7 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 11:02:24 +0100 Subject: [PATCH 12/26] Expdef refactored. Added minimal, dummy confs TODO add comments to non-dummy/minimal conf --- autosubmit/autosubmit.py | 477 +++++++-------------- autosubmit/experiment/experiment_common.py | 10 +- docs/source/userguide/create/index.rst | 7 +- 3 files changed, 175 insertions(+), 319 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 6fad187e8..0834bf96b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -87,7 +87,7 @@ import signal import datetime import portalocker -from pkg_resources import require, resource_listdir, resource_exists, resource_string +from pkg_resources import require, resource_listdir, resource_exists, resource_string, resource_filename from collections import defaultdict from pyparsing import nestedExpr from .history.experiment_status import ExperimentStatus @@ -211,14 +211,21 @@ class Autosubmit: '-y', '--copy', help='makes a copy of the specified experiment') group.add_argument('-dm', '--dummy', action='store_true', help='creates a new experiment with default values, usually for testing') + group.add_argument('-min', '--minimal_configuration', action='store_true', + help='creates a new experiment with minimal configuration, usually combined with -repo') + subparser.add_argument('-repo', '--git_repo', type=str, required=False, + help='sets a git repository for the experiment') + subparser.add_argument('-b', '--git_branch', type=str, required=False, + help='sets a git branch for the experiment') + subparser.add_argument('-conf', '--git_as_conf', type=str, required=False,help='sets the git path to as_conf') + group.add_argument('-op', '--operational', action='store_true', help='creates a new experiment with operational experiment id') subparser.add_argument('-H', '--HPC', required=True, help='specifies the HPC to use for the experiment') subparser.add_argument('-d', '--description', type=str, required=True, help='sets a description for the experiment to store in the database.') - subparser.add_argument('-c', '--config', type=str, required=False, - help='defines where are located the configuration files.') + # Delete subparser = subparsers.add_parser( 'delete', description="delete specified experiment") @@ -629,10 +636,9 @@ class Autosubmit: if args.command == 'run': return Autosubmit.run_experiment(args.expid, args.notransitive, args.update_version, args.start_time, - args.start_after, args.run_members) + args.start_after, args.run_only_members) elif args.command == 'expid': - return Autosubmit.expid(args.HPC, args.description, args.copy, args.dummy, False, - args.operational, args.config) != '' + return Autosubmit.expid(args.description,args.HPC,args.copy, args.dummy,args.minimal_configuration,args.git_repo,args.git_branch,args.git_as_conf,args.test,args.operational) != '' elif args.command == 'delete': return Autosubmit.delete(args.expid, args.force) elif args.command == 'monitor': @@ -1009,280 +1015,166 @@ class Autosubmit: error_message, 6004) @staticmethod - def expid(hpc, description, copy_id='', dummy=False, test=False, operational=False, root_folder=''): + def copy_as_config(exp_id,copy_id): + as_conf_files = os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id)) + for conf_file in as_conf_files: + # Copy only relevant files + if conf_file.endswith(".conf") or conf_file.endswith(".yml") or conf_file.endswith(".yaml"): + shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file)) + # if ends with .conf convert it to AS4 yaml file + if conf_file.endswith(".conf"): + try: + AutosubmitConfig.ini_to_yaml(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf"), + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file)) + except Exception as e: + Log.warning("Error converting {0} to yml: {1}".format(conf_file,str(e))) + @staticmethod + def generate_as_config(exp_id,dummy=False,minimal_configuration=False): + # obtain from autosubmitconfigparser package + # get all as_conf_files from autosubmitconfigparser package + files = resource_listdir('autosubmitconfigparser.config', 'files') + for as_conf_file in files: + if dummy: + if as_conf_file.endswith("dummy.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + elif minimal_configuration: + if as_conf_file.endswith("minimal.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + else: + if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + @staticmethod + def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): + """ + Replace default values in as_conf files + :param exp_id: experiment id + :param hpc: platform + :param minimal_configuration: minimal configuration + :param git_repo: path to project git repository + :param branch: main branch + :param config_path: path to config file + :return: + """ + # open and replace values + for as_conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf")): + if as_conf_file.endswith(".yml") or as_conf_file.endswith(".yaml"): + with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'r') as f: + # Copied files could not have default names. + content = f.read() + content = content.replace(re.search('AUTOSUBMIT_VERSION: .*', content, re.MULTILINE).group(0), "AUTOSUBMIT_VERSION: \""+Autosubmit.autosubmit_version+"\"") + content = content.replace(re.search('NOTIFICATIONS: .*', content, re.MULTILINE).group(0), + "NOTIFICATIONS: False") + content = content.replace(re.search('TO: .*', content, re.MULTILINE).group(0), "TO: \"\"") + content = content.replace(re.search('EXPID: .*', content, re.MULTILINE).group(0),"EXPID: \""+exp_id+"\"") + content = content.replace(re.search('HPCARCH: .*', content, re.MULTILINE).group(0),"HPCARCH: \""+hpc+"\"") + if minimal_configuration: + content = content.replace(re.search('CUSTOM_CONFIG: .*', content, re.MULTILINE).group(0), "CUSTOM_CONFIG: \"%ROOTDIR%/proj/git_project/"+git_as_conf+"\"") + content = content.replace(re.search('PROJECT_ORIGIN: .*', content, re.MULTILINE).group(0), "PROJECT_ORIGIN: \""+git_repo+"\"") + content = content.replace(re.search('PROJECT_BRANCH: .*', content, re.MULTILINE).group(0), "PROJECT_BRANCH: \""+git_branch+"\"") + + with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'w') as f: + f.write(content) + + @staticmethod + def expid(description,hpc="", copy_id='', dummy=False,minimal_configuration=False,git_repo="",git_branch="",git_as_conf="",test=False, operational=False): """ Creates a new experiment for given HPC + description: description of the experiment + hpc: HPC where the experiment will be executed + copy_id: if specified, experiment id to copy + dummy: if true, creates a dummy experiment + minimal_configuration: if true, creates a minimal configuration + git_repo: git repository to clone + test: if true, creates a test experiment + operational: if true, creates an operational experiment - :param operational: if true, creates an operational experiment - :type operational: bool - :type hpc: str - :type description: str - :type copy_id: str - :type dummy: bool - :type test: bool - :type root_folder: str - :param hpc: name of the main HPC for the experiment - :param description: short experiment's description. - :param copy_id: experiment identifier of experiment to copy - :param dummy: if true, writes a default dummy configuration for testing - :param test: if true, creates an experiment for testing - - :return: experiment identifier. If method fails, returns ''. - :rtype: str """ + exp_id = None + root_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR) + if description is None: + raise AutosubmitCritical( + "Check that the parameters are defined (-d) ", 7011) + if hpc is None and not minimal_configuration: + raise AutosubmitCritical( + "Check that the parameters are defined (-H) ", 7011) + # Register the experiment in the database try: - exp_id = None - if description is None or hpc is None: - raise AutosubmitCritical( - "Check that the parameters are defined (-d and -H) ", 7011) - if not copy_id: - exp_id = new_experiment( - description, Autosubmit.autosubmit_version, test, operational) - if exp_id == '': - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise AutosubmitCritical( - "Couldn't create a new experiment", 7011) - try: - os.mkdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id)) - os.mkdir(os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf')) - Log.info("Copying config files...") - - # autosubmit config and experiment copied from AS. - files = resource_listdir('autosubmitconfigparser.config', 'files') - for filename in files: - if resource_exists('autosubmitconfigparser.config', 'files/' + filename): - index = filename.index('.') - new_filename = filename[:index] + "_" + exp_id + filename[index:] - - if filename == 'platforms.yml' and BasicConfig.DEFAULT_PLATFORMS_CONF != '': - content = open(os.path.join( - BasicConfig.DEFAULT_PLATFORMS_CONF, filename), 'rb').read() - elif filename == 'jobs.yml' and BasicConfig.DEFAULT_JOBS_CONF != '': - content = open(os.path.join( - BasicConfig.DEFAULT_JOBS_CONF, filename), 'rb').read() - else: - content = resource_string( - 'autosubmitconfigparser.config', 'files/' + filename) - - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile(BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').read() - - conf_new_filename = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", new_filename) - Log.debug(conf_new_filename) - open(conf_new_filename, 'wb').write(content) - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy, copy_id) - except (OSError, IOError) as e: - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass + # Copy another experiment from the database + if copy_id != '': + copy_id_folder = os.path.join(root_folder, copy_id) + if not os.path.exists(copy_id_folder): raise AutosubmitCritical( - "Couldn't create a new experiment, permissions?", 7012) - - except AutosubmitCritical as e: - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise - except BaseException as e: - err=str(e) - try: - Autosubmit._delete_expid(exp_id) - except Exception as e: - pass - raise AutosubmitCritical("Couldn't create a new experiment", 7012, err) + "Experiment {0} doesn't exists".format(copy_id), 7011) + exp_id = copy_experiment(copy_id, description, hpc, test, operational, copy_id_folder) else: - try: - if root_folder == '' or root_folder is None: - root_folder = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, copy_id) - if os.path.exists(root_folder): - # List of allowed files from conf - conf_copy_filter_folder = [] - conf_copy_filter_old_format_folder = [] - conf_copy_filter = ["autosubmit_" + str(copy_id) + ".yml", - "expdef_" + str(copy_id) + ".yml", - "jobs_" + str(copy_id) + ".yml", - "platforms_" + str(copy_id) + ".yml", - "proj_" + str(copy_id) + ".yml", - "autosubmit_" + str(copy_id) + ".conf", - "expdef_" + str(copy_id) + ".conf", - "jobs_" + str(copy_id) + ".conf", - "platforms_" + str(copy_id) + ".conf", - "proj_" + str(copy_id) + ".conf"] - if root_folder != os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id): - conf_copy_filter_folder = ["autosubmit.yml", - "expdef.yml", - "jobs.yml", - "platforms.yml", - "proj.yml"] - - exp_id = new_experiment( - description, Autosubmit.autosubmit_version, test, operational) - else: - exp_id = copy_experiment( - copy_id, description, Autosubmit.autosubmit_version, test, operational) - - if exp_id == '': - return '' - dir_exp_id = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, exp_id) - os.mkdir(dir_exp_id) - os.mkdir(dir_exp_id + '/conf') - if root_folder == os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id): - Log.info( - "Copying previous experiment config directories") - conf_copy_id = os.path.join( - BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf") - else: - Log.info("Copying from folder: {0}", root_folder) - conf_copy_id = root_folder - files = os.listdir(conf_copy_id) - for filename in files: - # Allow only those files in the list - Log.info("Copying filename: {0}", filename) - if filename in conf_copy_filter: - if os.path.isfile(os.path.join(conf_copy_id, filename)): - new_filename = filename.replace( - copy_id, exp_id) - if new_filename[-4:] == "conf": - new_filename = new_filename[:-4] + "yml" - # Using readlines for replacement handling - content = open(os.path.join( - conf_copy_id, filename), 'r').readlines() - - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile( - BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').readlines() - # Setting email notifications to false - if filename.startswith("autosubmit") and filename.endswith("conf"): - content = ["NOTIFICATIONS = False\n" if line.startswith( - ("NOTIFICATIONS =", "notifications =")) else line for line in content] - elif filename.startswith("autosubmit") and filename.endswith("yml"): - content = [" NOTIFICATIONS: False\n" if "NOTIFICATIONS" in line else line for - line in content] - # Putting content together before writing - sep = "" - open(os.path.join(dir_exp_id, "conf", - new_filename), 'w').write(sep.join(content)) - if filename.endswith("conf"): - try: - AutosubmitConfig.ini_to_yaml(os.path.join(dir_exp_id, "conf"), - os.path.join(os.path.join(dir_exp_id, "conf"), - new_filename)) - except BaseException as e: - Log.warning("Couldn't convert conf file to yml: {0}", new_filename) - if filename in conf_copy_filter_folder: - if os.path.isfile(os.path.join(conf_copy_id, filename)): - new_filename = filename.split( - ".")[0] + "_" + exp_id + ".yml" - content = open(os.path.join( - conf_copy_id, filename), 'rb').read() - # If autosubmitrc [conf] custom_platforms has been set and file exists, replace content - if filename.startswith("platforms") and os.path.isfile( - BasicConfig.CUSTOM_PLATFORMS_PATH): - content = open( - BasicConfig.CUSTOM_PLATFORMS_PATH, 'rb').read() - - open(os.path.join(dir_exp_id, "conf", - new_filename), 'wb').write(content) - - Autosubmit._prepare_conf_files( - exp_id, hpc, Autosubmit.autosubmit_version, dummy, copy_id) - ##### - autosubmit_config = AutosubmitConfig( - exp_id, BasicConfig, YAMLParserFactory()) - try: - autosubmit_config.check_conf_files(False) - project_type = autosubmit_config.get_project_type() - if project_type == "git": - autosubmit_git = AutosubmitGit(copy_id[0]) - Log.info("checking model version...") - if not autosubmit_git.check_commit(autosubmit_config): - raise AutosubmitCritical( - "Uncommitted changes", 7013) - except Exception as error: - Log.warning( - "Trace: {0}\nCouldn't load experiment configuration, check the experiments files before perform a create".format( - str(error))) - else: - raise AutosubmitCritical( - "The experiment directory doesn't exist", 7012) - except (OSError, IOError) as e: - try: - Autosubmit._delete_expid(exp_id, True) - except Exception as e: - pass - raise AutosubmitCritical( - "Can not create experiment", 7012) - except BaseException as e: - try: - Autosubmit._delete_expid(exp_id, True) - except Exception as e: - pass - raise AutosubmitCritical( - "Can not create experiment", 7012) - - Log.debug("Creating temporal directory...") - exp_id_path = os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id) - tmp_path = os.path.join(exp_id_path, "tmp") - os.mkdir(tmp_path) - os.chmod(tmp_path, 0o775) - os.mkdir(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR)) - os.chmod(os.path.join(tmp_path, BasicConfig.LOCAL_ASLOG_DIR), 0o775) - Log.debug("Creating temporal remote directory...") - remote_tmp_path = os.path.join(tmp_path, "LOG_" + exp_id) - os.mkdir(remote_tmp_path) - os.chmod(remote_tmp_path, 0o755) - - Log.debug("Creating pkl directory...") - os.mkdir(os.path.join(exp_id_path, "pkl")) - - Log.debug("Creating plot directory...") - os.mkdir(os.path.join(exp_id_path, "plot")) - os.chmod(os.path.join(exp_id_path, "plot"), 0o775) - Log.result("Experiment registered successfully") - Log.warning("Remember to MODIFY the config files!") + # Create a new experiment from scratch + exp_id = new_experiment(description, Autosubmit.autosubmit_version, test, operational, minimal_configuration) + + Autosubmit.as_conf_default_values(exp_id,hpc) + if exp_id == '': + raise AutosubmitCritical("No expid", 7011) + except Exception as e: + raise AutosubmitCritical("Error while generating a new experiment in the db: {0}".format(str(e)), 7011) + + # Create the experiment structure + Log.info("Generating folder structure...") + + exp_folder = os.path.join(root_folder, exp_id) + try: + os.mkdir(exp_folder) + os.mkdir(os.path.join(exp_folder, "conf")) + os.mkdir(os.path.join(exp_folder, "pkl")) + os.mkdir(os.path.join(exp_folder, "tmp")) + os.mkdir(os.path.join(exp_folder, "tmp", "ASLOGS")) + os.mkdir(os.path.join(exp_folder, "tmp", "LOG_"+exp_id)) + os.mkdir(os.path.join(exp_folder, "plot")) + os.mkdir(os.path.join(exp_folder, "status")) + os.mkdir(os.path.join(exp_folder, "proj")) + # Setting permissions + os.chmod(exp_folder, 0o755) + os.chmod(os.path.join(exp_folder, "conf"), 0o755) + os.chmod(os.path.join(exp_folder, "pkl"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "ASLOGS"), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "LOG_"+exp_id), 0o755) + os.chmod(os.path.join(exp_folder, "plot"), 0o755) + os.chmod(os.path.join(exp_folder, "status"), 0o755) + os.chmod(os.path.join(exp_folder, "proj"), 0o755) + except OSError as e: try: - Log.debug("Setting the right permissions...") - os.chmod(os.path.join(exp_id_path, "conf"), 0o755) - os.chmod(os.path.join(exp_id_path, "pkl"), 0o755) - os.chmod(os.path.join(exp_id_path, "tmp"), 0o775) - os.chmod(os.path.join(exp_id_path, "plot"), 0o775) - os.chmod(os.path.join(exp_id_path, "conf/autosubmit_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/expdef_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/jobs_" + - str(exp_id) + ".yml"), 0o755) - os.chmod(os.path.join(exp_id_path, "conf/platforms_" + - str(exp_id) + ".yml"), 0o755) - try: - os.chmod(os.path.join(exp_id_path, "tmp/ASLOGS"), 0o755) - except Exception as e: - pass - os.chmod(os.path.join(exp_id_path, "conf/proj_" + - str(exp_id) + ".yml"), 0o755) - except Exception as e: - pass # some folder may no exists, like proj - Log.debug("Finished") + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while creating the experiment structure: {0}".format(str(e)), 7011) - return exp_id - except AutosubmitCritical: - raise + # Create the experiment configuration + Log.info("Generating config files...") + try: + if copy_id != '': + # Copy the configuration from selected experiment + Autosubmit.copy_as_config(exp_id, copy_id) + else: + # Create a new configuration + Autosubmit.generate_as_config(exp_id,dummy, minimal_configuration) + except Exception as e: + try: + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while creating the experiment configuration: {0}".format(str(e)), 7011) + # Change template values by default values specified from the commandline + try: + Autosubmit.as_conf_default_values(exp_id,hpc,minimal_configuration,git_repo,git_branch,git_as_conf) except Exception as e: - raise AutosubmitCritical("Couldn't create a new experiment", 7012, str(e)) + try: + Autosubmit._delete_expid(exp_id, True) + except: + pass + raise AutosubmitCritical("Error while setting the default values: {0}".format(str(e)), 7011) + + Log.result("Experiment {0} created".format(exp_id)) + return exp_id @staticmethod def delete(expid, force): @@ -1651,11 +1543,11 @@ class Autosubmit: @staticmethod def run_experiment(expid, notransitive=False, update_version=False, start_time=None, start_after=None, - run_members=None): + run_only_members=None): """ Runs and experiment (submitting all the jobs properly and repeating its execution in case of failure). - :param run_members: + :param run_only_ºmembers: :param start_after: :param start_time: :param update_version: @@ -1682,8 +1574,8 @@ class Autosubmit: # Start after completion trigger block AutosubmitHelper.handle_start_after(start_after, expid, BasicConfig()) - # Handling run_members - allowed_members = AutosubmitHelper.get_allowed_members(run_members, as_conf) + # Handling run_only_members + allowed_members = AutosubmitHelper.get_allowed_members(run_only_members, as_conf) except AutosubmitCritical as e: raise except BaseException as e: @@ -5453,43 +5345,6 @@ class Autosubmit: except ValueError: sys.stdout.write('Please respond with \'y\' or \'n\'.\n') - @staticmethod - def _prepare_conf_files(exp_id, hpc, autosubmit_version, dummy, copy_id): - """ - Changes default configuration files to match new experiment values - - :param exp_id: experiment identifier - :type exp_id: str - :param hpc: hpc to use - :type hpc: str - :param autosubmit_version: current autosubmit's version - :type autosubmit_version: str - :param dummy: if True, creates a dummy experiment adding some default values - :type dummy: bool - """ - as_conf = AutosubmitConfig(exp_id, BasicConfig, YAMLParserFactory()) - as_conf.set_version(autosubmit_version) - update_experiment_descrip_version(exp_id, version=Autosubmit.autosubmit_version) - as_conf.set_expid(exp_id) - as_conf.set_platform(hpc) - - if dummy: - content = open(as_conf.experiment_file).read() - - # Experiment - content = content.replace(re.search('DATELIST: .*', content, re.MULTILINE).group(0), - "DATELIST: 20000101") - content = content.replace(re.search('MEMBERS: .*', content, re.MULTILINE).group(0), - "MEMBERS: fc0") - content = content.replace(re.search('CHUNKSIZE: .*', content, re.MULTILINE).group(0), - "CHUNKSIZE: 4") - content = content.replace(re.search('NUMCHUNKS: .*', content, re.MULTILINE).group(0), - "NUMCHUNKS: 2") - content = content.replace(re.search('PROJECT_TYPE: .*', content, re.MULTILINE).group(0), - "PROJECT_TYPE: 'none'") - - open(as_conf.experiment_file, 'w').write(content) - @staticmethod def _get_status(s): """ diff --git a/autosubmit/experiment/experiment_common.py b/autosubmit/experiment/experiment_common.py index 52ff8b91b..17eba23fc 100644 --- a/autosubmit/experiment/experiment_common.py +++ b/autosubmit/experiment/experiment_common.py @@ -29,14 +29,14 @@ def new_experiment(description, version, test=False, operational=False): """ Stores a new experiment on the database and generates its identifier - :param version: autosubmit version associated to the experiment + :param description: description of the experiment + :type description: str + :param version: version of the experiment :type version: str - :param test: flag for test experiments + :param test: if True, the experiment is a test experiment :type test: bool - :param operational: flag for operational experiments + :param operational: if True, the experiment is an operational experiment :type operational: bool - :param description: experiment's description - :type description: str :return: experiment id for the new experiment :rtype: str """ diff --git a/docs/source/userguide/create/index.rst b/docs/source/userguide/create/index.rst index 72050ce84..328c12e33 100644 --- a/docs/source/userguide/create/index.rst +++ b/docs/source/userguide/create/index.rst @@ -15,7 +15,7 @@ To create a new experiment, just run the command: Options: :: - usage: autosubmit expid [-h] [-y COPY | -dm] [-p PATH] -H HPC -d DESCRIPTION + usage: autosubmit expid [-h] [-y COPY | -dm | -min [-repo GIT_PATH ] ] [-p PATH] -H HPC -d DESCRIPTION -h, --help show this help message and exit -y COPY, --copy COPY makes a copy of the specified experiment @@ -23,13 +23,14 @@ Options: -H HPC, --HPC HPC specifies the HPC to use for the experiment -d DESCRIPTION, --description DESCRIPTION sets a description for the experiment to store in the database. - -c PATH, --config PATH - if specified, copies config files from a folder + -min,--minimal_configuration creates a minimal configuration file + -repo GIT_PATH, --git_repo GIT_PATH sets the git_repository Example: :: autosubmit expid --HPC ithaca --description "experiment is about..." + autosubmit expid --minimal_config -repo --description "experiment is about..." If there is an autosubmitrc or .autosubmitrc file in your home directory (cd ~), you can setup a default file from where the contents of platforms_expid.yml should be copied. -- GitLab From 94bea0f13dc002769eaae408ba2605f0033aabec Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 11:39:39 +0100 Subject: [PATCH 13/26] -Copy working --- autosubmit/autosubmit.py | 45 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 0834bf96b..64d9f059a 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -638,7 +638,7 @@ class Autosubmit: return Autosubmit.run_experiment(args.expid, args.notransitive, args.update_version, args.start_time, args.start_after, args.run_only_members) elif args.command == 'expid': - return Autosubmit.expid(args.description,args.HPC,args.copy, args.dummy,args.minimal_configuration,args.git_repo,args.git_branch,args.git_as_conf,args.test,args.operational) != '' + return Autosubmit.expid(args.description,args.HPC,args.copy, args.dummy,args.minimal_configuration,args.git_repo,args.git_branch,args.git_as_conf,args.operational) != '' elif args.command == 'delete': return Autosubmit.delete(args.expid, args.force) elif args.command == 'monitor': @@ -1016,8 +1016,7 @@ class Autosubmit: @staticmethod def copy_as_config(exp_id,copy_id): - as_conf_files = os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id)) - for conf_file in as_conf_files: + for conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id,"conf")): # Copy only relevant files if conf_file.endswith(".conf") or conf_file.endswith(".yml") or conf_file.endswith(".yaml"): shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), @@ -1062,22 +1061,37 @@ class Autosubmit: with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'r') as f: # Copied files could not have default names. content = f.read() - content = content.replace(re.search('AUTOSUBMIT_VERSION: .*', content, re.MULTILINE).group(0), "AUTOSUBMIT_VERSION: \""+Autosubmit.autosubmit_version+"\"") - content = content.replace(re.search('NOTIFICATIONS: .*', content, re.MULTILINE).group(0), - "NOTIFICATIONS: False") - content = content.replace(re.search('TO: .*', content, re.MULTILINE).group(0), "TO: \"\"") - content = content.replace(re.search('EXPID: .*', content, re.MULTILINE).group(0),"EXPID: \""+exp_id+"\"") - content = content.replace(re.search('HPCARCH: .*', content, re.MULTILINE).group(0),"HPCARCH: \""+hpc+"\"") + search = re.search('AUTOSUBMIT_VERSION: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "AUTOSUBMIT_VERSION: \""+Autosubmit.autosubmit_version+"\"") + search = re.search('NOTIFICATIONS: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"NOTIFICATIONS: False") + search = re.search('TO: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "TO: \"\"") + search = re.search('EXPID: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"EXPID: \""+exp_id+"\"") + search = re.search('HPCARCH: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0),"HPCARCH: \""+hpc+"\"") if minimal_configuration: - content = content.replace(re.search('CUSTOM_CONFIG: .*', content, re.MULTILINE).group(0), "CUSTOM_CONFIG: \"%ROOTDIR%/proj/git_project/"+git_as_conf+"\"") - content = content.replace(re.search('PROJECT_ORIGIN: .*', content, re.MULTILINE).group(0), "PROJECT_ORIGIN: \""+git_repo+"\"") - content = content.replace(re.search('PROJECT_BRANCH: .*', content, re.MULTILINE).group(0), "PROJECT_BRANCH: \""+git_branch+"\"") + search = re.search('CUSTOM_CONFIG: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "CUSTOM_CONFIG: \"%ROOTDIR%/proj/git_project/"+git_as_conf+"\"") + search = re.search('PROJECT_ORIGIN: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "PROJECT_ORIGIN: \""+git_repo+"\"") + search = re.search('PROJECT_BRANCH: .*', content, re.MULTILINE) + if search is not None: + content = content.replace(search.group(0), "PROJECT_BRANCH: \""+git_branch+"\"") with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'w') as f: f.write(content) @staticmethod - def expid(description,hpc="", copy_id='', dummy=False,minimal_configuration=False,git_repo="",git_branch="",git_as_conf="",test=False, operational=False): + def expid(description,hpc="", copy_id='', dummy=False,minimal_configuration=False,git_repo="",git_branch="",git_as_conf="",operational=False): """ Creates a new experiment for given HPC description: description of the experiment @@ -1106,12 +1120,11 @@ class Autosubmit: if not os.path.exists(copy_id_folder): raise AutosubmitCritical( "Experiment {0} doesn't exists".format(copy_id), 7011) - exp_id = copy_experiment(copy_id, description, hpc, test, operational, copy_id_folder) + exp_id = copy_experiment(copy_id, description, Autosubmit.autosubmit_version, False, operational) else: # Create a new experiment from scratch - exp_id = new_experiment(description, Autosubmit.autosubmit_version, test, operational, minimal_configuration) + exp_id = new_experiment(description, Autosubmit.autosubmit_version, False, operational, minimal_configuration) - Autosubmit.as_conf_default_values(exp_id,hpc) if exp_id == '': raise AutosubmitCritical("No expid", 7011) except Exception as e: -- GitLab From 04afb7af3d0830be4ff6165bf578dbf914f9917c Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 12:00:57 +0100 Subject: [PATCH 14/26] -min working in expdef --- autosubmit/autosubmit.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 64d9f059a..7d1c727de 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -213,15 +213,15 @@ class Autosubmit: help='creates a new experiment with default values, usually for testing') group.add_argument('-min', '--minimal_configuration', action='store_true', help='creates a new experiment with minimal configuration, usually combined with -repo') - subparser.add_argument('-repo', '--git_repo', type=str, required=False, + subparser.add_argument('-repo', '--git_repo', type=str, default="", required=False, help='sets a git repository for the experiment') - subparser.add_argument('-b', '--git_branch', type=str, required=False, + subparser.add_argument('-b', '--git_branch', type=str, default="", required=False, help='sets a git branch for the experiment') - subparser.add_argument('-conf', '--git_as_conf', type=str, required=False,help='sets the git path to as_conf') + subparser.add_argument('-conf', '--git_as_conf', type=str, default="", required=False,help='sets the git path to as_conf') group.add_argument('-op', '--operational', action='store_true', help='creates a new experiment with operational experiment id') - subparser.add_argument('-H', '--HPC', required=True, + subparser.add_argument('-H', '--HPC', required=False, default="local", help='specifies the HPC to use for the experiment') subparser.add_argument('-d', '--description', type=str, required=True, help='sets a description for the experiment to store in the database.') @@ -1036,13 +1036,13 @@ class Autosubmit: for as_conf_file in files: if dummy: if as_conf_file.endswith("dummy.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) elif minimal_configuration: if as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) else: if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files',as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ @@ -1115,7 +1115,7 @@ class Autosubmit: # Register the experiment in the database try: # Copy another experiment from the database - if copy_id != '': + if copy_id != '' and copy_id is not None: copy_id_folder = os.path.join(root_folder, copy_id) if not os.path.exists(copy_id_folder): raise AutosubmitCritical( @@ -1123,7 +1123,7 @@ class Autosubmit: exp_id = copy_experiment(copy_id, description, Autosubmit.autosubmit_version, False, operational) else: # Create a new experiment from scratch - exp_id = new_experiment(description, Autosubmit.autosubmit_version, False, operational, minimal_configuration) + exp_id = new_experiment(description, Autosubmit.autosubmit_version, False, operational) if exp_id == '': raise AutosubmitCritical("No expid", 7011) @@ -1164,7 +1164,7 @@ class Autosubmit: # Create the experiment configuration Log.info("Generating config files...") try: - if copy_id != '': + if copy_id != '' and copy_id is not None: # Copy the configuration from selected experiment Autosubmit.copy_as_config(exp_id, copy_id) else: -- GitLab From e8843e77bd5c262b9b671bc84a62be3402aa69b2 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 12:54:19 +0100 Subject: [PATCH 15/26] -min working --- autosubmit/autosubmit.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 7d1c727de..aee639d53 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1020,14 +1020,14 @@ class Autosubmit: # Copy only relevant files if conf_file.endswith(".conf") or conf_file.endswith(".yml") or conf_file.endswith(".yaml"): shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), - os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file)) + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file.replace(copy_id,exp_id))) # if ends with .conf convert it to AS4 yaml file if conf_file.endswith(".conf"): try: AutosubmitConfig.ini_to_yaml(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf"), - os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file)) + os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file.replace(copy_id,exp_id))) except Exception as e: - Log.warning("Error converting {0} to yml: {1}".format(conf_file,str(e))) + Log.warning("Error converting {0} to yml: {1}".format(conf_file.replace(copy_id,exp_id),str(e))) @staticmethod def generate_as_config(exp_id,dummy=False,minimal_configuration=False): # obtain from autosubmitconfigparser package @@ -1036,13 +1036,13 @@ class Autosubmit: for as_conf_file in files: if dummy: if as_conf_file.endswith("dummy.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) elif minimal_configuration: if as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+".yml")) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) else: if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file)) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file[:-4]+"_"+exp_id+".yml")) @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ @@ -1140,20 +1140,18 @@ class Autosubmit: os.mkdir(os.path.join(exp_folder, "pkl")) os.mkdir(os.path.join(exp_folder, "tmp")) os.mkdir(os.path.join(exp_folder, "tmp", "ASLOGS")) - os.mkdir(os.path.join(exp_folder, "tmp", "LOG_"+exp_id)) + os.mkdir(os.path.join(exp_folder, "tmp", "LOG_"+exp_id.upper())) os.mkdir(os.path.join(exp_folder, "plot")) os.mkdir(os.path.join(exp_folder, "status")) - os.mkdir(os.path.join(exp_folder, "proj")) # Setting permissions os.chmod(exp_folder, 0o755) os.chmod(os.path.join(exp_folder, "conf"), 0o755) os.chmod(os.path.join(exp_folder, "pkl"), 0o755) os.chmod(os.path.join(exp_folder, "tmp"), 0o755) os.chmod(os.path.join(exp_folder, "tmp", "ASLOGS"), 0o755) - os.chmod(os.path.join(exp_folder, "tmp", "LOG_"+exp_id), 0o755) + os.chmod(os.path.join(exp_folder, "tmp", "LOG_"+exp_id.upper()), 0o755) os.chmod(os.path.join(exp_folder, "plot"), 0o755) os.chmod(os.path.join(exp_folder, "status"), 0o755) - os.chmod(os.path.join(exp_folder, "proj"), 0o755) except OSError as e: try: Autosubmit._delete_expid(exp_id, True) -- GitLab From cbb1b630ce61b5678cf90c44f5924d4b0f63ef66 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 14 Feb 2023 13:00:23 +0100 Subject: [PATCH 16/26] min rst --- docs/source/userguide/create/index.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/source/userguide/create/index.rst b/docs/source/userguide/create/index.rst index 328c12e33..05e575c38 100644 --- a/docs/source/userguide/create/index.rst +++ b/docs/source/userguide/create/index.rst @@ -19,18 +19,26 @@ Options: -h, --help show this help message and exit -y COPY, --copy COPY makes a copy of the specified experiment + -op, -operational creates a new experiment, starting with "o" -dm, --dummy creates a new experiment with default values, usually for testing + -min, --minimal_config + creates a new experiment with minimal configuration files, usually for using a custom configuration + -repo GIT_PATH, --git_repo GIT_PATH + sets the git_repository + -b BRANCH, --git_branch BRANCH + sets the branch to use for the git repository + -config, --git_as_conf + sets the configuration folder to use for the experiment -H HPC, --HPC HPC specifies the HPC to use for the experiment -d DESCRIPTION, --description DESCRIPTION sets a description for the experiment to store in the database. - -min,--minimal_configuration creates a minimal configuration file -repo GIT_PATH, --git_repo GIT_PATH sets the git_repository Example: :: autosubmit expid --HPC ithaca --description "experiment is about..." - autosubmit expid --minimal_config -repo --description "experiment is about..." + autosubmit expid -min -repo https://earth.bsc.es/gitlab/ces/auto-advanced_config_example -b main -conf as_conf -d "minimal config example" If there is an autosubmitrc or .autosubmitrc file in your home directory (cd ~), you can setup a default file from where the contents of platforms_expid.yml should be copied. -- GitLab From 225bd014835c3e5f9de721723723e34a57ab9122 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 16 Feb 2023 09:44:30 +0100 Subject: [PATCH 17/26] [wip] config changes --- docs/source/userguide/create/index.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/userguide/create/index.rst b/docs/source/userguide/create/index.rst index 05e575c38..36881d14d 100644 --- a/docs/source/userguide/create/index.rst +++ b/docs/source/userguide/create/index.rst @@ -28,8 +28,8 @@ Options: -b BRANCH, --git_branch BRANCH sets the branch to use for the git repository -config, --git_as_conf - sets the configuration folder to use for the experiment - -H HPC, --HPC HPC specifies the HPC to use for the experiment + sets the configuration folder to use for the experiment, relative to repo root + -H HPC, --HPC HPC specifies the HPC to use for the experiment, default is localhost -d DESCRIPTION, --description DESCRIPTION sets a description for the experiment to store in the database. -repo GIT_PATH, --git_repo GIT_PATH sets the git_repository @@ -37,8 +37,10 @@ Options: Example: :: - autosubmit expid --HPC ithaca --description "experiment is about..." + autosubmit expid --HPC marenostrum4 --description "experiment is about..." autosubmit expid -min -repo https://earth.bsc.es/gitlab/ces/auto-advanced_config_example -b main -conf as_conf -d "minimal config example" + autosubmit expid -dm -d "dummy test" + If there is an autosubmitrc or .autosubmitrc file in your home directory (cd ~), you can setup a default file from where the contents of platforms_expid.yml should be copied. -- GitLab From e5e9e49de1590954812a41d86b832f8a83745bc5 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 21 Feb 2023 15:35:34 +0100 Subject: [PATCH 18/26] Still some bugs --- .gitmodules | 3 --- autosubmit/autosubmit.py | 22 ++++++++++++---------- autosubmit/git/autosubmit_git.py | 4 ++-- autosubmit/job/job.py | 2 +- autosubmit/job/job_list.py | 2 +- autosubmit/statistics/test.py | 2 +- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/.gitmodules b/.gitmodules index 26eeaaf28..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "autosubmit4-config-parser"] - path = autosubmit4-config-parser - url = ../../ces/autosubmit4-config-parser diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index aee639d53..fb2504da2 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -773,7 +773,7 @@ class Autosubmit: raise AutosubmitCritical(message, 7071) if expid != 'None' and args.command not in expid_less and args.command not in global_log_command: as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) if len(as_conf.experiment_data) == 0: if args.command not in ["expid", "upgrade"]: raise AutosubmitCritical( @@ -1941,7 +1941,7 @@ class Autosubmit: consecutive_retrials = consecutive_retrials + 1 Log.info("Waiting {0} seconds before continue".format(delay)) try: - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) Log.info("Recovering job_list...") job_list = Autosubmit.load_job_list( expid, as_conf, notransitive=notransitive) @@ -3819,7 +3819,7 @@ class Autosubmit: try: Autosubmit._check_ownership(expid, raise_error=True) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - # as_conf.reload(first_load=True) + # as_conf.reload(force_load=True) as_conf.check_conf_files(refresh=True) except (AutosubmitError, AutosubmitCritical): raise @@ -3849,7 +3849,7 @@ class Autosubmit: Autosubmit._check_ownership(expid, raise_error=True) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) as_conf.check_expdef_conf() Log.info("Changing {0} experiment version from {1} to {2}", @@ -3968,7 +3968,7 @@ class Autosubmit: except Exception as e: Log.warning("Couldn't convert conf file to yml: {0}", Path(f).parent) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) # Load current variables as_conf.check_conf_files() # Load current parameters ( this doesn't read job parameters) @@ -4306,8 +4306,8 @@ class Autosubmit: @staticmethod def _create_project_associated_conf(as_conf, force_model_conf, force_jobs_conf): - project_destiny = as_conf.project_file - jobs_destiny = as_conf.jobs_file + project_destiny = as_conf.get_file_project_conf() + jobs_destiny = as_conf.get_file_jobs_conf() if as_conf.get_project_type() != 'none': if as_conf.get_file_project_conf(): @@ -4378,9 +4378,11 @@ class Autosubmit: project_type = as_conf.get_project_type() # Getting output type provided by the user in config, 'pdf' as default output_type = as_conf.get_output_type() - - if not Autosubmit._copy_code(as_conf, expid, project_type, False): - return False + try: + if not Autosubmit._copy_code(as_conf, expid, project_type, False): + return False + except: + raise AutosubmitCritical("Error obtaining the project data, check the parameters related to PROJECT and GIT/SVN or LOCAL sections", code=7014) # Update configuration with the new config in the dist ( if any ) as_conf.check_conf_files(False) diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index b95ed4ce8..29e16e8b6 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . - +import locale from os import path import os from shutil import rmtree @@ -205,7 +205,7 @@ class AutosubmitGit: Log.debug('Clone command: {0}', command_0) try: git_version = subprocess.check_output("git --version",shell=True) - git_version = git_version.split(" ")[2].strip("\n") + git_version = git_version.split(" ")[2].decode(locale.getlocale()[1]).strip("\n") version_int = "" for number in git_version.split("."): version_int += number diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index a966c8fb0..637ba33a9 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -655,7 +655,7 @@ class Job(object): while (count < retries) or not success: try: as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) max_retrials = as_conf.get_retrials() max_logs = int(as_conf.get_retrials()) - fail_count last_log = int(as_conf.get_retrials()) - fail_count diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 1dcde217f..3ab0af5de 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -2085,7 +2085,7 @@ class JobList(object): :param as_conf: experiment configuration :type as_conf: AutosubmitConfig """ - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) out = True for job in self._job_list: show_logs = job.check_warnings diff --git a/autosubmit/statistics/test.py b/autosubmit/statistics/test.py index 6a95f8397..64bf893b8 100644 --- a/autosubmit/statistics/test.py +++ b/autosubmit/statistics/test.py @@ -27,7 +27,7 @@ class TestStatistics(unittest.TestCase): path_structure = BasicConfig.STRUCTURES_DIR path_local_root = BasicConfig.LOCAL_ROOT_DIR as_conf = AutosubmitConfig(expid) - as_conf.reload(first_load=True) + as_conf.reload(force_load=True) job_list = Autosubmit.load_job_list(expid, as_conf, False) jobs_considered = [job for job in job_list.get_job_list() if job.status not in [ Status.READY, Status.WAITING]] -- GitLab From 263854722545b8030901eeaaf0c2ea6ab410dd33 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 22 Feb 2023 14:51:19 +0100 Subject: [PATCH 19/26] Recursive config working --- autosubmit/job/job.py | 2 +- autosubmit/job/job_list.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 637ba33a9..5632cedb0 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -994,7 +994,7 @@ class Job(object): """ chunk = 1 as_conf.reload() - #parameters = as_conf.substitute_dynamic_variables(parameters,25) + parameters = as_conf.sustitute_dynamic_variables(parameters,25) parameters = parameters.copy() parameters.update(default_parameters) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 3ab0af5de..034ba8522 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -285,9 +285,15 @@ class JobList(object): dependencies_keys = jobs_data[job_section].get(option,{}) if type(dependencies_keys) is str: - dependencies_keys = dependencies_keys.split() + if "," in dependencies_keys: + dependencies_list = dependencies_keys.split(",") + else: + dependencies_list = dependencies_keys.split(" ") + dependencies_keys = {} + for dependency in dependencies_list: + dependencies_keys[dependency] = {} if dependencies_keys is None: - dependencies_keys = [] + dependencies_keys = {} dependencies = JobList._manage_dependencies(dependencies_keys, dic_jobs, job_section) for job in dic_jobs.get_jobs(job_section): -- GitLab From 17f37debadcbfb2c4fd2ecdd83b288c5c242ef9e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 22 Feb 2023 15:47:02 +0100 Subject: [PATCH 20/26] Now autosubmit will ignore -minimal files. Now autosubmit will create a version.yml if AUTOSUBMIT_VERSION is not found --- autosubmit/autosubmit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index fb2504da2..a38818e3e 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1039,7 +1039,7 @@ class Autosubmit: shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) elif minimal_configuration: if as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) + shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf","minimal.yml")) else: if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file[:-4]+"_"+exp_id+".yml")) @@ -1577,7 +1577,7 @@ class Autosubmit: raise AutosubmitCritical("Failure during the loading of the experiment configuration, check file paths", 7014, str(e)) as_conf = AutosubmitConfig(expid, BasicConfig, YAMLParserFactory()) - as_conf.check_conf_files(True, True) + as_conf.check_conf_files(running_time=True, force_load=True) try: # Handling starting time AutosubmitHelper.handle_start_time(start_time) -- GitLab From 163cae97b5e1b6d524dc836a83553cd0e6b5fe4b Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 24 Feb 2023 18:01:10 +0100 Subject: [PATCH 21/26] rebase and typo in function name --- autosubmit/job/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 5632cedb0..d01fac5c7 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -994,7 +994,7 @@ class Job(object): """ chunk = 1 as_conf.reload() - parameters = as_conf.sustitute_dynamic_variables(parameters,25) + parameters = as_conf.substitute_dynamic_variables(parameters,25) parameters = parameters.copy() parameters.update(default_parameters) -- GitLab From 2122218629713d05f9d6b3cbf9fee81ac35c753d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 28 Feb 2023 14:50:02 +0100 Subject: [PATCH 22/26] Refactored parameters function, added nodes tasks processors and threads at platform level --- autosubmit/job/job.py | 278 +++++++++++----------- autosubmit/platforms/paramiko_platform.py | 1 + 2 files changed, 141 insertions(+), 138 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index d01fac5c7..41963c5aa 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -978,26 +978,101 @@ class Job(object): self.status = default_status else: return default_status + def update_platform_parameters(self,as_conf,parameters,job_platform): + parameters['CURRENT_ARCH'] = job_platform.name + parameters['CURRENT_HOST'] = job_platform.host + parameters['CURRENT_USER'] = job_platform.user + parameters['CURRENT_PROJ'] = job_platform.project + parameters['CURRENT_BUDG'] = job_platform.budget + parameters['CURRENT_RESERVATION'] = job_platform.reservation + parameters['CURRENT_EXCLUSIVITY'] = job_platform.exclusivity + parameters['CURRENT_HYPERTHREADING'] = job_platform.hyperthreading + parameters['CURRENT_TYPE'] = job_platform.type + parameters['CURRENT_SCRATCH_DIR'] = job_platform.scratch + parameters['CURRENT_PROJ_DIR'] = job_platform.project_dir + parameters['CURRENT_ROOTDIR'] = job_platform.root_dir + parameters['CURRENT_LOGDIR'] = job_platform.get_files_path() - def update_parameters(self, as_conf, parameters, - default_parameters={'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', - 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'}): - """ - Refresh parameters value - - :param default_parameters: - :type default_parameters: dict - :param as_conf: - :type as_conf: AutosubmitConfig - :param parameters: - :type parameters: dict - """ - chunk = 1 - as_conf.reload() - parameters = as_conf.substitute_dynamic_variables(parameters,25) + return parameters + def update_platform_associated_parameters(self,as_conf, parameters, job_platform, chunk): + self.queue = self.queue + self.processors = str(as_conf.jobs_data[self.section].get("PROCESSORS",as_conf.platforms_data.get(job_platform.name,{}).get("PROCESSORS","1"))) + self.threads = str(as_conf.jobs_data[self.section].get("THREADS",as_conf.platforms_data.get(job_platform.name,{}).get("THREADS","1"))) + self.tasks = str(as_conf.jobs_data[self.section].get("TASKS",as_conf.platforms_data.get(job_platform.name,{}).get("TASKS","1"))) + self.nodes = str(as_conf.jobs_data[self.section].get("NODES",as_conf.platforms_data.get(job_platform.name,{}).get("NODES","1"))) + self.hyperthreading = str(as_conf.jobs_data[self.section].get("HYPERTHREADING",as_conf.platforms_data.get(job_platform.name,{}).get("HYPERTHREADING","none"))) + if int(self.tasks) <= 1 and int(job_platform.processors_per_node) > 1 and int(self.processors) > int(job_platform.processors_per_node): + self.tasks = job_platform.processors_per_node + self.memory = str(as_conf.jobs_data[self.section].get("MEMORY",as_conf.platforms_data.get(job_platform.name,{}).get("MEMORY",""))) + self.memory_per_task = str(as_conf.jobs_data[self.section].get("MEMORY_PER_TASK",as_conf.platforms_data.get(job_platform.name,{}).get("MEMORY_PER_TASK",""))) + self.wallclock = as_conf.jobs_data[self.section].get("WALLCLOCK",as_conf.platforms_data.get(self.platform_name,{}).get("MAX_WALLCLOCK",None)) + if self.wallclock is None and job_platform.type not in ['ps',"local","PS","LOCAL"]: + self.wallclock = "01:59" + elif self.wallclock is None and job_platform.type in ['ps','local',"PS","LOCAL"]: + self.wallclock = "00:00" + # Increasing according to chunk + self.wallclock = increase_wallclock_by_chunk( + self.wallclock, self.wchunkinc, chunk) + self.scratch_free_space = int(as_conf.jobs_data[self.section].get("SCRATCH_FREE_SPACE",as_conf.platforms_data.get(job_platform.name,{}).get("SCRATCH_FREE_SPACE",0))) + try: + self.custom_directives = as_conf.jobs_data[self.section].get("CUSTOM_DIRECTIVES","").replace("\'", "\"").strip("[]").strip(", ") + if self.custom_directives == '': + if job_platform.custom_directives is None: + job_platform.custom_directives = '' + self.custom_directives = job_platform.custom_directives.replace("\'", "\"").strip("[]").strip(", ") + if self.custom_directives != '': + if self.custom_directives[0] != "\"": + self.custom_directives = "\""+self.custom_directives + if self.custom_directives[-1] != "\"": + self.custom_directives = self.custom_directives+"\"" + self.custom_directives = "[" + self.custom_directives + "]" + self.custom_directives = json.loads(self.custom_directives) + else: + self.custom_directives = [] + except BaseException as e: + raise AutosubmitCritical(f"Error in CUSTOM_DIRECTIVES({self.custom_directives}) for job {self.section}",7014,str(e)) + parameters['NUMPROC'] = self.processors + parameters['PROCESSORS'] = self.processors + parameters['MEMORY'] = self.memory + parameters['MEMORY_PER_TASK'] = self.memory_per_task + parameters['NUMTHREADS'] = self.threads + parameters['THREADS'] = self.threads + parameters['CPUS_PER_TASK'] = self.threads + parameters['NUMTASK'] = self.tasks + parameters['TASKS'] = self.tasks + parameters['NODES'] = self.nodes + parameters['TASKS_PER_NODE'] = self.tasks + parameters['WALLCLOCK'] = self.wallclock + parameters['TASKTYPE'] = self.section + parameters['SCRATCH_FREE_SPACE'] = self.scratch_free_space + parameters['CUSTOM_DIRECTIVES'] = self.custom_directives + parameters['HYPERTHREADING'] = self.hyperthreading + parameters['CURRENT_QUEUE'] = self.queue + return parameters + def update_wrapper_parameters(self,as_conf, parameters): + wrappers = as_conf.experiment_data.get("WRAPPERS", {}) + if len(wrappers) > 0: + parameters['WRAPPER'] = as_conf.get_wrapper_type() + parameters['WRAPPER' + "_POLICY"] = as_conf.get_wrapper_policy() + parameters['WRAPPER' + "_METHOD"] = as_conf.get_wrapper_method().lower() + parameters['WRAPPER' + "_JOBS"] = as_conf.get_wrapper_jobs() + parameters['WRAPPER' + "_EXTENSIBLE"] = as_conf.get_extensible_wallclock() - parameters = parameters.copy() - parameters.update(default_parameters) + for wrapper_section, wrapper_val in wrappers.items(): + if type(wrapper_val) is not dict: + continue + parameters[wrapper_section] = as_conf.get_wrapper_type( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) + parameters[wrapper_section + "_POLICY"] = as_conf.get_wrapper_policy( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) + parameters[wrapper_section + "_METHOD"] = as_conf.get_wrapper_method( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)).lower() + parameters[wrapper_section + "_JOBS"] = as_conf.get_wrapper_jobs( + as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) + parameters[wrapper_section + "_EXTENSIBLE"] = int( + as_conf.get_extensible_wallclock(as_conf.experiment_data["WRAPPERS"].get(wrapper_section))) + return parameters + def update_job_parameters(self,as_conf, parameters): parameters['JOBNAME'] = self.name parameters['FAIL_COUNT'] = str(self.fail_count) parameters['SDATE'] = date2str(self.date, self.date_format) @@ -1007,6 +1082,7 @@ class Job(object): parameters['FREQUENCY'] = self.frequency parameters['SYNCHRONIZE'] = self.synchronize parameters['PACKED'] = self.packed + parameters['CHUNK'] = 1 if hasattr(self, 'RETRIALS'): parameters['RETRIALS'] = self.retrials if hasattr(self, 'delay_retrials'): @@ -1018,10 +1094,10 @@ class Job(object): chunk = self.chunk parameters['CHUNK'] = chunk - total_chunk = int(parameters.get('EXPERIMENT.NUMCHUNKS',1)) - chunk_length = int(parameters.get('EXPERIMENT.CHUNKSIZE',1)) - chunk_unit = str(parameters.get('EXPERIMENT.CHUNKSIZEUNIT',"")).lower() - cal = str(parameters.get('EXPERIMENT.CALENDAR',"")).lower() + total_chunk = int(parameters.get('EXPERIMENT.NUMCHUNKS', 1)) + chunk_length = int(parameters.get('EXPERIMENT.CHUNKSIZE', 1)) + chunk_unit = str(parameters.get('EXPERIMENT.CHUNKSIZEUNIT', "")).lower() + cal = str(parameters.get('EXPERIMENT.CALENDAR', "")).lower() chunk_start = chunk_start_date( self.date, chunk, chunk_length, chunk_unit, cal) chunk_end = chunk_end_date( @@ -1046,7 +1122,6 @@ class Job(object): parameters['CHUNK_START_DAY'] = str(chunk_start.day).zfill(2) parameters['CHUNK_START_HOUR'] = str(chunk_start.hour).zfill(2) - parameters['CHUNK_SECOND_TO_LAST_DATE'] = date2str( chunk_end_1, self.date_format) parameters['CHUNK_SECOND_TO_LAST_YEAR'] = str(chunk_end_1.year) @@ -1072,128 +1147,55 @@ class Job(object): parameters['Chunk_LAST'] = 'TRUE' else: parameters['Chunk_LAST'] = 'FALSE' - - job_platform = self._platform - self.queue = self.queue - self.processors = str(as_conf.jobs_data[self.section].get("PROCESSORS","1")) - self.threads = str(as_conf.jobs_data[self.section].get("THREADS","1")) - self.tasks = str(as_conf.jobs_data[self.section].get("TASKS","1")) - self.nodes = str(as_conf.jobs_data[self.section].get("NODES","")) - - self.hyperthreading = str(as_conf.jobs_data[self.section].get("HYPERTHREADING","none")) - if self.hyperthreading == 'none' and len(self.hyperthreading) > 0: - self.hyperthreading = job_platform.hyperthreading - if int(self.tasks) <= 1 and int(job_platform.processors_per_node) > 1 and int(self.processors) > int(job_platform.processors_per_node): - self.tasks = job_platform.processors_per_node - self.memory = str(as_conf.jobs_data[self.section].get("MEMORY","")) - self.memory_per_task = str(as_conf.jobs_data[self.section].get("MEMORY_PER_TASK","")) - remote_max_wallclock = as_conf.experiment_data["PLATFORMS"].get(self.platform_name,{}) - remote_max_wallclock = remote_max_wallclock.get("MAX_WALLCLOCK",None) - self.wallclock = as_conf.jobs_data[self.section].get("WALLCLOCK",remote_max_wallclock) - self.wchunkinc = str(as_conf.jobs_data[self.section].get("WCHUNKINC","")) - if self.wallclock is None and job_platform.type not in ['ps',"local","PS","LOCAL"]: - self.wallclock = "01:59" - elif self.wallclock is None and job_platform.type in ['ps','local',"PS","LOCAL"]: - self.wallclock = "00:00" + parameters['NUMMEMBERS'] = len(as_conf.get_member_list()) + parameters['DEPENDENCIES'] = str(as_conf.jobs_data[self.section].get("DEPENDENCIES","")) + # This shouldn't be necessary anymore as now all sub is done in the as_conf.reload() + # if len(self.export) > 0: + # variables = re.findall('%(? 0: + # variables = [variable[1:-1] for variable in variables] + # for key in variables: + # try: + # self.export = re.sub( + # '%(? 0: - parameters['WRAPPER'] = as_conf.get_wrapper_type() - parameters['WRAPPER' + "_POLICY"] = as_conf.get_wrapper_policy() - parameters['WRAPPER' + "_METHOD"] = as_conf.get_wrapper_method().lower() - parameters['WRAPPER' + "_JOBS"] = as_conf.get_wrapper_jobs() - parameters['WRAPPER' + "_EXTENSIBLE"] = as_conf.get_extensible_wallclock() - - for wrapper_section,wrapper_val in wrappers.items(): - if type(wrapper_val) is not dict: - continue - parameters[wrapper_section] = as_conf.get_wrapper_type(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) - parameters[wrapper_section+"_POLICY"] = as_conf.get_wrapper_policy(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) - parameters[wrapper_section+"_METHOD"] = as_conf.get_wrapper_method(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)).lower() - parameters[wrapper_section+"_JOBS"] = as_conf.get_wrapper_jobs(as_conf.experiment_data["WRAPPERS"].get(wrapper_section)) - parameters[wrapper_section+"_EXTENSIBLE"] = int(as_conf.get_extensible_wallclock(as_conf.experiment_data["WRAPPERS"].get(wrapper_section))) - self.dependencies = parameters['DEPENDENCIES'] - - # This shouldn't be necessary anymore as now all sub is done in the as_conf.reload() - if len(self.export) > 0: - variables = re.findall('%(? 0: - variables = [variable[1:-1] for variable in variables] - for key in variables: - try: - self.export = re.sub( - '%(? Date: Tue, 28 Feb 2023 16:02:40 +0100 Subject: [PATCH 23/26] This should fix the pipeline, also fixed an possible issue with EXPORT parameter, but have to doble check --- autosubmit/job/job.py | 2 +- environment.yml | 2 +- requeriments.txt | 2 +- setup.py | 2 +- test/unit/test_job.py | 15 ++++++++++----- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 41963c5aa..21d5d1cbb 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1183,9 +1183,9 @@ class Job(object): """ chunk = 1 as_conf.reload() - parameters = as_conf.substitute_dynamic_variables(parameters,25) parameters = parameters.copy() parameters.update(default_parameters) + parameters = as_conf.substitute_dynamic_variables(parameters,25) parameters['ROOTDIR'] = os.path.join( BasicConfig.LOCAL_ROOT_DIR, self.expid) parameters['PROJDIR'] = as_conf.get_project_dir() diff --git a/environment.yml b/environment.yml index c16c57bd4..584c57f18 100644 --- a/environment.yml +++ b/environment.yml @@ -18,7 +18,7 @@ dependencies: - networkx - sqlite - pip: - - autosubmitconfigparser==0.0.74 + - autosubmitconfigparser==1.0.0 - argparse>=1.4.0 - python-dateutil>=2.8.2 - matplotlib>=3.5.1 diff --git a/requeriments.txt b/requeriments.txt index 929e7c706..4b55c25a3 100644 --- a/requeriments.txt +++ b/requeriments.txt @@ -1,4 +1,4 @@ -autosubmitconfigparser==0.0.78 +autosubmitconfigparser==1.0.0 paramiko>=2.9.2 PyNaCl>=1.5.0 configobj>=5.0.6 diff --git a/setup.py b/setup.py index 6a8e9b0dc..90d217405 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=['autosubmitconfigparser==0.0.78','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl>=1.4.0','Pygments'], + install_requires=['autosubmitconfigparser==1.0.0','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','ruamel.yaml','pythondialog','pytest','nose','coverage','PyNaCl>=1.4.0','Pygments'], classifiers=[ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.9", diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 08ec3d781..5982ac6be 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -262,8 +262,9 @@ class TestJob(TestCase): # This test (and feature) was implemented in order to avoid # false positives on the checking process with auto-ecearth3 # Arrange - section = "random-section" - self.job.section = "random-section" + section = "RANDOM-SECTION" + self.job.section = section + self.job.parameters['ROOTDIR'] = "none" self.job.parameters['PROJECT_TYPE'] = "none" processors = 80 threads = 1 @@ -278,7 +279,8 @@ class TestJob(TestCase): 'TASKS': tasks, 'MEMORY': memory, 'WALLCLOCK': wallclock, - 'CUSTOM_DIRECTIVES': custom_directives + 'CUSTOM_DIRECTIVES': custom_directives, + 'SCRATCH_FREE_SPACE': 0 } self.as_conf.jobs_data[section] = options @@ -286,12 +288,15 @@ class TestJob(TestCase): dummy_serial_platform.name = 'serial' dummy_platform = MagicMock() dummy_platform.serial_platform = dummy_serial_platform + self.as_conf.substitute_dynamic_variables = MagicMock() + self.as_conf.substitute_dynamic_variables.return_value = {'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', + 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'} dummy_platform.custom_directives = '["whatever"]' self.as_conf.dynamic_variables = MagicMock() self.job._platform = dummy_platform - + parameters = {} # Act - parameters = self.job.update_parameters(self.as_conf, dict()) + parameters = self.job.update_parameters(self.as_conf, parameters) # Assert self.assertTrue('d' in parameters) self.assertTrue('d_' in parameters) -- GitLab From 6c7080c03c749b67e29aebbb20288dff7b1c5735 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 28 Feb 2023 16:13:57 +0100 Subject: [PATCH 24/26] git test,git_branch cleaned expdef-minimal for unused parameter --- autosubmit/autosubmit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index a38818e3e..e82fdee94 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1100,9 +1100,11 @@ class Autosubmit: dummy: if true, creates a dummy experiment minimal_configuration: if true, creates a minimal configuration git_repo: git repository to clone - test: if true, creates a test experiment + git_branch: git branch to clone + git_as_conf: path to as_conf file in git repository operational: if true, creates an operational experiment + """ exp_id = None root_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR) -- GitLab From 9a5172e1054e9be7a31a9471dc7b4ff092e16233 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 28 Feb 2023 16:19:25 +0100 Subject: [PATCH 25/26] more docstrings --- autosubmit/autosubmit.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index e82fdee94..b799e5f74 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1051,9 +1051,9 @@ class Autosubmit: :param hpc: platform :param minimal_configuration: minimal configuration :param git_repo: path to project git repository - :param branch: main branch - :param config_path: path to config file - :return: + :param git_branch: main branch + :param git_as_conf: path to as_conf file in git repository + :return: None """ # open and replace values for as_conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf")): @@ -1103,10 +1103,8 @@ class Autosubmit: git_branch: git branch to clone git_as_conf: path to as_conf file in git repository operational: if true, creates an operational experiment - - """ - exp_id = None + exp_id = "" root_folder = os.path.join(BasicConfig.LOCAL_ROOT_DIR) if description is None: raise AutosubmitCritical( -- GitLab From f82bfd2ade86420edbc34d7f3eed7a178c6cb514 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 1 Mar 2023 10:07:01 +0100 Subject: [PATCH 26/26] simplified expression --- autosubmit/autosubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index b799e5f74..2cdcb6298 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1018,7 +1018,7 @@ class Autosubmit: def copy_as_config(exp_id,copy_id): for conf_file in os.listdir(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id,"conf")): # Copy only relevant files - if conf_file.endswith(".conf") or conf_file.endswith(".yml") or conf_file.endswith(".yaml"): + if conf_file.endswith((".conf, .yml, .yaml")): shutil.copy(os.path.join(BasicConfig.LOCAL_ROOT_DIR, copy_id, "conf", conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", conf_file.replace(copy_id,exp_id))) # if ends with .conf convert it to AS4 yaml file -- GitLab