From 2a90bf4d2abeb555c73f93aacb1febc5bdcb8c04 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 17 May 2023 19:37:23 +0200 Subject: [PATCH 1/8] Create decorators to meta-programmatically create the documentation for variables. Two decorators are available now, @autosubmit_parameters, used to annotate classes with a dictionary with groups and their parameters. And @autosubmit_parameter (singular) used to annotate @property members. This latter decorator uses the __doc__ (docstring) of the properties to create the variable documentation. --- autosubmit/helpers/parameters.py | 96 ++++ autosubmit/job/job.py | 462 +++++++++++++----- autosubmit/platforms/platform.py | 155 +++++- .../_templates/autosummary/variables.rst | 42 ++ docs/source/conf.py | 7 +- docs/source/database/index.rst | 2 +- docs/source/ext/autosubmit_variables.py | 66 +++ docs/source/userguide/variables.rst | 247 ++++++---- test/unit/helpers/test_parameters.py | 53 ++ 9 files changed, 897 insertions(+), 233 deletions(-) create mode 100644 autosubmit/helpers/parameters.py create mode 100644 docs/source/_templates/autosummary/variables.rst create mode 100644 docs/source/ext/autosubmit_variables.py create mode 100644 test/unit/helpers/test_parameters.py diff --git a/autosubmit/helpers/parameters.py b/autosubmit/helpers/parameters.py new file mode 100644 index 000000000..d28e95b9a --- /dev/null +++ b/autosubmit/helpers/parameters.py @@ -0,0 +1,96 @@ +import functools +import inspect +from collections import defaultdict + +PARAMETERS = defaultdict(defaultdict) +"""Global default dictionary holding a multi-level dictionary with the Autosubmit +parameters. At the first level we have the paramete groups. + + - ``JOB`` + + - ``PLATFORM`` + + - ``PROJECT`` + +Each entry in the ``PARAMETERS`` dictionary holds another default dictionary. Finally, +the lower level in the dictionary has a ``key=value`` where ``key`` is the parameter +name, and ``value`` the parameter documentation. + +These values are used to create the Sphinx documentation for variables, as well as +to populate the comments in the Autosubmit YAML configuration files. +""" + + +def autosubmit_parameters(cls=None, /, *, parameters: dict): + """Decorator for Autosubmit configuration parameters defined in a class. + + This is useful for parameters that are not defined in a single function or + class (e.g. parameters that are created on-the-fly in functions).""" + + def wrap(cls): + parameters = wrap.parameters + + for group, group_parameters in parameters.items(): + group = group.upper() + + if group not in PARAMETERS: + PARAMETERS[group] = defaultdict(defaultdict) + + for parameter_name, parameter_value in group_parameters.items(): + if parameter_name not in PARAMETERS[group]: + PARAMETERS[group][parameter_name] = parameter_value.strip() + + return cls + + wrap.parameters = parameters + + if cls is None: + return wrap + + return wrap(cls) + + +def autosubmit_parameter(func=None, *, name, group=None): + """Decorator for Autosubmit configuration parameters. + + Used to annotate properties of classes + + Attributes: + func (Callable): wrapped function. + name (Union[str, List[str]]): parameter name. + group (str): group name. Default to caller module name. + """ + if group is None: + stack = inspect.stack() + group: str = stack[1][0].f_locals['__qualname__'].rsplit('.', 1)[-1] + + group = group.upper() + + if group not in PARAMETERS: + PARAMETERS[group] = defaultdict(defaultdict) + + names = name + if type(name) is not list: + names = [name] + + for parameter_name in names: + if parameter_name not in PARAMETERS[group]: + PARAMETERS[group][parameter_name] = None + + def parameter_decorator(func): + group = parameter_decorator.group + names = parameter_decorator.names + for name in names: + if func.__doc__: + PARAMETERS[group][name] = func.__doc__.strip().split('\n')[0] + + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + parameter_decorator.group = group + parameter_decorator.names = names + + return parameter_decorator diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index fc9e80e68..3598e8bb2 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -46,8 +46,10 @@ from autosubmit.platforms.paramiko_submitter import ParamikoSubmitter from log.log import Log, AutosubmitCritical, AutosubmitError from typing import List, Union from functools import reduce -Log.get_logger("Autosubmit") from autosubmitconfigparser.config.yamlparser import YAMLParserFactory +from autosubmit.helpers.parameters import autosubmit_parameter, autosubmit_parameters + +Log.get_logger("Autosubmit") # A wrapper for encapsulate threads , TODO: Python 3+ to be replaced by the < from concurrent.futures > @@ -61,21 +63,70 @@ def threaded(fn): return wrapper +# This decorator contains groups of parameters, with each +# parameter described. This is only for parameters which +# are not properties of Job. Otherwise, please use the +# ``autosubmit_parameter`` (singular!) decorator for the +# ``@property`` annotated members. The variable groups +# are cumulative, so you can add to ``job``, for instance, +# in multiple files as long as the variable names are +# unique per group. +@autosubmit_parameters( + parameters={ + 'chunk': { + 'day_before': 'Day before the start date.', + 'chunk_end_in_days': 'Chunk length in days.', + 'chunk_start_date': 'Chunk start date.', + 'chunk_start_year': 'Chunk start year.', + 'chunk_start_month': 'Chunk start month.', + 'chunk_start_day': 'Chunk start day.', + 'chunk_start_hour': 'Chunk start hour.', + 'chunk_end_date': 'Chunk end date.', + 'chunk_end_year': 'Chunk end year.', + 'chunk_end_month': 'Chunk end month.', + 'chunk_end_day': 'Chunk end day.', + 'chunk_end_hour': 'Chunk end hour.', + 'prev': 'Days since start date at the chunk\'s start.', + 'chunk_first': 'True if the current chunk is the first, false otherwise.', + 'chunk_last': 'True if the current chunk is the last, false otherwise.', + 'notify_on': 'Determine the job statuses you want to be notified.' + }, + 'config': { + 'config.autosubmit_version': 'Current version of Autosubmit.', + 'config.totaljobs': 'Total number of jobs in the workflow.', + 'config.maxwaitingjobs': 'Maximum number of jobs permitted in the waiting status.' + }, + 'experiment': { + 'experiment.datelist': 'List of start dates', + 'experiment.calendar': 'Calendar used for the experiment. Can be standard or noleap.', + 'experiment.chunksize': 'Size of each chunk.', + 'experiment.numchunks': 'Number of chunks of the experiment.', + 'experiment.chunksizeunit': 'Unit of the chunk size. Can be hour, day, month, or year.', + 'experiment.members': 'List of members.' + }, + 'default': { + 'default.expid': 'Job experiment ID.', + 'default.hpcarch': 'Default HPC platform name.', + 'default.custom_config': 'Custom configuration location.', + }, + 'job': { + 'rootdir': 'Experiment folder path.', + 'projdir': 'Project folder path.', + 'nummembers': 'Number of members of the experiment.' + }, + 'project': { + 'project.project_type': 'Type of the project.', + 'project.project_destination': 'Folder to hold the project sources.' + } + } +) class Job(object): - """ - Class to handle all the tasks with Jobs at HPC. + """Class to handle all the tasks with Jobs at HPC. + A job is created by default with a name, a jobid, a status and a type. It can have children and parents. The inheritance reflects the dependency between jobs. - If Job2 must wait until Job1 is completed then Job2 is a child of Job1. Inversely Job1 is a parent of Job2 - - :param name: job's name - :type name: str - :param job_id: job's id - :type job_id: int - :param status: job initial status - :type status: Status - :param priority: job's priority - :type priority: int + If Job2 must wait until Job1 is completed then Job2 is a child of Job1. + Inversely Job1 is a parent of Job2 """ CHECK_ON_SUBMISSION = 'on_submission' @@ -86,7 +137,7 @@ class Job(object): def __init__(self, name, job_id, status, priority): self.script_name_wrapper = None self.delay_end = datetime.datetime.now() - self.delay_retrials = "0" + self._delay_retrials = "0" self.wrapper_type = None self._wrapper_queue = None self._platform = None @@ -95,34 +146,35 @@ class Job(object): self.retry_delay = "0" self.platform_name = None # type: str - self.section = None # type: str - self.wallclock = None # type: str + #: (str): Type of the job, as given on job configuration file. (job: TASKTYPE) + self._section = None # type: str + self._wallclock = None # type: str self.wchunkinc = None - self.tasks = '1' - self.nodes = "" + self._tasks = '1' + self._nodes = "" self.default_parameters = {'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'} - self.threads = '1' - self.processors = '1' - self.memory = '' - self.memory_per_task = '' - self.chunk = None - self.member = None + self._threads = '1' + self._processors = '1' + self._memory = '' + self._memory_per_task = '' + self._chunk = None + self._member = None self.date = None self.name = name - self.split = None - self.delay = None - self.frequency = None - self.synchronize = None + self._split = None + self._delay = None + self._frequency = None + self._synchronize = None self.skippable = False self.repacked = 0 self._long_name = None self.long_name = name self.date_format = '' self.type = Type.BASH - self.hyperthreading = "none" - self.scratch_free_space = None - self.custom_directives = [] + self._hyperthreading = "none" + self._scratch_free_space = None + self._custom_directives = [] self.undefined_variables = set() self.log_retries = 5 self.id = job_id @@ -140,7 +192,8 @@ class Job(object): self.priority = priority self._parents = set() self._children = set() - self.fail_count = 0 + #: (int) Number of failed attempts to run this job. (FAIL_COUNT) + self._fail_count = 0 self.expid = name.split('_')[0] # type: str self.parameters = dict() self._tmp_path = os.path.join( @@ -149,18 +202,249 @@ class Job(object): self._platform = None self.check = 'true' self.check_warnings = False - self.packed = False + self._packed = False self.hold = False # type: bool self.distance_weight = 0 self.level = 0 - self.export = "none" - self.dependencies = [] + self._export = "none" + self._dependencies = [] self.running = "once" self.start_time = None self.edge_info = dict() self.total_jobs = None self.max_waiting_jobs = None self.exclusive = "" + + @property + @autosubmit_parameter(name='tasktype') + def section(self): + """Type of the job, as given on job configuration file.""" + return self._section + + @section.setter + def section(self, value): + self._section = value + + @property + @autosubmit_parameter(name='jobname') + def name(self): + """Current job full name.""" + return self._name + + @name.setter + def name(self, value): + self._name = value + + @property + @autosubmit_parameter(name='fail_count') + def fail_count(self): + """Number of failed attempts to run this job.""" + return self._fail_count + + @fail_count.setter + def fail_count(self, value): + self._fail_count = value + + @property + @autosubmit_parameter(name='sdate') + def sdate(self): + """Current start date.""" + return date2str(self.date, self.date_format) + + @property + @autosubmit_parameter(name='member') + def member(self): + """Current member.""" + return self._member + + @member.setter + def member(self, value): + self.value = value + + @property + @autosubmit_parameter(name='chunk') + def chunk(self): + """Current chunk.""" + return self._chunk + + @chunk.setter + def chunk(self, value): + self.value = value + + @property + @autosubmit_parameter(name='split') + def split(self): + """Current split.""" + return self._split + + @split.setter + def split(self, value): + self._split = value + + @property + @autosubmit_parameter(name='delay') + def delay(self): + """Current delay.""" + return self._delay + + @delay.setter + def delay(self, value): + self._delay = value + + @delay.setter + def delay(self, value): + self._delay = value + + @property + @autosubmit_parameter(name='wallclock') + def wallclock(self): + """Duration for which nodes used by job will remain allocated.""" + return self._wallclock + + @wallclock.setter + def wallclock(self, value): + self._wallclock = value + + @property + @autosubmit_parameter(name='hyperthreading') + def hyperthreading(self): + """Detects if hyperthreading is enabled or not.""" + return self._hyperthreading + + @hyperthreading.setter + def hyperthreading(self, value): + self._hyperthreading = value + + @property + @autosubmit_parameter(name='nodes') + def nodes(self): + """Number of nodes that the job will use.""" + return self._nodes + + @nodes.setter + def nodes(self, value): + self._nodes = value + + @property + @autosubmit_parameter(name=['numthreads', 'threads', 'cpus_per_task']) + def threads(self): + """Number of threads that the job will use.""" + return self._threads + + @threads.setter + def threads(self, value): + self._threads = value + + @property + @autosubmit_parameter(name=['numtask', 'tasks', 'tasks_per_node']) + def tasks(self): + """Number of tasks that the job will use.""" + return self._tasks + + @tasks.setter + def tasks(self, value): + self._tasks = value + + @property + @autosubmit_parameter(name='scratch_free_space') + def scratch_free_space(self): + """Percentage of free space required on the ``scratch``.""" + return self._scratch_free_space + + @scratch_free_space.setter + def scratch_free_space(self, value): + self._scratch_free_space = value + + @property + @autosubmit_parameter(name='memory') + def memory(self): + """Memory requested for the job.""" + return self._memory + + @memory.setter + def memory(self, value): + self._memory = value + + @property + @autosubmit_parameter(name='memory_per_task') + def memory_per_task(self): + """Memory requested per task.""" + return self._memory_per_task + + @memory_per_task.setter + def memory_per_task(self, value): + self._memory_per_task = value + + @property + @autosubmit_parameter(name='frequency') + def frequency(self): + """TODO.""" + return self._frequency + + @frequency.setter + def frequency(self, value): + self._frequency = value + + @property + @autosubmit_parameter(name='synchronize') + def synchronize(self): + """TODO.""" + return self._synchronize + + @synchronize.setter + def synchronize(self, value): + self._synchronize = value + + @property + @autosubmit_parameter(name='dependencies') + def dependencies(self): + """Current job dependencies.""" + return self._dependencies + + @dependencies.setter + def dependencies(self, value): + self._dependencies = value + + @property + @autosubmit_parameter(name='delay_retrials') + def delay_retrials(self): + """TODO""" + return self._delay_retrials + + @delay_retrials.setter + def delay_retrials(self, value): + self._delay_retrials = value + + @property + @autosubmit_parameter(name='packed') + def packed(self): + """TODO""" + return self._packed + + @packed.setter + def packed(self, value): + self._packed = value + + @property + @autosubmit_parameter(name='export') + def export(self): + """TODO.""" + return self._export + + @export.setter + def export(self, value): + self._export = value + + @property + @autosubmit_parameter(name='custom_directives') + def custom_directives(self): + """List of custom directives.""" + return self._custom_directives + + @custom_directives.setter + def custom_directives(self, value): + self._custom_directives = value + def __getstate__(self): odict = self.__dict__ if '_platform' in odict: @@ -168,23 +452,6 @@ class Job(object): del odict['_platform'] # remove filehandle entry return odict - # def __str__(self): - # return self.name - - def print_job(self): - """ - Prints debug information about the job - """ - Log.debug('NAME: {0}', self.name) - Log.debug('JOBID: {0}', self.id) - Log.debug('STATUS: {0}', self.status) - Log.debug('PRIORITY: {0}', self.priority) - Log.debug('TYPE: {0}', self.type) - Log.debug('PARENTS: {0}', [p.name for p in self.parents]) - Log.debug('CHILDREN: {0}', [c.name for c in self.children]) - Log.debug('FAIL_COUNT: {0}', self.fail_count) - Log.debug('EXPID: {0}', self.expid) - @property def parents(self): """ @@ -244,9 +511,10 @@ class Job(object): self._platform = value @property + @autosubmit_parameter(name="current_queue") def queue(self): """ - Returns the queue to be used by the job. Chooses between serial and parallel platforms + Returns the queue to be used by the job. Chooses between serial and parallel platforms. :return HPCPlatform object for the job to use :rtype: HPCPlatform @@ -366,21 +634,15 @@ class Job(object): return float(minutes) / 60 + float(hours) return 0 - def log_job(self): - """ - Prints job information in log - """ - Log.debug("{0}\t{1}\t{2}", "Job Name", "Job Id", "Job Status") - Log.debug("{0}\t\t{1}\t{2}", self.name, self.id, self.status) - - #Log.status("{0}\t{1}\t{2}", "Job Name", "Job Id", "Job Status") - #Log.status("{0}\t\t{1}\t{2}", self.name, self.id, self.status) + @property + @autosubmit_parameter(name=['numproc', 'processors']) + def processors(self): + """Number of processors that the job will use.""" + return self._processors - def print_parameters(self): - """ - Print sjob parameters in log - """ - Log.info(self.parameters) + @processors.setter + def processors(self, value): + self._processors = value def inc_fail_count(self): """ @@ -465,39 +727,6 @@ class Job(object): """ return self.parents.__len__() - def compare_by_status(self, other): - """ - Compare jobs by status value - - :param other: job to compare - :type other: Job - :return: comparison result - :rtype: bool - """ - return self.status < other.status - - def compare_by_id(self, other): - """ - Compare jobs by ID - - :param other: job to compare - :type other: Job - :return: comparison result - :rtype: bool - """ - return self.id < other.id - - def compare_by_name(self, other): - """ - Compare jobs by name - - :param other: job to compare - :type other: Job - :return: comparison result - :rtype: bool - """ - return self.name < other.name - def _get_from_stat(self, index): """ Returns value from given row index position in STAT file associated to job @@ -555,15 +784,6 @@ class Job(object): """ return self._get_from_stat(0) - def check_retrials_submit_time(self): - """ - Returns list of submit datetime for retrials from total stats file - - :return: date and time - :rtype: list[int] - """ - return self._get_from_total_stats(0) - def check_retrials_end_time(self): """ Returns list of end datetime for retrials from total stats file @@ -849,6 +1069,7 @@ class Job(object): if param: time_params[name] = int(param) return datetime.timedelta(**time_params),format_ + # Duplicated for wrappers and jobs to fix in 4.0.0 def is_over_wallclock(self, start_time, wallclock): """ @@ -1019,6 +1240,7 @@ class Job(object): parameters['CURRENT_LOGDIR'] = job_platform.get_files_path() return parameters + def update_platform_associated_parameters(self,as_conf, parameters, job_platform, chunk): self.executable = str(as_conf.jobs_data[self.section].get("EXECUTABLE", as_conf.platforms_data.get(job_platform.name,{}).get("EXECUTABLE",""))) self.total_jobs = int(as_conf.jobs_data[self.section].get("TOTALJOBS", job_platform.total_jobs)) @@ -1073,10 +1295,10 @@ class Job(object): parameters['NUMTHREADS'] = self.threads parameters['THREADS'] = self.threads parameters['CPUS_PER_TASK'] = self.threads - parameters['NUMTASK'] = self.tasks - parameters['TASKS'] = self.tasks + parameters['NUMTASK'] = self._tasks + parameters['TASKS'] = self._tasks parameters['NODES'] = self.nodes - parameters['TASKS_PER_NODE'] = self.tasks + parameters['TASKS_PER_NODE'] = self._tasks parameters['WALLCLOCK'] = self.wallclock parameters['TASKTYPE'] = self.section parameters['SCRATCH_FREE_SPACE'] = self.scratch_free_space @@ -1084,6 +1306,7 @@ class Job(object): parameters['HYPERTHREADING'] = self.hyperthreading parameters['CURRENT_QUEUE'] = self.queue return parameters + def update_wrapper_parameters(self,as_conf, parameters): wrappers = as_conf.experiment_data.get("WRAPPERS", {}) if len(wrappers) > 0: @@ -1107,10 +1330,11 @@ class Job(object): parameters[wrapper_section + "_EXTENSIBLE"] = int( as_conf.get_extensible_wallclock(as_conf.experiment_data["WRAPPERS"].get(wrapper_section))) return parameters + def update_job_parameters(self,as_conf, parameters): parameters['JOBNAME'] = self.name parameters['FAIL_COUNT'] = str(self.fail_count) - parameters['SDATE'] = date2str(self.date, self.date_format) + parameters['SDATE'] = self.sdate parameters['MEMBER'] = self.member parameters['SPLIT'] = self.split parameters['DELAY'] = self.delay @@ -1203,6 +1427,7 @@ class Job(object): parameters['PROJECT_TYPE'] = as_conf.get_project_type() self.wchunkinc = as_conf.get_wchunkinc(self.section) return parameters + def update_parameters(self, as_conf, parameters, default_parameters={'d': '%d%', 'd_': '%d_%', 'Y': '%Y%', 'Y_': '%Y_%', 'M': '%M%', 'M_': '%M_%', 'm': '%m%', 'm_': '%m_%'}): @@ -1216,7 +1441,6 @@ class Job(object): :param parameters: :type parameters: dict """ - chunk = 1 as_conf.reload() parameters = parameters.copy() parameters.update(as_conf.parameters) @@ -1233,8 +1457,9 @@ class Job(object): parameters = as_conf.normalize_parameters_keys(parameters,default_parameters) # For some reason, there is return but the assignee is also necessary self.parameters = parameters - # This returns is only being used by the mock , to change the mock + # This return is only being used by the mock , to change the mock return parameters + def update_content_extra(self,as_conf,files): additional_templates = [] for file in files: @@ -1244,6 +1469,7 @@ class Job(object): template = open(os.path.join(as_conf.get_project_dir(), file), 'r').read() additional_templates += [template] return additional_templates + def update_content(self, as_conf): """ Create the script content to be run for the job @@ -1581,12 +1807,6 @@ class Job(object): thread_write_finish.name = "JOB_data_{}".format(self.name) thread_write_finish.start() - def write_total_stat_by_retries_fix_newline(self): - path = os.path.join(self._tmp_path, self.name + '_TOTAL_STATS') - f = open(path, 'a') - f.write('\n') - f.close() - def write_total_stat_by_retries(self,total_stats, first_retrial = False): """ Writes all data to TOTAL_STATS file diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index a3af43674..1f23cc6fc 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -5,6 +5,7 @@ import traceback from autosubmit.job.job_common import Status from typing import List, Union +from autosubmit.helpers.parameters import autosubmit_parameter from log.log import AutosubmitCritical, AutosubmitError, Log class Platform(object): @@ -20,7 +21,7 @@ class Platform(object): """ self.connected = False self.expid = expid # type: str - self.name = name # type: str + self._name = name # type: str self.config = config self.tmp_path = os.path.join( self.config.get("LOCAL_ROOT_DIR"), self.expid, self.config.get("LOCAL_TMP_DIR")) @@ -33,21 +34,21 @@ class Platform(object): self.processors_per_node = "1" self.scratch_free_space = None self.custom_directives = None - self.host = '' - self.user = '' - self.project = '' - self.budget = '' - self.reservation = '' - self.exclusivity = '' - self.type = '' - self.scratch = '' - self.project_dir = '' + self._host = '' + self._user = '' + self._project = '' + self._budget = '' + self._reservation = '' + self._exclusivity = '' + self._type = '' + self._scratch = '' + self._project_dir = '' self.temp_dir = '' - self.root_dir = '' + self._root_dir = '' self.service = None self.scheduler = None self.directory = None - self.hyperthreading = False + self._hyperthreading = False self.max_wallclock = '2:00' self.total_jobs = 20 self.max_processors = "480" @@ -62,6 +63,127 @@ class Platform(object): self._submit_cmd = None self._checkhost_cmd = None self.cancel_cmd = None + + @property + @autosubmit_parameter(name='current_arch') + def name(self): + """Platform name.""" + return self._name + + @name.setter + def name(self, value): + self._name = value + + @property + @autosubmit_parameter(name='current_host') + def host(self): + """Platform url.""" + return self._host + + @host.setter + def host(self, value): + self._host = value + + @property + @autosubmit_parameter(name='current_user') + def user(self): + """Platform user.""" + return self._user + + @user.setter + def user(self, value): + self._user = value + + @property + @autosubmit_parameter(name='current_proj') + def project(self): + """Platform project.""" + return self._project + + @project.setter + def project(self, value): + self._project = value + + @property + @autosubmit_parameter(name='current_budg') + def budget(self): + """Platform budget.""" + return self._budget + + @budget.setter + def budget(self, value): + self._budget = value + + @property + @autosubmit_parameter(name='current_reservation') + def reservation(self): + """You can configure your reservation id for the given platform.""" + return self._reservation + + @reservation.setter + def reservation(self, value): + self._reservation = value + + @property + @autosubmit_parameter(name='current_exclusivity') + def exclusivity(self): + """True if you want to request exclusivity nodes.""" + return self._exclusivity + + @exclusivity.setter + def exclusivity(self, value): + self._exclusivity = value + + @property + @autosubmit_parameter(name='current_hyperthreading') + def hyperthreading(self): + """TODO""" + return self._hyperthreading + + @hyperthreading.setter + def hyperthreading(self, value): + self._hyperthreading = value + + @property + @autosubmit_parameter(name='current_type') + def type(self): + """Platform scheduler type.""" + return self._type + + @type.setter + def type(self, value): + self._type = value + + @property + @autosubmit_parameter(name='current_scratch_dir') + def scratch(self): + """Platform's scratch folder path.""" + return self._scratch + + @scratch.setter + def scratch(self, value): + self._scratch = value + + @property + @autosubmit_parameter(name='current_proj_dir') + def project_dir(self): + """Platform's project folder path.""" + return self._project_dir + + @project_dir.setter + def project_dir(self, value): + self._project_dir = value + + @property + @autosubmit_parameter(name='current_rootdir') + def root_dir(self): + """Platform's experiment folder path.""" + return self._root_dir + + @root_dir.setter + def root_dir(self, value): + self._root_dir = value + def get_exclusive_directive(self, job): """ Returns exclusive directive for the specified job @@ -197,10 +319,13 @@ class Platform(object): @serial_platform.setter def serial_platform(self, value): self._serial_platform = value + @property + @autosubmit_parameter(name='current_partition') def partition(self): """ - Partition to use for jobs + Partition to use for jobs. + :return: queue's name :rtype: str """ @@ -211,6 +336,7 @@ class Platform(object): @partition.setter def partition(self, value): self._partition = value + @property def queue(self): """ @@ -524,9 +650,10 @@ class Platform(object): else: return False + @autosubmit_parameter(name='current_logdir') def get_files_path(self): """ - Get the path to the platform's LOG directory + The platform's LOG directory. :return: platform's LOG directory :rtype: str diff --git a/docs/source/_templates/autosummary/variables.rst b/docs/source/_templates/autosummary/variables.rst new file mode 100644 index 000000000..8c3055412 --- /dev/null +++ b/docs/source/_templates/autosummary/variables.rst @@ -0,0 +1,42 @@ +.. currentmodule:: {{ module }} + +################### +Variables reference +################### + +Autosubmit uses a variable substitution system to facilitate the development +of the templates. This variables can be used on the template in the form +``%VARIABLE_NAME%``. + +All configuration variables non related to current_job or platform are +accessible by calling first to their parents. ex: ``%PROJECT.PROJECT_TYPE%`` +or ``%DEFAULT.EXPID%``. + +You can review all variables at any given time by using the :ref:`report ` +command, as shown in the example below. + + +.. code-block:: console + + $ autosubmit report $expid -all + + +Job variables +============= + +{{ attributes }} + + +.. autoclass:: {{ objname }} + :noindex: + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 85e0befed..44c17dc01 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,6 +20,7 @@ import os # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../..')) +sys.path.append(os.path.abspath('ext')) # path to custom extensions. # -- General configuration ------------------------------------------------ @@ -34,8 +35,12 @@ extensions = [ 'sphinx.ext.imgmath', 'sphinx.ext.autosectionlabel', 'sphinx_rtd_theme', - 'sphinx_reredirects' + 'sphinx_reredirects', + 'sphinx.ext.autosummary', + 'sphinx.ext.napoleon', + 'autosubmit_variables' ] +autosummary_generate = True autosectionlabel_prefix_document = True diff --git a/docs/source/database/index.rst b/docs/source/database/index.rst index f6fe1113b..32f3896e7 100644 --- a/docs/source/database/index.rst +++ b/docs/source/database/index.rst @@ -12,7 +12,7 @@ There is one central database that supports the core functionality of experiment The name and location of the central database are defined in the .autosubmitrc config file while the other auxiliary DBs have a predefined name. There are also log files with important information about experiment execution and some other relevant information such as experiment job statuses, timestamps, error messages among other things inside these files. .. figure:: fig/dbs-highlevel.png - :name: simple + :name: simple_database :width: 100% :align: center :alt: High level view of the Autosubmit storage system diff --git a/docs/source/ext/autosubmit_variables.py b/docs/source/ext/autosubmit_variables.py new file mode 100644 index 000000000..0cc7d8200 --- /dev/null +++ b/docs/source/ext/autosubmit_variables.py @@ -0,0 +1,66 @@ +"""Autosubmit variables directive.""" +import logging + +from docutils.parsers.rst import Directive +from docutils.statemachine import StringList + +from sphinx import addnodes + +from autosubmit.helpers.parameters import PARAMETERS + +__version__ = 0.1 +logger = logging.getLogger(__name__) + + +class AutosubmitVariablesDirective(Directive): + """A custom Sphinx directive that prints Autosubmit variables. + + It is able to recognize variables and separate them in groups, + producing valid Sphinx documentation directly from the Python + docstrings. + """ + + has_content = True + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = False + + def run(self): + rst = [ + '', + '.. list-table::', + ' :widths: 25 75', + ' :header-rows: 1', + ' ', + ' * - Variable', + ' - Description' + ] + + parameters_group = self.arguments[0].upper() + if parameters_group not in PARAMETERS: + logger.error(f'Parameter group {parameters_group} not set') + return [] + + parameters = sorted(PARAMETERS[parameters_group].items()) + + for parameter_name, parameter_doc in parameters: + # rst.append(f'- **{parameter_name.upper()}**: {parameter_doc}') + rst.extend([f' * - **{parameter_name.upper()}**', f' - {parameter_doc}']) + + rst.extend(['', '']) + + node = addnodes.desc() + self.state.nested_parse( + StringList(rst), + self.content_offset, + node + ) + return [node] + + +def setup(app): + app.add_directive('autosubmit-variables', AutosubmitVariablesDirective) + return { + 'version': __version__, + 'parallel_read_safe': True + } diff --git a/docs/source/userguide/variables.rst b/docs/source/userguide/variables.rst index 6919ba639..b7847a8e5 100644 --- a/docs/source/userguide/variables.rst +++ b/docs/source/userguide/variables.rst @@ -2,119 +2,174 @@ Variables reference ################### -Autosubmit uses a variable substitution system to facilitate the development of the templates. This variables can be -used on the template in the form %VARIABLE_NAME%. +Autosubmit uses a variable substitution system to facilitate the +development of the templates. These variables can be used on templates +with the syntax ``%VARIABLE_NAME%``. -All configuration variables non related to current_job or platform are accessible by calling first to their parents. ex: %PROJECT.PROJECT_TYPE% or %DEFAULT.EXPID% +All configuration variables that are not related to the current job +or platform are accessible by accessing first their parents, e.g. +``%PROJECT.PROJECT_TYPE% or %DEFAULT.EXPID%``. -You can review all variables at any given time by using the command :ref:`report`: +You can review all variables at any given time by using the +:ref:`report ` command, as illustrated below. - $ autosubmit report expid -all + +.. code-block:: console + :caption: Example usage of ``autosubmit report`` + + $ autosubmit report $expid -all + +The command will save the list of variables available to a file +in the experiment area. Each group of variables of Autosubmit are +detailed in the next sections on this page. + +.. note:: All the variable tables are displayed in alphabetical order. + + +.. note:: + + Configuration files such as ``myapp.yml`` may contain some + configuration like: + + .. code-block:: yaml + + MYAPP: + MYPARAMETER: 42 + ANOTHER_PARAMETER: 1984 + + If you configured Autosubmit to include this file with the + rest of your configuration, then those variables will be + available to each job, and can be accessed with: + ``%MYAPP.MYPARAMETER%`` and ``%MYAPP.ANOTHER_PARAMETER%``. Job variables ============= -This variables are relatives to the current job. - -- **TASKTYPE**: type of the job, as given on job configuration file. -- **JOBNAME**: current job full name. -- **FAIL_COUNT**: number of failed attempts to run this job. -- **SDATE**: current startdate. -- **MEMBER**: current member. -- **CHUNK**: current chunk. -- **SPLIT**: current split. -- **DELAY**: current delay. -- **DAY_BEFORE**: day before the startdate -- **Chunk_End_IN_DAYS**: chunk's length in days -- **Chunk_START_DATE**: chunk's start date -- **Chunk_START_YEAR**: chunk's start year -- **Chunk_START_MONTH**: chunk's start month -- **Chunk_START_DAY**: chunk's start day -- **Chunk_START_HOUR**: chunk's start hour -- **Chunk_END_DATE**: chunk's end date -- **Chunk_END_YEAR**: chunk's end year -- **Chunk_END_MONTH**: chunk's end month -- **Chunk_END_DAY**: chunk's end day -- **Chunk_END_HOUR**: chunk's end hour -- **STARTDATES**: list of startdates. -- **PREV**: days since startdate at the chunk's start -- **Chunk_FIRST**: True if the current chunk is the first, false otherwise. -- **Chunk_LAST**: True if the current chunk is the last, false otherwise. -- **NUMPROC**: Number of processors that the job will use. -- **NUMTHREADS**: Number of threads that the job will use. -- **NUMTASKS**: Number of tasks that the job will use. -- **NODES**: Number of nodes that the job will use. -- **HYPERTHREADING**: Detects if hyperthreading is enabled or not. -- **WALLCLOCK**: Number of processors that the job will use. -- **SCRATCH_FREE_SPACE**: Percentage of free space required on the ``scratch``. -- **NOTIFY_ON**: Determine the job statuses you want to be notified. +These variables are relatives to the current job. These variables +appear in the output of the :ref:`report ` command with the +pattern ``JOBS.${JOB_ID}.${JOB_VARIABLE}=${VALUE}``. They can be used in +templates with ``%JOB_VARIABLE%``. + +.. autosubmit-variables:: job + + +The following variables are present only in jobs that contain a date +(e.g. ``RUNNING=date``). + + +.. autosubmit-variables:: chunk + Platform variables ================== -This variables are relative to the platforms defined on the jobs conf. A full set of the next variables are defined for -each platform defined on the platforms configuration file, substituting {PLATFORM_NAME} for each platform's name. Also, a -suite of variables is defined for the current platform where {PLATFORM_NAME} is substituted by CURRENT. - -- **{PLATFORM_NAME}_ARCH**: Platform name -- **{PLATFORM_NAME}_HOST**: Platform url -- **{PLATFORM_NAME}_USER**: Platform user -- **{PLATFORM_NAME}_PROJ**: Platform project -- **{PLATFORM_NAME}_BUDG**: Platform budget -- **{PLATFORM_NAME}_PARTITION**: Platform partition -- **{PLATFORM_NAME}_RESERVATION**: You can configure your reservation id for the given platform. -- **{PLATFORM_NAME}_EXCLUSIVITY**: True if you want to request exclusivity nodes. -- **{PLATFORM_NAME}_TYPE**: Platform scheduler type -- **{PLATFORM_NAME}_VERSION**: Platform scheduler version -- **{PLATFORM_NAME}_SCRATCH_DIR**: Platform's scratch folder path -- **{PLATFORM_NAME}_ROOTDIR**: Platform's experiment folder path -- **{PLATFORM_NAME}_CUSTOM_DIRECTIVES**: Platform's custom directives for the resource manager. - -.. hint:: - The variables ``_USER``, ``_PROJ`` and ``_BUDG`` has no value on the LOCAL platform. - -.. hint:: - Until now, the variables ``_RESERVATION`` and ``_EXCLUSIVITY`` are only available for MN. - -It is also defined a suite of variables for the experiment's default platform: - -- **HPCARCH**: Default HPC platform name -- **HPCHOST**: Default HPC platform url -- **HPCUSER**: Default HPC platform user -- **HPCPROJ**: Default HPC platform project -- **HPCBUDG**: Default HPC platform budget -- **HPCTYPE**: Default HPC platform scheduler type -- **HPCVERSION**: Default HPC platform scheduler version -- **SCRATCH_DIR**: Default HPC platform scratch folder path -- **HPCROOTDIR**: Default HPC platform experiment's folder path - - -Project variables +These variables are relative to the platforms defined in each +job configuration. The table below shows the complete set of variables +available in the current platform. These variables appear in the +output of the :ref:`report ` command with the pattern +``JOBS.${JOB_ID}.${PLATFORM_VARIABLE}=${VALUE}``. They can be used in +templates with ``%PLATFORM_VARIABLE%``. + +A series of variables is also available in each platform, and appear +in the output of the :ref:`report ` command with the pattern +``JOBS.${JOB_ID}.PLATFORMS.${PLATFORM_ID}.${PLATFORM_VARIABLE}=${VALUE}``. +They can be used in templates with ``PLATFORMS.%PLATFORM_ID%.%PLATFORM_VARIABLE%``. + +.. autosubmit-variables:: platform + + +.. note:: + The variables ``_USER``, ``_PROJ`` and ``_BUDG`` + have no value on the LOCAL platform. + + Certain variables (e.g. ``_RESERVATION``, + ``_EXCLUSIVITY``) are only available for certain + platforms (e.g. MareNostrum). + +A set of variables for the experiment's default platform are +also available. + +.. TODO: Some variables do not exist anymore, like HPCHOST, HPCUSER, HPCDUG, etc. + +.. list-table:: + :widths: 25 75 + :header-rows: 1 + + * - Variable + - Description + * - **HPCARCH** + - Default HPC platform name. + * - **HPCHOST** + - Default HPC platform url. + * - **HPCUSER** + - Default HPC platform user. + * - **HPCPROJ** + - Default HPC platform project. + * - **HPCBUDG** + - Default HPC platform budget. + * - **HPCTYPE** + - Default HPC platform scheduler type. + * - **HPCVERSION** + - Default HPC platform scheduler version. + * - **SCRATCH_DIR** + - Default HPC platform scratch folder path. + * - **HPCROOTDIR** + - Default HPC platform experiment's folder path. + +Other variables ================= -- **NUMMEMBERS**: number of members of the experiment -- **NUMCHUNKS**: number of chunks of the experiment -- **CHUNKSIZE**: size of each chunk -- **CHUNKSIZEUNIT**: unit of the chuk size. Can be hour, day, month or year. -- **CALENDAR**: calendar used for the experiment. Can be standard or noleap. -- **ROOTDIR**: local path to experiment's folder -- **PROJDIR**: local path to experiment's proj folder -- **STARTDATES**: Startdates of the experiment +.. autosubmit-variables:: config + + +.. autosubmit-variables:: default + + +.. autosubmit-variables:: experiment + + +.. autosubmit-variables:: project + + +.. note:: + + Depending on your project type other variables may + be available. For example, if you choose Git, then + you should have ``%PROJECT_ORIGIN%``. If you choose + Subversion, then you will have ``%PROJECT_URL%``. + The same variables from the project template (created + with the ``expid`` subcommand) are available in your + job template scripts. + + +Performance Metrics variables +============================= -Performance Metrics -=================== +These variables apply only to the :ref:`report ` subcommand. -Currently, these variables apply only to the report function of Autosubmit. See :ref:`report`. +.. list-table:: + :widths: 25 75 + :header-rows: 1 -- **SYPD**: Simulated years per day. -- **ASYPD**: Actual simulated years per day. -- **RSYPD**: Raw simulated years per day. -- **CHSY**: Core hours per simulated year. -- **JPSY**: Joules per simulated year. -- **Parallelization**: Number of cores requested for the simulation job. + * - Variable + - Description + * - **ASYPD** + - Actual simulated years per day. + * - **CHSY** + - Core hours per simulated year. + * - **JPSY** + - Joules per simulated year. + * - **Parallelization** + - Number of cores requested for the simulation job. + * - **RSYPD** + - Raw simulated years per day. + * - **SYPD** + - Simulated years per day. -For more information about these metrics please visit: -https://earth.bsc.es/gitlab/wuruchi/autosubmitreact/-/wikis/Performance-Metrics. +.. FIXME: this link is broken, and should probably not be under wuruchi's +.. gitlab account. +.. For more information about these metrics please visit +.. https://earth.bsc.es/gitlab/wuruchi/autosubmitreact/-/wikis/Performance-Metrics. diff --git a/test/unit/helpers/test_parameters.py b/test/unit/helpers/test_parameters.py new file mode 100644 index 000000000..a3f2db551 --- /dev/null +++ b/test/unit/helpers/test_parameters.py @@ -0,0 +1,53 @@ +from unittest import TestCase + +from autosubmit.helpers.parameters import autosubmit_parameter, PARAMETERS + + +class TestParameters(TestCase): + """Tests for the ``helpers.parameters`` module.""" + + def test_autosubmit_decorator(self): + """Test the ``autosubmit_decorator``.""" + + parameter_name = 'JOBNAME' + parameter_group = 'PLATFORM' + + class Job: + @property + @autosubmit_parameter(name=parameter_name, group=parameter_group) + def name(self): + """This parameter is the job name.""" + return 'FOO' + + job = Job() + + self.assertEqual('FOO', job.name) + self.assertTrue(len(PARAMETERS) > 0) + # Defaults to the module name if not provided! So the class name + # ``Job`` becomes ``JOB``. + self.assertTrue(parameter_group in PARAMETERS) + self.assertTrue(parameter_name in PARAMETERS[parameter_group]) + self.assertEquals('This parameter is the job name.', PARAMETERS[parameter_group][parameter_name]) + + + def test_autosubmit_decorator_no_group(self): + """Test the ``autosubmit_decorator`` when ``group`` is not provided.""" + + parameter_name = 'JOBNAME' + + class Job: + @property + @autosubmit_parameter(name=parameter_name) + def name(self): + """This parameter is the job name.""" + return 'FOO' + + job = Job() + + self.assertEqual('FOO', job.name) + self.assertTrue(len(PARAMETERS) > 0) + # Defaults to the module name if not provided! So the class name + # ``Job`` becomes ``JOB``. + self.assertTrue(Job.__name__.upper() in PARAMETERS) + self.assertTrue(parameter_name in PARAMETERS[Job.__name__.upper()]) + self.assertEquals('This parameter is the job name.', PARAMETERS[Job.__name__.upper()][parameter_name]) -- GitLab From 5bcc3722620fb41a304659e09be13130a839fa8d Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Tue, 23 May 2023 16:09:48 +0200 Subject: [PATCH 2/8] Augment the copied/generated Autosubmit configuration with the comments for parameters from the code --- autosubmit/autosubmit.py | 104 ++++++++++++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 17 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 4b85f66e7..09124a05c 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -29,6 +29,7 @@ from .notifications.mail_notifier import MailNotifier from bscearth.utils.date import date2str from pathlib import Path from autosubmitconfigparser.config.yamlparser import YAMLParserFactory +from ruamel.yaml import YAML from configparser import ConfigParser from .monitor.monitor import Monitor @@ -46,6 +47,7 @@ from .job.job_packages import JobPackageThread, JobPackageBase from .job.job_list import JobList from .job.job_utils import SubJob, SubJobManager from .job.job import Job +from autosubmit.helpers.parameters import PARAMETERS from .git.autosubmit_git import AutosubmitGit from .job.job_common import Status from autosubmitconfigparser.config.configcommon import AutosubmitConfig @@ -53,7 +55,7 @@ from autosubmitconfigparser.config.basicconfig import BasicConfig import locale from distutils.util import strtobool from log.log import Log, AutosubmitError, AutosubmitCritical -from typing import Set, Union +from typing import Dict, List, Set, Tuple, Union from autosubmit.database.db_common import update_experiment_descrip_version import sqlite3 @@ -1038,25 +1040,93 @@ class Autosubmit: os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf",conf_file.replace(copy_id,exp_id))) except Exception as e: Log.warning("Error converting {0} to yml: {1}".format(conf_file.replace(copy_id,exp_id),str(e))) - @staticmethod - def generate_as_config(exp_id,dummy=False,minimal_configuration=False,local=False): - # obtain from autosubmitconfigparser package - # get all as_conf_files from autosubmitconfigparser package - files = resource_listdir('autosubmitconfigparser.config', 'files') - for as_conf_file in files: + + def generate_as_config(exp_id: str, dummy: bool=False, minimal_configuration: bool=False, local: bool=False) -> None: + """Retrieve the configuration from autosubmitconfigparser package. + + :param exp_id: Experiment ID + :param dummy: Whether the experiment is a dummy one or not. + :param minimal_configuration: Whether the experiment is configured with minimal configuration or not. + :param local: Whether the experiment project type is local or not. + :return: None + """ + + def add_comments_to_yaml(yaml_data, parameters, /, keys=None): + """A recursive generator that visits every leaf node and yields the flatten parameter.""" + if keys is None: + keys = [] + if isinstance(yaml_data, dict): + for key, value in yaml_data.items(): + if isinstance(value, dict): + add_comments_to_yaml(value, parameters, [*keys, key]) + else: + parameter_key = '.'.join([*keys, key]).upper() + if parameter_key in parameters: + comment = parameters[parameter_key] + yaml_data.yaml_set_comment_before_after_key(key, before=comment, indent=yaml_data.lc.col) + + def recurse_into_parameters(parameters: Dict[str, Union[Dict, List, str]], keys=None) -> Tuple[str, str]: + """Recurse into the ``PARAMETERS`` dictionary, and emits a dictionary. + + The key in the dictionary is the flattened parameter key/ID, and the value + is the parameter documentation. + + :param parameters: Global parameters dictionary. + :param keys: For recursion, the accumulated keys. + :return: A dictionary with the + """ + if keys is None: + keys = [] + if isinstance(parameters, dict): + for key, value in parameters.items(): + if isinstance(value, dict): + yield from recurse_into_parameters(value, [*keys, key]) + else: + key = key.upper() + # Here's the reason why ``recurse_into_yaml`` and ``recurse_into_parameters`` + # are not one single ``recurse_into_dict`` function. The parameters have some + # keys that contain ``${PARENT}.key`` as that is how they are displayed in + # the Sphinx docs. So we need to detect it and handle it. p.s. We also know + # the max-length of the parameters dict is 2! See the ``autosubmit.helpers.parameters`` + # module for more. + if not key.startswith(f'{keys[0]}.'): + yield '.'.join([*keys, key]).upper(), value + else: + yield key, value + + template_files = resource_listdir('autosubmitconfigparser.config', 'files') + parameter_comments = dict(recurse_into_parameters(PARAMETERS)) + + for as_conf_file in template_files: + origin = resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file) + target = None + if dummy: - if as_conf_file.endswith("dummy.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file.split("-")[0]+"_"+exp_id+".yml")) + if as_conf_file.endswith('dummy.yml'): + file_name = f'{as_conf_file.split("-")[0]}_{exp_id}.yml' + target = Path(BasicConfig.LOCAL_ROOT_DIR / exp_id / 'conf' / file_name) elif minimal_configuration: - if not local: - if as_conf_file.endswith("git-minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf","minimal.yml")) - else: - if as_conf_file.endswith("local-minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/' + as_conf_file),os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf", "minimal.yml")) + if (not local and as_conf_file.endswith('git-minimal.yml')) or as_conf_file.endswith("local-minimal.yml"): + target = Path(BasicConfig.LOCAL_ROOT_DIR / exp_id / 'conf/minimal.yml') else: - if not as_conf_file.endswith("dummy.yml") and not as_conf_file.endswith("minimal.yml"): - shutil.copy(resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file), os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id, "conf",as_conf_file[:-4]+"_"+exp_id+".yml")) + if not as_conf_file.endswith('dummy.yml') and not as_conf_file.endswith('minimal.yml'): + file_name = f'{as_conf_file[:-4]}_{exp_id}.yml' + target = Path(BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf', file_name) + + # Here we annotate the copied configuration with comments from the Python source code. + # This means the YAML configuration files contain the exact same comments from our + # Python code, which is also displayed in our Sphinx documentation (be careful with what + # you write!) + # + # The previous code was simply doing a shutil(origin, target). This does not modify + # much that logic, except we add comments before writing the copy... + if origin and target: + with open(origin, 'r') as input, open(target, 'w+') as output: + yaml = YAML(typ='rt') + yaml_data = yaml.load(input) + add_comments_to_yaml(yaml_data, parameter_comments) + yaml.dump(yaml_data, output) + @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ -- GitLab From 128f88b5560382d43932edb0bfbfb05e6e390de4 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Tue, 23 May 2023 21:00:45 +0200 Subject: [PATCH 3/8] Fix the build --- autosubmit/autosubmit.py | 2 +- autosubmit/helpers/parameters.py | 3 ++- autosubmit/job/job.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 09124a05c..3654b89d3 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1051,7 +1051,7 @@ class Autosubmit: :return: None """ - def add_comments_to_yaml(yaml_data, parameters, /, keys=None): + def add_comments_to_yaml(yaml_data, parameters, keys=None): """A recursive generator that visits every leaf node and yields the flatten parameter.""" if keys is None: keys = [] diff --git a/autosubmit/helpers/parameters.py b/autosubmit/helpers/parameters.py index d28e95b9a..c5e97d138 100644 --- a/autosubmit/helpers/parameters.py +++ b/autosubmit/helpers/parameters.py @@ -1,6 +1,7 @@ import functools import inspect from collections import defaultdict +from typing import Dict PARAMETERS = defaultdict(defaultdict) """Global default dictionary holding a multi-level dictionary with the Autosubmit @@ -21,7 +22,7 @@ to populate the comments in the Autosubmit YAML configuration files. """ -def autosubmit_parameters(cls=None, /, *, parameters: dict): +def autosubmit_parameters(cls=None, *, parameters: Dict): """Decorator for Autosubmit configuration parameters defined in a class. This is useful for parameters that are not defined in a single function or diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 3598e8bb2..5390a69c1 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -259,7 +259,7 @@ class Job(object): @member.setter def member(self, value): - self.value = value + self._member = value @property @autosubmit_parameter(name='chunk') @@ -269,7 +269,7 @@ class Job(object): @chunk.setter def chunk(self, value): - self.value = value + self._chunk = value @property @autosubmit_parameter(name='split') -- GitLab From fea3719fe1099d748925dece7104bed1cec5c21e Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 24 May 2023 12:15:18 +0200 Subject: [PATCH 4/8] Remove unused code and conf, add unit tests for the docs code --- autosubmit/helpers/parameters.py | 7 ++- autosubmit/job/job.py | 4 -- .../_templates/autosummary/variables.rst | 42 -------------- docs/source/conf.py | 3 - test/unit/helpers/test_parameters.py | 56 ++++++++++++++++++- test/unit/test_slurm_platform.py | 22 ++++++++ 6 files changed, 81 insertions(+), 53 deletions(-) delete mode 100644 docs/source/_templates/autosummary/variables.rst diff --git a/autosubmit/helpers/parameters.py b/autosubmit/helpers/parameters.py index c5e97d138..e6001763e 100644 --- a/autosubmit/helpers/parameters.py +++ b/autosubmit/helpers/parameters.py @@ -45,10 +45,11 @@ def autosubmit_parameters(cls=None, *, parameters: Dict): wrap.parameters = parameters - if cls is None: - return wrap + # NOTE: This is not reachable code, as the parameters must be defined! + # if cls is not None: + # raise ValueError(f'You must provide a list of parameters') - return wrap(cls) + return wrap def autosubmit_parameter(func=None, *, name, group=None): diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 5390a69c1..ab754734e 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -291,10 +291,6 @@ class Job(object): def delay(self, value): self._delay = value - @delay.setter - def delay(self, value): - self._delay = value - @property @autosubmit_parameter(name='wallclock') def wallclock(self): diff --git a/docs/source/_templates/autosummary/variables.rst b/docs/source/_templates/autosummary/variables.rst deleted file mode 100644 index 8c3055412..000000000 --- a/docs/source/_templates/autosummary/variables.rst +++ /dev/null @@ -1,42 +0,0 @@ -.. currentmodule:: {{ module }} - -################### -Variables reference -################### - -Autosubmit uses a variable substitution system to facilitate the development -of the templates. This variables can be used on the template in the form -``%VARIABLE_NAME%``. - -All configuration variables non related to current_job or platform are -accessible by calling first to their parents. ex: ``%PROJECT.PROJECT_TYPE%`` -or ``%DEFAULT.EXPID%``. - -You can review all variables at any given time by using the :ref:`report ` -command, as shown in the example below. - - -.. code-block:: console - - $ autosubmit report $expid -all - - -Job variables -============= - -{{ attributes }} - - -.. autoclass:: {{ objname }} - :noindex: - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - {% for item in attributes %} - ~{{ name }}.{{ item }} - {%- endfor %} - {% endif %} - {% endblock %} \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 44c17dc01..5c7b344ad 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -36,11 +36,8 @@ extensions = [ 'sphinx.ext.autosectionlabel', 'sphinx_rtd_theme', 'sphinx_reredirects', - 'sphinx.ext.autosummary', - 'sphinx.ext.napoleon', 'autosubmit_variables' ] -autosummary_generate = True autosectionlabel_prefix_document = True diff --git a/test/unit/helpers/test_parameters.py b/test/unit/helpers/test_parameters.py index a3f2db551..93b2c290a 100644 --- a/test/unit/helpers/test_parameters.py +++ b/test/unit/helpers/test_parameters.py @@ -1,6 +1,10 @@ from unittest import TestCase -from autosubmit.helpers.parameters import autosubmit_parameter, PARAMETERS +from autosubmit.helpers.parameters import ( + autosubmit_parameter, + autosubmit_parameters, + PARAMETERS +) class TestParameters(TestCase): @@ -30,6 +34,31 @@ class TestParameters(TestCase): self.assertEquals('This parameter is the job name.', PARAMETERS[parameter_group][parameter_name]) + def test_autosubmit_decorator_using_array(self): + """Test the ``autosubmit_decorator``.""" + + parameter_names = ['JOBNAME', 'JOB____NAME'] + parameter_group = 'PLATFORM' + + class Job: + @property + @autosubmit_parameter(name=parameter_names, group=parameter_group) + def name(self): + """This parameter is the job name.""" + return 'FOO' + + job = Job() + + self.assertEqual('FOO', job.name) + self.assertTrue(len(PARAMETERS) > 0) + # Defaults to the module name if not provided! So the class name + # ``Job`` becomes ``JOB``. + self.assertTrue(parameter_group in PARAMETERS) + for parameter_name in parameter_names: + self.assertTrue(parameter_name in PARAMETERS[parameter_group]) + self.assertEquals('This parameter is the job name.', PARAMETERS[parameter_group][parameter_name]) + + def test_autosubmit_decorator_no_group(self): """Test the ``autosubmit_decorator`` when ``group`` is not provided.""" @@ -51,3 +80,28 @@ class TestParameters(TestCase): self.assertTrue(Job.__name__.upper() in PARAMETERS) self.assertTrue(parameter_name in PARAMETERS[Job.__name__.upper()]) self.assertEquals('This parameter is the job name.', PARAMETERS[Job.__name__.upper()][parameter_name]) + + + def test_autosubmit_class_decorator(self): + """Test the ``autosubmit_decorator`` when ``group`` is not provided.""" + + @autosubmit_parameters(parameters={ + 'job': { + 'JOBNAME': 'The value!' + } + }) + class Job: + @property + def name(self): + """This parameter is the job name.""" + return 'FOO' + + job = Job() + + self.assertEqual('FOO', job.name) + self.assertTrue(len(PARAMETERS) > 0) + self.assertTrue('JOB' in PARAMETERS) + self.assertTrue('JOBNAME' in PARAMETERS['JOB']) + self.assertEquals('The value!', PARAMETERS['JOB']['JOBNAME']) + + diff --git a/test/unit/test_slurm_platform.py b/test/unit/test_slurm_platform.py index 589cebdeb..0bfe84955 100644 --- a/test/unit/test_slurm_platform.py +++ b/test/unit/test_slurm_platform.py @@ -31,6 +31,28 @@ class TestSlurmPlatform(TestCase): def tearDown(self) -> None: self.local_root_dir.cleanup() + def test_properties(self): + props = { + 'name': 'foo', + 'host': 'localhost1', + 'user': 'sam', + 'project': 'proj1', + 'budget': 100, + 'reservation': 1, + 'exclusivity': True, + 'hyperthreading': True, + 'type': 'SuperSlurm', + 'scratch': '/scratch/1', + 'project_dir': '/proj1', + 'root_dir': '/root_1', + 'partition': 'inter', + 'queue': 'prio1' + } + for prop, value in props.items(): + setattr(self.platform, prop, value) + for prop, value in props.items(): + self.assertEqual(value, getattr(self.platform, prop)) + def test_slurm_platform_submit_script_raises_autosubmit_critical_with_trace(self): package = MagicMock() package.jobs.return_value = [] -- GitLab From 76527bec1a7bef12d5febc46abe8397f53571b36 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 24 May 2023 14:27:47 +0200 Subject: [PATCH 5/8] Add unit test for the YAML comments --- autosubmit/autosubmit.py | 22 ++++++++++++----- test/unit/test_expid.py | 52 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 3654b89d3..1f128baf9 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1041,13 +1041,21 @@ class Autosubmit: except Exception as e: Log.warning("Error converting {0} to yml: {1}".format(conf_file.replace(copy_id,exp_id),str(e))) - def generate_as_config(exp_id: str, dummy: bool=False, minimal_configuration: bool=False, local: bool=False) -> None: + @staticmethod + def generate_as_config( + exp_id: str, + dummy: bool=False, + minimal_configuration: bool=False, + local: bool=False, + parameters: Dict[str, Union[Dict, List, str]] = None + ) -> None: """Retrieve the configuration from autosubmitconfigparser package. :param exp_id: Experiment ID :param dummy: Whether the experiment is a dummy one or not. :param minimal_configuration: Whether the experiment is configured with minimal configuration or not. :param local: Whether the experiment project type is local or not. + :param parameters: Optional list of parameters to be used when processing the configuration files. :return: None """ @@ -1095,22 +1103,24 @@ class Autosubmit: yield key, value template_files = resource_listdir('autosubmitconfigparser.config', 'files') - parameter_comments = dict(recurse_into_parameters(PARAMETERS)) + if parameters is None: + parameters = PARAMETERS + parameter_comments = dict(recurse_into_parameters(parameters)) for as_conf_file in template_files: - origin = resource_filename('autosubmitconfigparser.config', 'files/'+as_conf_file) + origin = resource_filename('autosubmitconfigparser.config', str(Path('files', as_conf_file))) target = None if dummy: if as_conf_file.endswith('dummy.yml'): file_name = f'{as_conf_file.split("-")[0]}_{exp_id}.yml' - target = Path(BasicConfig.LOCAL_ROOT_DIR / exp_id / 'conf' / file_name) + target = Path(BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf', file_name) elif minimal_configuration: if (not local and as_conf_file.endswith('git-minimal.yml')) or as_conf_file.endswith("local-minimal.yml"): - target = Path(BasicConfig.LOCAL_ROOT_DIR / exp_id / 'conf/minimal.yml') + target = Path(BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf/minimal.yml') else: if not as_conf_file.endswith('dummy.yml') and not as_conf_file.endswith('minimal.yml'): - file_name = f'{as_conf_file[:-4]}_{exp_id}.yml' + file_name = f'{Path(as_conf_file).stem}_{exp_id}.yml' target = Path(BasicConfig.LOCAL_ROOT_DIR, exp_id, 'conf', file_name) # Here we annotate the copied configuration with comments from the Python source code. diff --git a/test/unit/test_expid.py b/test/unit/test_expid.py index f818ad188..5828cf04c 100644 --- a/test/unit/test_expid.py +++ b/test/unit/test_expid.py @@ -1,6 +1,11 @@ +import tempfile from unittest import TestCase from mock import Mock, patch -from autosubmit.experiment.experiment_common import new_experiment, next_experiment_id +from autosubmit.autosubmit import Autosubmit +from autosubmit.experiment.experiment_common import new_experiment +from textwrap import dedent +from pathlib import Path +from autosubmitconfigparser.config.basicconfig import BasicConfig class TestExpid(TestCase): @@ -54,3 +59,48 @@ class TestExpid(TestCase): def _build_db_mock(current_experiment_id, mock_db_common): mock_db_common.last_name_used = Mock(return_value=current_experiment_id) mock_db_common.check_experiment_exists = Mock(return_value=False) + + @patch('autosubmit.autosubmit.resource_listdir') + @patch('autosubmit.autosubmit.resource_filename') + def test_autosubmit_generate_config(self, resource_filename_mock, resource_listdir_mock): + expid = 'ff99' + original_local_root_dir = BasicConfig.LOCAL_ROOT_DIR + + with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as source_yaml, tempfile.TemporaryDirectory() as temp_dir: + # Our processed and commented YAML output file must be written here + Path(temp_dir, expid, 'conf').mkdir(parents=True) + BasicConfig.LOCAL_ROOT_DIR = temp_dir + + source_yaml.write( +dedent('''JOB: + JOBNAME: SIM + PLATFORM: local +CONFIG: + TEST: The answer? + ROOT: No''')) + source_yaml.flush() + resource_listdir_mock.return_value = [Path(source_yaml.name).name] + resource_filename_mock.return_value = source_yaml.name + + parameters = { + 'JOB': { + 'JOBNAME': 'sim' + }, + 'CONFIG': { + 'CONFIG.TEST': '42' + } + } + Autosubmit.generate_as_config(exp_id=expid, parameters=parameters) + + source_text = Path(source_yaml.name).read_text() + source_name = Path(source_yaml.name) + output_text = Path(temp_dir, expid, 'conf', f'{source_name.stem}_{expid}.yml').read_text() + + self.assertNotEquals(source_text, output_text) + self.assertFalse('# sim' in source_text) + self.assertTrue('# sim' in output_text) + self.assertFalse('# 42' in source_text) + self.assertTrue('# 42' in output_text) + + # Reset the local root dir. + BasicConfig.LOCAL_ROOT_DIR = original_local_root_dir -- GitLab From 1225f3a268461cbe215aa96ab70c14c71c9035c4 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Wed, 24 May 2023 15:46:06 +0200 Subject: [PATCH 6/8] Change RTD configuration to remove the ruamel.yaml CLIB package --- .readthedocs.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 6afe3912a..001a2fe97 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,9 +7,17 @@ version: 2 # Set the version of Python and other tools you might need build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: python: "3.9" + jobs: + post_install: + # ruamel.yaml.clib appears to cause the build to fail when + # using new API in ruamel.yaml + # refs: + # - https://earth.bsc.es/gitlab/es/autosubmit/-/merge_requests/340/ + # - https://yaml.readthedocs.io/en/latest/api.html + - pip uninstall ruamel.yaml.clib -y # Build documentation in the docs/ directory with Sphinx sphinx: @@ -21,6 +29,6 @@ formats: all # Optionally declare the Python requirements required to build your docs python: install: - - method: setuptools + - method: pip path: . - requirements: docs/requirements.txt -- GitLab From a31f47b78bf600dd91f769b705c51ee86ee1fabd Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Thu, 25 May 2023 13:48:19 +0200 Subject: [PATCH 7/8] Delete properties after they have been used (to save memory) --- autosubmit/helpers/parameters.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/autosubmit/helpers/parameters.py b/autosubmit/helpers/parameters.py index e6001763e..7e9ca011a 100644 --- a/autosubmit/helpers/parameters.py +++ b/autosubmit/helpers/parameters.py @@ -58,7 +58,7 @@ def autosubmit_parameter(func=None, *, name, group=None): Used to annotate properties of classes Attributes: - func (Callable): wrapped function. + func (Callable): wrapped function. Always ``None`` due to how we call the decorator. name (Union[str, List[str]]): parameter name. group (str): group name. Default to caller module name. """ @@ -79,20 +79,24 @@ def autosubmit_parameter(func=None, *, name, group=None): if parameter_name not in PARAMETERS[group]: PARAMETERS[group][parameter_name] = None - def parameter_decorator(func): - group = parameter_decorator.group - names = parameter_decorator.names - for name in names: - if func.__doc__: - PARAMETERS[group][name] = func.__doc__.strip().split('\n')[0] + def parameter_decorator(wrapped_func): + parameter_group = parameter_decorator.__group + parameter_names = parameter_decorator.__names + for p_name in parameter_names: + if wrapped_func.__doc__: + PARAMETERS[parameter_group][p_name] = wrapped_func.__doc__.strip().split('\n')[0] - @functools.wraps(func) + # Delete the members created as we are not using them hereafter + del parameter_decorator.__group + del parameter_decorator.__names + + @functools.wraps(wrapped_func) def wrapper(*args, **kwargs): - return func(*args, **kwargs) + return wrapped_func(*args, **kwargs) return wrapper - parameter_decorator.group = group - parameter_decorator.names = names + parameter_decorator.__group = group + parameter_decorator.__names = names return parameter_decorator -- GitLab From 8c7b2e29ece2b04c1067107fdce73234e04be756 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 29 May 2023 09:09:20 +0200 Subject: [PATCH 8/8] Add unit test for job sdate, fix typo (thanks @dbeltran) --- autosubmit/helpers/parameters.py | 2 +- test/unit/test_job.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/autosubmit/helpers/parameters.py b/autosubmit/helpers/parameters.py index 7e9ca011a..97ddb6235 100644 --- a/autosubmit/helpers/parameters.py +++ b/autosubmit/helpers/parameters.py @@ -5,7 +5,7 @@ from typing import Dict PARAMETERS = defaultdict(defaultdict) """Global default dictionary holding a multi-level dictionary with the Autosubmit -parameters. At the first level we have the paramete groups. +parameters. At the first level we have the parameter groups. - ``JOB`` diff --git a/test/unit/test_job.py b/test/unit/test_job.py index b3eb35532..4ee2903d7 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -7,6 +7,7 @@ from autosubmit.job.job import Job from autosubmit.platforms.platform import Platform from mock import Mock, MagicMock from mock import patch +import datetime # compatibility with both versions (2 & 3) from sys import version_info @@ -312,6 +313,19 @@ class TestJob(TestCase): self.assertEqual('%Y%', parameters['Y']) self.assertEqual('%Y_%', parameters['Y_']) + def test_sdate(self): + """Test that the property getter for ``sdate`` works as expected.""" + for test in [ + [None, None, ''], + [datetime.datetime(1975, 5, 25, 22, 0, 0, 0, datetime.timezone.utc), 'H', '1975052522'], + [datetime.datetime(1975, 5, 25, 22, 30, 0, 0, datetime.timezone.utc), 'M', '197505252230'], + [datetime.datetime(1975, 5, 25, 22, 30, 0, 0, datetime.timezone.utc), 'S', '19750525223000'], + [datetime.datetime(1975, 5, 25, 22, 30, 0, 0, datetime.timezone.utc), None, '19750525'] + ]: + self.job.date = test[0] + self.job.date_format = test[1] + self.assertEquals(test[2], self.job.sdate) + import inspect class FakeBasicConfig: def __init__(self): -- GitLab