diff --git a/autosubmit_api/autosubmit_legacy/job/job_dict.py b/autosubmit_api/autosubmit_legacy/job/job_dict.py index 0f833cc31ed384196246fc53b9290ed9be3c5dbd..f67f7748b6c9d132ef4fc26842a50ba7fd617d1a 100644 --- a/autosubmit_api/autosubmit_legacy/job/job_dict.py +++ b/autosubmit_api/autosubmit_legacy/job/job_dict.py @@ -371,6 +371,9 @@ class DicJobs: :param default: value to return if not defined in configuration file :type default: object """ + #todo: here should be implemented some of the changes for supporting yaml files + + if self._parser.has_option(section, option): return self._parser.get(section, option) else: diff --git a/autosubmit_api/common/utils.py b/autosubmit_api/common/utils.py index 611a1fa726f252c1c5d219c58116f93ce94df58e..2dd157078984bd9fcb1cd5a99c704d428aeb6eed 100644 --- a/autosubmit_api/common/utils.py +++ b/autosubmit_api/common/utils.py @@ -201,13 +201,14 @@ class Status: HELD = 6 PREPARED = 7 SKIPPED = 8 + DELAYED = 9 FAILED = -1 UNKNOWN = -2 SUSPENDED = -3 ####### # Note: any change on constants must be applied on the dict below!!! VALUE_TO_KEY = {-3: 'SUSPENDED', -2: 'UNKNOWN', -1: 'FAILED', 0: 'WAITING', 1: 'READY', - 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED'} + 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED', 9: 'DELAYED'} STRING_TO_CODE = {v: k for k, v in list(VALUE_TO_KEY.items())} def retval(self, value): diff --git a/autosubmit_api/components/experiment/configuration_facade.py b/autosubmit_api/components/experiment/configuration_facade.py index e5651e885f762823ea6f3eaa5d0d462bc66e2897..e349861b951145e6adaf007890af7e265ce6467c 100644 --- a/autosubmit_api/components/experiment/configuration_facade.py +++ b/autosubmit_api/components/experiment/configuration_facade.py @@ -190,7 +190,7 @@ class AutosubmitConfigurationFacade(ConfigurationFacade): def get_wrapper_type(self): # type: () -> str | None - if self.autosubmit_conf.get_wrapper_type().upper() != "NONE": + if self.autosubmit_conf.get_wrapper_type() and self.autosubmit_conf.get_wrapper_type().upper() != "NONE": return self.autosubmit_conf.get_wrapper_type().upper() return None diff --git a/autosubmit_api/components/jobs/joblist_loader.py b/autosubmit_api/components/jobs/joblist_loader.py index 50a94d8922869860d561471991f23301740be44b..88abb2a161d05e1c941ae75902cbc5014d4e9648 100644 --- a/autosubmit_api/components/jobs/joblist_loader.py +++ b/autosubmit_api/components/jobs/joblist_loader.py @@ -14,6 +14,10 @@ from typing import Dict, List, Set, Tuple from ...config.config_common import AutosubmitConfig from ...config.basicConfig import BasicConfig import json +import logging + + +logger = logging.getLogger('gunicorn.error') class JobListLoader(object): """ Class that manages loading the list of jobs from the pkl. Adds other resources. """ @@ -187,14 +191,16 @@ class JobListLoader(object): else: job_qos = self.configuration_facade.get_section_qos(job.section) if len(job_qos.strip()) == 0: - job_qos = self.configuration_facade.get_platform_qos(job.platform, job.ncpus) + if job.platform != "None": + job_qos = self.configuration_facade.get_platform_qos(job.platform, job.ncpus) return job_qos def _determine_wallclock(self, job): # type: (Job) -> None wallclock = self.configuration_facade.get_section_wallclock(job.section) if len(wallclock.strip()) == 0: - wallclock = self.configuration_facade.get_platform_max_wallclock(job.platform) + if job.platform != "None": + wallclock = self.configuration_facade.get_platform_max_wallclock(job.platform) return wallclock def assign_packages_to_jobs(self): diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py new file mode 100644 index 0000000000000000000000000000000000000000..0fbc61fa254033cd814e14dcfe8d0e8d8d0045db --- /dev/null +++ b/autosubmit_api/config/IConfigStrategy.py @@ -0,0 +1,817 @@ +#!/usr/bin/env python + +# Copyright 2015 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . + +try: + # noinspection PyCompatibility + from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config +except ImportError: + # noinspection PyCompatibility + from configparser import SafeConfigParser + + +import os +import re +import subprocess +import json +import logging +import locale + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.basicConfig import BasicConfig +from abc import ABC, abstractmethod + + +class IConfigStrategy(ABC): + """ + Public interface for Autosubmit config files + + :param expid: experiment identifier + :type expid: str + """ + + @abstractmethod + def jobs_parser(self): + raise NotImplementedError + + @abstractmethod + def experiment_file(self): + """ + Returns experiment's config file name + """ + return self._exp_parser_file + + @abstractmethod + def platforms_parser(self): + """ + Returns experiment's platforms parser object + + :return: platforms config parser object + :rtype: SafeConfigParser + """ + pass + + @abstractmethod + def platforms_file(self): + """ + Returns experiment's platforms config file name + + :return: platforms config file's name + :rtype: str + """ + pass + + @abstractmethod + def project_file(self): + """ + Returns project's config file name + """ + pass + + @abstractmethod + def jobs_file(self): + """ + Returns project's jobs file name + """ + pass + + @abstractmethod + def get_full_config_as_dict(self): + """ + Returns full configuration as json object + """ + pass + + @abstractmethod + def get_project_dir(self): + """ + Returns experiment's project directory + + :return: experiment's project directory + :rtype: str + """ + pass + + @abstractmethod + def get_queue(self, section): + """ + Get queue for the given job type + :param section: job type + :type section: str + :return: queue + :rtype: str + """ + pass + + @abstractmethod + def get_job_platform(self, section): + pass + + @abstractmethod + def get_platform_queue(self, platform): + pass + + @abstractmethod + def get_platform_serial_queue(self, platform): + pass + + @abstractmethod + def get_platform_project(self, platform): + pass + + @abstractmethod + def get_platform_wallclock(self, platform): + pass + + def get_wallclock(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + pass + + def get_synchronize(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + pass + + def get_processors(self, section): + """ + Gets processors needed for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + pass + + def get_threads(self, section): + """ + Gets threads needed for the given job type + :param section: job type + :type section: str + :return: threads needed + :rtype: str + """ + pass + + def get_tasks(self, section): + """ + Gets tasks needed for the given job type + :param section: job type + :type section: str + :return: tasks (processes) per host + :rtype: str + """ + pass + + def get_scratch_free_space(self, section): + """ + Gets scratch free space needed for the given job type + :param section: job type + :type section: str + :return: percentage of scratch free space needed + :rtype: int + """ + pass + + def get_memory(self, section): + """ + Gets memory needed for the given job type + :param section: job type + :type section: str + :return: memory needed + :rtype: str + """ + pass + + def get_memory_per_task(self, section): + """ + Gets memory per task needed for the given job type + :param section: job type + :type section: str + :return: memory per task needed + :rtype: str + """ + pass + + def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + + :return: migrate user to + :rtype: str + """ + #return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + pass + + def get_current_user(self, section): + """ + Returns the user to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + pass + + def get_current_project(self, section): + """ + Returns the project to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + return str(self._platforms_data[section]["USER"]).lower() + + def set_new_user(self, section, new_user): + """ + Sets new user for given platform + :param new_user: + :param section: platform name + :type: str + """ + pass + + def get_migrate_project_to(self, section): + """ + Returns the project to change to from platform config file. + + :return: migrate project to + :rtype: str + """ + pass + + + + def set_new_project(self, section, new_project): + """ + Sets new project for given platform + :param new_project: + :param section: platform name + :type: str + """ + pass + + + def get_custom_directives(self, section): + """ + Gets custom directives needed for the given job type + :param section: job type + :type section: str + :return: custom directives needed + :rtype: str + """ + pass + + + def check_conf_files(self): + """ + Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing + required options. Prints results in log + + :return: True if everything is correct, False if it finds any error + :rtype: bool + """ + pass + + def check_autosubmit_conf(self): + """ + Checks experiment's autosubmit configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + pass + + def check_platforms_conf(self): + """ + Checks experiment's queues configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + + + def check_jobs_conf(self): + """ + Checks experiment's jobs configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + pass + + + def check_expdef_conf(self): + """ + Checks experiment's experiment configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + pass + + + def check_proj(self): + """ + Checks project config file + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + pass + + + def check_wrapper_conf(self): + pass + + + def reload(self): + """ + Creates parser objects for configuration files + """ + pass + + def load_parameters(self): + """ + Load parameters from experiment and autosubmit config files. If experiment's type is not none, + also load parameters from model's config file + + :return: a dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + + + def load_project_parameters(self): + """ + Loads parameters from model config file + + :return: dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + pass + + + def set_expid(self, exp_id): + """ + Set experiment identifier in autosubmit and experiment config files + + :param exp_id: experiment identifier to store + :type exp_id: str + """ + + + def get_project_type(self): + """ + Returns project type from experiment config file + + :return: project type + :rtype: str + """ + pass + + def get_file_project_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + pass + + def get_file_jobs_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + + + def get_git_project_origin(self): + """ + Returns git origin from experiment config file + + :return: git origin + :rtype: str + """ + + def get_git_project_branch(self): + """ + Returns git branch from experiment's config file + + :return: git branch + :rtype: str + """ + + def get_git_project_commit(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + + def get_submodules_list(self): + """ + Returns submodules list from experiment's config file + Default is --recursive + :return: submodules to load + :rtype: list + """ + + def get_project_destination(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + + + def set_git_project_commit(self, as_conf): + """ + Function to register in the configuration the commit SHA of the git project version. + :param as_conf: Configuration class for exteriment + :type as_conf: AutosubmitConfig + """ + + def get_svn_project_url(self): + """ + Gets subversion project url + + :return: subversion project url + :rtype: str + """ + + def get_svn_project_revision(self): + """ + Get revision for subversion project + + :return: revision for subversion project + :rtype: str + """ + + def get_local_project_path(self): + """ + Gets path to origin for local project + + :return: path to local project + :rtype: str + """ + + def get_date_list(self): + """ + Returns startdates list from experiment's config file + + :return: experiment's startdates + :rtype: list + """ + + + def get_num_chunks(self): + """ + Returns number of chunks to run for each member + + :return: number of chunks + :rtype: int + """ + + def get_chunk_ini(self, default=1): + """ + Returns the first chunk from where the experiment will start + + :param default: + :return: initial chunk + :rtype: int + """ + + + def get_chunk_size_unit(self): + # type: () -> str + """ + Unit for the chunk length + + :return: Unit for the chunk length Options: {hour, day, month, year} + :rtype: str + """ + + pass + + def get_chunk_size(self, default=1): + # type: (int) -> int + """ + Chunk Size as defined in the expdef file. + + :return: Chunksize, 1 as default. + :rtype: int + """ + pass + + def get_member_list(self, run_only=False): + """ + Returns members list from experiment's config file + + :return: experiment's members + :rtype: list + """ + pass + + def get_rerun(self): + """ + Returns startdates list from experiment's config file + + :return: rerurn value + :rtype: list + """ + + pass + + def get_chunk_list(self): + """ + Returns chunk list from experiment's config file + + :return: experiment's chunks + :rtype: list + """ + pass + + def get_platform(self): + """ + Returns main platforms from experiment's config file + + :return: main platforms + :rtype: str + """ + pass + + def set_platform(self, hpc): + """ + Sets main platforms in experiment's config file + + :param hpc: main platforms + :type: str + """ + pass + + def set_version(self, autosubmit_version): + """ + Sets autosubmit's version in autosubmit's config file + + :param autosubmit_version: autosubmit's version + :type autosubmit_version: str + """ + pass + + def get_version(self): + """ + Returns version number of the current experiment from autosubmit's config file + + :return: version + :rtype: str + """ + pass + + def get_total_jobs(self): + """ + Returns max number of running jobs from autosubmit's config file + + :return: max number of running jobs + :rtype: int + """ + pass + + def get_max_wallclock(self): + """ + Returns max wallclock + + :rtype: str + """ + pass + + def get_max_processors(self): + """ + Returns max processors from autosubmit's config file + + :rtype: str + """ + pass + + def get_max_waiting_jobs(self): + """ + Returns max number of waiting jobs from autosubmit's config file + + :return: main platforms + :rtype: int + """ + pass + + def get_default_job_type(self): + """ + Returns the default job type from experiment's config file + + :return: default type such as bash, python, r.. + :rtype: str + """ + pass + + def get_safetysleeptime(self): + """ + Returns safety sleep time from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + pass + + def set_safetysleeptime(self, sleep_time): + """ + Sets autosubmit's version in autosubmit's config file + + :param sleep_time: value to set + :type sleep_time: int + """ + pass + + def get_retrials(self): + """ + Returns max number of retrials for job from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + pass + + def get_notifications(self): + """ + Returns if the user has enabled the notifications from autosubmit's config file + + :return: if notifications + :rtype: string + """ + pass + + def get_remote_dependencies(self): + """ + Returns if the user has enabled the remote dependencies from autosubmit's config file + + :return: if remote dependencies + :rtype: bool + """ + pass + + def get_wrapper_type(self): + """ + Returns what kind of wrapper (VERTICAL, MIXED-VERTICAL, HORIZONTAL, HYBRID, NONE) the user has configured in the autosubmit's config + + :return: wrapper type (or none) + :rtype: string + """ + pass + + def get_wrapper_jobs(self): + """ + Returns the jobs that should be wrapped, configured in the autosubmit's config + + :return: expression (or none) + :rtype: string + """ + pass + + def get_max_wrapped_jobs(self): + """ + Returns the maximum number of jobs that can be wrapped together as configured in autosubmit's config file + + :return: maximum number of jobs (or total jobs) + :rtype: string + """ + # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs())) + + pass + + def get_wrapper_check_time(self): + """ + Returns time to check the status of jobs in the wrapper + + :return: wrapper check time + :rtype: int + """ + pass + + def get_wrapper_machinefiles(self): + """ + Returns the strategy for creating the machinefiles in wrapper jobs + + :return: machinefiles function to use + :rtype: string + """ + pass + + def get_wrapper_queue(self): + """ + Returns the wrapper queue if not defined, will be the one of the first job wrapped + + :return: expression (or none) + :rtype: string + """ + pass + + def get_jobs_sections(self): + """ + Returns the list of sections defined in the jobs config file + + :return: sections + :rtype: list + """ + return self._jobs_parser.sections() + + def get_copy_remote_logs(self): + """ + Returns if the user has enabled the logs local copy from autosubmit's config file + + :return: if logs local copy + :rtype: bool + """ + pass + + def get_mails_to(self): + """ + Returns the address where notifications will be sent from autosubmit's config file + + :return: mail address + :rtype: [str] + """ + pass + + def get_communications_library(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + pass + + def get_storage_type(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + pass + + @staticmethod + def is_valid_mail_address(mail_address): + if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): + return True + else: + return False + @classmethod + def is_valid_communications_library(self): + library = self.get_communications_library() + return library in ['paramiko', 'saga'] + + @classmethod + def is_valid_storage_type(self): + storage_type = self.get_storage_type() + return storage_type in ['pkl', 'db'] + + + def is_valid_jobs_in_wrapper(self): + pass + + def is_valid_git_repository(self): + pass + + @staticmethod + def get_parser(parser_factory, file_path): + # type: (ConfigParserFactory, str) -> ConfigParser + pass \ No newline at end of file diff --git a/autosubmit_api/config/basicConfig.py b/autosubmit_api/config/basicConfig.py index 3c43686608c026b58e35854050519e0928404e3a..38a25576784770a95009fa5e9ec6696ec339bba2 100644 --- a/autosubmit_api/config/basicConfig.py +++ b/autosubmit_api/config/basicConfig.py @@ -19,6 +19,7 @@ try: # noinspection PyCompatibility from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py new file mode 100644 index 0000000000000000000000000000000000000000..fbeba43851bceb4fe026c099338f02d88d50f483 --- /dev/null +++ b/autosubmit_api/config/confConfigStrategy.py @@ -0,0 +1,1344 @@ + +# !/usr/bin/env python + +# Copyright 2015 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . +try: + # noinspection PyCompatibility + from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config +except ImportError: + # noinspection PyCompatibility + from configparser import SafeConfigParser + + +import os +import re +import subprocess +import json +import logging + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.basicConfig import BasicConfig +from ..config.IConfigStrategy import IConfigStrategy + +logger = logging.getLogger('gunicorn.error') + +class confConfigStrategy(IConfigStrategy): + """ + Class to handle experiment configuration coming from file or database + + :param expid: experiment identifier + :type expid: str + """ + + def __init__(self, expid, basic_config, parser_factory, extension=".conf"): + # type: (str, BasicConfig, ConfigParserFactory, Extension) -> None + + self.expid = expid + self.basic_config = basic_config + self.parser_factory = parser_factory + + # By default check for .yml files first as it is the new standard for AS 4.0 + self._conf_parser = None # type: ConfigParser + self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "autosubmit_" + expid + extension) + if os.path.exists(self._conf_parser_file) == False: + return None + + self._exp_parser = None # type: ConfigParser + self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "expdef_" + expid + extension) + if os.path.exists(self._exp_parser_file) == False: + return None + + self._platforms_parser = None # type: ConfigParser + self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "platforms_" + expid + extension) + if os.path.exists(self._platforms_parser_file) == False: + return None + + self._jobs_parser = None # type: ConfigParser + self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "jobs_" + expid + extension) + if os.path.exists(self._jobs_parser_file) == False: + return None + + self._proj_parser = None # type: ConfigParser + self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "proj_" + expid + extension) + if os.path.exists(self._proj_parser_file) == False: + return None + + + @property + def jobs_parser(self): + return self._jobs_parser + + @property + def jobs_parser(self): + return self._jobs_parser + + @property + def experiment_file(self): + """ + Returns experiment's config file name + """ + return self._exp_parser_file + + @property + def platforms_parser(self): + """ + Returns experiment's platforms parser object + + :return: platforms config parser object + :rtype: SafeConfigParser + """ + return self._platforms_parser + + @property + def platforms_file(self): + """ + Returns experiment's platforms config file name + + :return: platforms config file's name + :rtype: str + """ + return self._platforms_parser_file + + @property + def project_file(self): + """ + Returns project's config file name + """ + return self._proj_parser_file + + @property + def jobs_file(self): + """ + Returns project's jobs file name + """ + return self._jobs_parser_file + + def get_full_config_as_dict(self): + """ + Returns full configuration as json object + """ + _conf = _exp = _platforms = _jobs = _proj = None + result = {} + + def get_data(parser): + """ + dictionary comprehension to get data from parser + """ + res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ + section for section in parser.sections()]} + return res + + # print(self._conf_parser) + result["conf"] = get_data( + self._conf_parser) if self._conf_parser else None + result["exp"] = get_data( + self._exp_parser) if self._exp_parser else None + result["platforms"] = get_data( + self._platforms_parser) if self._platforms_parser else None + result["jobs"] = get_data( + self._jobs_parser) if self._jobs_parser else None + result["proj"] = get_data( + self._proj_parser) if self._proj_parser else None + return result + + def get_full_config_as_json(self): + """ + Return config as json object + """ + try: + return json.dumps(self.get_full_config_as_dict()) + except Exception as exp: + Log.warning( + "Autosubmit was not able to retrieve and save the configuration into the historical database.") + return "" + + def get_project_dir(self): + """ + Returns experiment's project directory + + :return: experiment's project directory + :rtype: str + """ + dir_templates = os.path.join(self.basic_config.LOCAL_ROOT_DIR, self.expid, BasicConfig.LOCAL_PROJ_DIR, + self.get_project_destination()) + return dir_templates + + def get_queue(self, section): + """ + Get queue for the given job type + :param section: job type + :type section: str + :return: queue + :rtype: str + """ + return self._jobs_parser.get_option(section, 'QUEUE', '') + + def get_job_platform(self, section): + return self._jobs_parser.get_option(section, 'PLATFORM', '') + + def get_platform_queue(self, platform): + return self._platforms_parser.get_option(platform, 'QUEUE', '') + + def get_platform_serial_queue(self, platform): + return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '') + + def get_platform_project(self, platform): + return self._platforms_parser.get_option(platform, "PROJECT", "") + + def get_platform_wallclock(self, platform): + return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '') + + def get_wallclock(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + return self._jobs_parser.get_option(section, 'WALLCLOCK', '') + + def get_synchronize(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + return self._jobs_parser.get_option(section, 'SYNCHRONIZE', '') + + def get_processors(self, section): + """ + Gets processors needed for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'PROCESSORS', 1)) + + def get_threads(self, section): + """ + Gets threads needed for the given job type + :param section: job type + :type section: str + :return: threads needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'THREADS', 1)) + + def get_tasks(self, section): + """ + Gets tasks needed for the given job type + :param section: job type + :type section: str + :return: tasks (processes) per host + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'TASKS', 0)) + + def get_scratch_free_space(self, section): + """ + Gets scratch free space needed for the given job type + :param section: job type + :type section: str + :return: percentage of scratch free space needed + :rtype: int + """ + return int(self._jobs_parser.get_option(section, 'SCRATCH_FREE_SPACE', 0)) + + def get_memory(self, section): + """ + Gets memory needed for the given job type + :param section: job type + :type section: str + :return: memory needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'MEMORY', '')) + + def get_memory_per_task(self, section): + """ + Gets memory per task needed for the given job type + :param section: job type + :type section: str + :return: memory per task needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', '')) + + def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + + def get_current_user(self, section): + """ + Returns the user to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'USER', '').lower() + + def get_current_project(self, section): + """ + Returns the project to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'PROJECT', '').lower() + + def set_new_user(self, section, new_user): + """ + Sets new user for given platform + :param new_user: + :param section: platform name + :type: str + """ + with open(self._platforms_parser_file) as p_file: + contentLine = p_file.readline() + contentToMod = "" + content = "" + mod = False + while contentLine: + if re.search(section, contentLine): + mod = True + if mod: + contentToMod += contentLine + else: + content += contentLine + contentLine = p_file.readline() + if mod: + old_user = self.get_current_user(section) + contentToMod = contentToMod.replace(re.search( + r'[^#]\bUSER\b =.*', contentToMod).group(0)[1:], "USER = " + new_user) + contentToMod = contentToMod.replace(re.search( + r'[^#]\bUSER_TO\b =.*', contentToMod).group(0)[1:], "USER_TO = " + old_user) + open(self._platforms_parser_file, 'w').write(content) + open(self._platforms_parser_file, 'a').write(contentToMod) + + def get_migrate_project_to(self, section): + """ + Returns the project to change to from platform config file. + + :return: migrate project to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() + + def set_new_project(self, section, new_project): + """ + Sets new project for given platform + :param new_project: + :param section: platform name + :type: str + """ + with open(self._platforms_parser_file) as p_file: + contentLine = p_file.readline() + contentToMod = "" + content = "" + mod = False + while contentLine: + if re.search(section, contentLine): + mod = True + if mod: + contentToMod += contentLine + else: + content += contentLine + contentLine = p_file.readline() + if mod: + old_project = self.get_current_project(section) + contentToMod = contentToMod.replace(re.search( + r"[^#]\bPROJECT\b =.*", contentToMod).group(0)[1:], "PROJECT = " + new_project) + contentToMod = contentToMod.replace(re.search( + r"[^#]\bPROJECT_TO\b =.*", contentToMod).group(0)[1:], "PROJECT_TO = " + old_project) + open(self._platforms_parser_file, 'w').write(content) + open(self._platforms_parser_file, 'a').write(contentToMod) + + def get_custom_directives(self, section): + """ + Gets custom directives needed for the given job type + :param section: job type + :type section: str + :return: custom directives needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'CUSTOM_DIRECTIVES', '')) + + def check_conf_files(self): + """ + Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing + required options. Prints results in log + + :return: True if everything is correct, False if it finds any error + :rtype: bool + """ + Log.debug('\nChecking configuration files...') + self.reload() + # result = self.check_platforms_conf() + result = True + result = result and self.check_jobs_conf() + result = result and self.check_autosubmit_conf() + result = result and self.check_expdef_conf() + if result: + Log.debug("Configuration files OK\n") + else: + Log.error("Configuration files invalid\n") + return result + + def check_autosubmit_conf(self): + """ + Checks experiment's autosubmit configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + + self._conf_parser.read(self._conf_parser_file) + # result = result and self._conf_parser.check_exists( + # 'config', 'AUTOSUBMIT_VERSION') + # result = result and self._conf_parser.check_is_int( + # 'config', 'MAXWAITINGJOBS', True) + # result = result and self._conf_parser.check_is_int( + # 'config', 'TOTALJOBS', True) + # result = result and self._conf_parser.check_is_int( + # 'config', 'SAFETYSLEEPTIME', True) + # result = result and self._conf_parser.check_is_int( + # 'config', 'RETRIALS', True) + # result = result and self._conf_parser.check_is_boolean( + # 'mail', 'NOTIFICATIONS', False) + result = result and self.is_valid_communications_library() + result = result and self.is_valid_storage_type() + if self.get_wrapper_type() != 'None': + result = result and self.check_wrapper_conf() + + if self.get_notifications() == 'true': + for mail in self.get_mails_to(): + if not self.is_valid_mail_address(mail): + # Log.warning( + # 'One or more of the email addresses configured for the mail notifications are wrong') + break + + if not result: + # Log.critical("{0} is not a valid config file".format( + # os.path.basename(self._conf_parser_file))) + raise Exception("Permission denied for " + + str(os.path.basename(self._conf_parser_file))) + else: + Log.debug('{0} OK'.format( + os.path.basename(self._conf_parser_file))) + return result + + def check_platforms_conf(self): + """ + Checks experiment's queues configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + if len(self._platforms_parser.sections()) == 0: + Log.warning("No remote platforms configured") + + if len(self._platforms_parser.sections()) != len(set(self._platforms_parser.sections())): + Log.error('There are repeated platforms names') + + for section in self._platforms_parser.sections(): + result = result and self._platforms_parser.check_exists( + section, 'TYPE') + platform_type = self._platforms_parser.get_option( + section, 'TYPE', '').lower() + if platform_type != 'ps': + result = result and self._platforms_parser.check_exists( + section, 'PROJECT') + result = result and self._platforms_parser.check_exists( + section, 'USER') + + result = result and self._platforms_parser.check_exists( + section, 'HOST') + result = result and self._platforms_parser.check_exists( + section, 'SCRATCH_DIR') + result = result and self._platforms_parser.check_is_boolean(section, + 'ADD_PROJECT_TO_HOST', False) + result = result and self._platforms_parser.check_is_boolean( + section, 'TEST_SUITE', False) + result = result and self._platforms_parser.check_is_int(section, 'MAX_WAITING_JOBS', + False) + result = result and self._platforms_parser.check_is_int( + section, 'TOTAL_JOBS', False) + + if not result: + Log.critical("{0} is not a valid config file".format( + os.path.basename(self._platforms_parser_file))) + raise Exception("Permission denied for " + + str(os.path.basename(self._platforms_parser_file))) + else: + Log.info('{0} OK'.format( + os.path.basename(self._platforms_parser_file))) + return result + + def check_jobs_conf(self): + """ + Checks experiment's jobs configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + parser = self._jobs_parser + sections = parser.sections() + platforms = self._platforms_parser.sections() + platforms.append('LOCAL') + possible_exception = "" + if len(sections) == 0: + possible_exception += "No remote platforms configured\n" + + # if len(sections) != len(set(sections)): + # Log.error('There are repeated job names') + + for section in sections: + + result = result and parser.check_exists(section, 'FILE') + result = result and parser.check_is_boolean( + section, 'RERUN_ONLY', False) + + if parser.has_option(section, 'PLATFORM'): + # result = result and parser.check_is_choice( + # section, 'PLATFORM', False, platforms) + pass + if parser.has_option(section, 'DEPENDENCIES'): + for dependency in str(parser.get_option(section, 'DEPENDENCIES', '')).split(' '): + if '-' in dependency: + dependency = dependency.split('-')[0] + elif '+' in dependency: + dependency = dependency.split('+')[0] + if '[' in dependency: + dependency = dependency[:dependency.find('[')] + # if dependency not in sections: + # Log.error( + # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, + # dependency)) + + if parser.has_option(section, 'RERUN_DEPENDENCIES'): + for dependency in str(parser.get_option(section, 'RERUN_DEPENDENCIES', + '')).split(' '): + if '-' in dependency: + dependency = dependency.split('-')[0] + if '[' in dependency: + dependency = dependency[:dependency.find('[')] + # if dependency not in sections: + # Log.error( + # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, + # dependency)) + result = result and parser.check_is_choice(section, 'RUNNING', False, + ['once', 'date', 'member', 'chunk']) + + if not result: + # Log.critical("{0} is not a valid config file".format( + # os.path.basename(self._jobs_parser_file))) + raise Exception("Exception while checking jobs_expid.conf " + + str(os.path.basename(self._jobs_parser_file)) + possible_exception) + else: + Log.debug('{0} OK'.format( + os.path.basename(self._jobs_parser_file))) + + return result + + def check_expdef_conf(self): + """ + Checks experiment's experiment configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + parser = self._exp_parser + + # result = result and parser.check_exists('DEFAULT', 'EXPID') + # result = result and parser.check_exists('DEFAULT', 'HPCARCH') + + result = result and parser.check_exists('experiment', 'DATELIST') + result = result and parser.check_exists('experiment', 'MEMBERS') + result = result and parser.check_is_choice('experiment', 'CHUNKSIZEUNIT', True, + ['year', 'month', 'day', 'hour']) + result = result and parser.check_is_int( + 'experiment', 'CHUNKSIZE', True) + result = result and parser.check_is_int( + 'experiment', 'NUMCHUNKS', True) + result = result and parser.check_is_choice('experiment', 'CALENDAR', True, + ['standard', 'noleap']) + + result = result and parser.check_is_boolean('rerun', 'RERUN', True) + + if parser.check_is_choice('project', 'PROJECT_TYPE', True, + ['none', 'git', 'svn', 'local']): + project_type = parser.get_option('project', 'PROJECT_TYPE', '') + + if project_type == 'git': + result = result and parser.check_exists( + 'git', 'PROJECT_ORIGIN') + result = result and parser.check_exists( + 'git', 'PROJECT_BRANCH') + + elif project_type == 'svn': + result = result and parser.check_exists('svn', 'PROJECT_URL') + result = result and parser.check_exists( + 'svn', 'PROJECT_REVISION') + elif project_type == 'local': + result = result and parser.check_exists( + 'local', 'PROJECT_PATH') + + if project_type != 'none': + result = result and parser.check_exists( + 'project_files', 'FILE_PROJECT_CONF') + else: + result = True + + if not result: + # Log.critical("{0} is not a valid config file".format( + # os.path.basename(self._exp_parser_file))) + raise Exception("Permission denied for " + + str(os.path.basename(self._exp_parser_file))) + else: + Log.debug('{0} OK'.format( + os.path.basename(self._exp_parser_file))) + return result + + def check_proj(self): + """ + Checks project config file + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + try: + if self._proj_parser_file == '': + self._proj_parser = None + else: + self._proj_parser = AutosubmitConfig.get_parser( + self.parser_factory, self._proj_parser_file) + return True + except Exception as e: + Log.error('Project conf file error: {0}', e) + return False + + def check_wrapper_conf(self): + result = True + result = result and self.is_valid_jobs_in_wrapper() + if not result: + Log.error( + "There are sections in JOBS_IN_WRAPPER that are not defined in your jobs.conf file") + + if 'horizontal' in self.get_wrapper_type(): + result = result and self._platforms_parser.check_exists( + self.get_platform(), 'PROCESSORS_PER_NODE') + result = result and self._platforms_parser.check_exists( + self.get_platform(), 'MAX_PROCESSORS') + if 'vertical' in self.get_wrapper_type(): + result = result and self._platforms_parser.check_exists( + self.get_platform(), 'MAX_WALLCLOCK') + return result + + def reload(self): + """ + Creates parser objects for configuration files + """ + if not os.path.exists(self._conf_parser_file): raise IOError( + "Required file not found {0}".format(self._conf_parser_file)) + if not os.path.exists(self._platforms_parser_file): raise IOError( + "Required file not found {0}".format(self._platforms_parser_file)) + if not os.path.exists(self._jobs_parser_file): raise IOError( + "Required file not found {0}".format(self._jobs_parser_file)) + if not os.path.exists(self._exp_parser_file): raise IOError( + "Required file not found {0}".format(self._exp_parser_file)) + self._conf_parser = confConfigStrategy.get_parser(self.parser_factory, self._conf_parser_file) + self._platforms_parser = confConfigStrategy.get_parser(self.parser_factory, self._platforms_parser_file) + self._jobs_parser = confConfigStrategy.get_parser(self.parser_factory, self._jobs_parser_file) + self._exp_parser = confConfigStrategy.get_parser(self.parser_factory, self._exp_parser_file) + if self._proj_parser_file == '': + self._proj_parser = None + else: + self._proj_parser = confConfigStrategy.get_parser(self.parser_factory, self._proj_parser_file) + + def load_parameters(self): + """ + Load parameters from experiment and autosubmit config files. If experiment's type is not none, + also load parameters from model's config file + + :return: a dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + parameters = dict() + for section in self._exp_parser.sections(): + for option in self._exp_parser.options(section): + parameters[option] = self._exp_parser.get(section, option) + for section in self._conf_parser.sections(): + for option in self._conf_parser.options(section): + parameters[option] = self._conf_parser.get(section, option) + + # project_type = self.get_project_type() + # if project_type != "none" and self._proj_parser is not None: + # # Load project parameters + # Log.debug("Loading project parameters...") + # parameters2 = parameters.copy() + # parameters2.update(self.load_project_parameters()) + # parameters = parameters2 + + return parameters + + def load_project_parameters(self): + """ + Loads parameters from model config file + + :return: dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + pass + # projdef = [] + # for section in self._proj_parser.sections(): + # print("SEction" + section) + # projdef += self._proj_parser.items(section) + + # parameters = dict() + # for item in projdef: + # parameters[item[0]] = item[1] + + # return parameters + + def set_expid(self, exp_id): + """ + Set experiment identifier in autosubmit and experiment config files + + :param exp_id: experiment identifier to store + :type exp_id: str + """ + # Experiment conf + content = open(self._exp_parser_file).read() + if re.search('EXPID =.*', content): + content = content.replace( + re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) + open(self._exp_parser_file, 'w').write(content) + + content = open(self._conf_parser_file).read() + if re.search('EXPID =.*', content): + content = content.replace( + re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) + open(self._conf_parser_file, 'w').write(content) + + def get_project_type(self): + """ + Returns project type from experiment config file + + :return: project type + :rtype: str + """ + return self._exp_parser.get_option('project', 'PROJECT_TYPE', "NA").lower() + + def get_file_project_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + return self._exp_parser.get('project_files', 'FILE_PROJECT_CONF') + + def get_file_jobs_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + return self._exp_parser.get_option('project_files', 'FILE_JOBS_CONF', '') + + def get_git_project_origin(self): + """ + Returns git origin from experiment config file + + :return: git origin + :rtype: str + """ + return self._exp_parser.get_option('git', 'PROJECT_ORIGIN', '') + + def get_git_project_branch(self): + """ + Returns git branch from experiment's config file + + :return: git branch + :rtype: str + """ + return self._exp_parser.get_option('git', 'PROJECT_BRANCH', 'master') + + def get_git_project_commit(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) + + def get_submodules_list(self): + """ + Returns submodules list from experiment's config file + Default is --recursive + :return: submodules to load + :rtype: list + """ + return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES', '').split()).split() + + def get_project_destination(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + value = self._exp_parser.get('project', 'PROJECT_DESTINATION') + if not value: + if self.get_project_type().lower() == "local": + value = os.path.split(self.get_local_project_path())[1] + elif self.get_project_type().lower() == "svn": + value = self.get_svn_project_url().split('/')[-1] + elif self.get_project_type().lower() == "git": + value = self.get_git_project_origin().split( + '/')[-1].split('.')[-2] + return value + + def set_git_project_commit(self, as_conf): + """ + Function to register in the configuration the commit SHA of the git project version. + :param as_conf: Configuration class for exteriment + :type as_conf: AutosubmitConfig + """ + full_project_path = as_conf.get_project_dir() + try: + output = subprocess.check_output( + "cd {0}; git rev-parse --abbrev-ref HEAD".format(full_project_path), + shell=True) + except subprocess.CalledProcessError: + Log.critical("Failed to retrieve project branch...") + return False + + project_branch = output + Log.debug("Project branch is: " + project_branch) + try: + output = subprocess.check_output( + "cd {0}; git rev-parse HEAD".format(full_project_path), shell=True) + except subprocess.CalledProcessError: + Log.critical("Failed to retrieve project commit SHA...") + return False + project_sha = output + Log.debug("Project commit SHA is: " + project_sha) + + # register changes + content = open(self._exp_parser_file).read() + if re.search('PROJECT_BRANCH =.*', content): + content = content.replace(re.search('PROJECT_BRANCH =.*', content).group(0), + "PROJECT_BRANCH = " + project_branch) + if re.search('PROJECT_COMMIT =.*', content): + content = content.replace(re.search('PROJECT_COMMIT =.*', content).group(0), + "PROJECT_COMMIT = " + project_sha) + open(self._exp_parser_file, 'w').write(content) + Log.debug( + "Project commit SHA succesfully registered to the configuration file.") + return True + + def get_svn_project_url(self): + """ + Gets subversion project url + + :return: subversion project url + :rtype: str + """ + return self._exp_parser.get_option('svn', 'PROJECT_URL', 'NA') + + def get_svn_project_revision(self): + """ + Get revision for subversion project + + :return: revision for subversion project + :rtype: str + """ + return self._exp_parser.get('svn', 'PROJECT_REVISION') + + def get_local_project_path(self): + """ + Gets path to origin for local project + + :return: path to local project + :rtype: str + """ + return self._exp_parser.get('local', 'PROJECT_PATH') + + def get_date_list(self): + """ + Returns startdates list from experiment's config file + + :return: experiment's startdates + :rtype: list + """ + date_list = list() + string = self._exp_parser.get('experiment', 'DATELIST') + if not string.startswith("["): + string = '[{0}]'.format(string) + split_string = nestedExpr('[', ']').parseString(string).asList() + string_date = None + for split in split_string[0]: + if type(split) is list: + for split_in in split: + if split_in.find("-") != -1: + numbers = split_in.split("-") + for count in range(int(numbers[0]), int(numbers[1]) + 1): + date_list.append(parse_date( + string_date + str(count).zfill(len(numbers[0])))) + else: + date_list.append(parse_date(string_date + split_in)) + string_date = None + else: + if string_date is not None: + date_list.append(parse_date(string_date)) + string_date = split + if string_date is not None: + date_list.append(parse_date(string_date)) + return date_list + + def get_num_chunks(self): + """ + Returns number of chunks to run for each member + + :return: number of chunks + :rtype: int + """ + return int(self._exp_parser.get('experiment', 'NUMCHUNKS')) + + def get_chunk_ini(self, default=1): + """ + Returns the first chunk from where the experiment will start + + :param default: + :return: initial chunk + :rtype: int + """ + chunk_ini = self._exp_parser.get_option( + 'experiment', 'CHUNKINI', default) + if chunk_ini == '': + return default + return int(chunk_ini) + + def get_chunk_size_unit(self): + # type: () -> str + """ + Unit for the chunk length + + :return: Unit for the chunk length Options: {hour, day, month, year} + :rtype: str + """ + + # try: + # res = self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() + # except Exception as e: + # try: + # res = self.autosubmit_conf.get('experiment') + return self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() + + def get_chunk_size(self, default=1): + # type: (int) -> int + """ + Chunk Size as defined in the expdef file. + + :return: Chunksize, 1 as default. + :rtype: int + """ + try: + chunk_size = self._exp_parser.get_option( + 'experiment', 'CHUNKSIZE', default) + except Exception as exp: + print(exp) + chunk_size = '' + pass + if chunk_size == '': + return default + return int(chunk_size) + + def get_member_list(self, run_only=False): + """ + Returns members list from experiment's config file + + :return: experiment's members + :rtype: list + """ + member_list = list() + string = self._exp_parser.get('experiment', + 'MEMBERS') if run_only == False else self._exp_parser.get_option( + 'experiment', 'RUN_ONLY_MEMBERS', '') + if not string.startswith("["): + string = '[{0}]'.format(string) + split_string = nestedExpr('[', ']').parseString(string).asList() + string_member = None + for split in split_string[0]: + if type(split) is list: + for split_in in split: + if split_in.find("-") != -1: + numbers = split_in.split("-") + for count in range(int(numbers[0]), int(numbers[1]) + 1): + member_list.append( + string_member + str(count).zfill(len(numbers[0]))) + else: + member_list.append(string_member + split_in) + string_member = None + else: + if string_member is not None: + member_list.append(string_member) + string_member = split + if string_member is not None: + member_list.append(string_member) + return member_list + + def get_rerun(self): + """ + Returns startdates list from experiment's config file + + :return: rerurn value + :rtype: list + """ + + return self._exp_parser.get('rerun', 'RERUN').lower() + + def get_chunk_list(self): + """ + Returns chunk list from experiment's config file + + :return: experiment's chunks + :rtype: list + """ + return self._exp_parser.get('rerun', 'CHUNKLIST') + + def get_platform(self): + """ + Returns main platforms from experiment's config file + + :return: main platforms + :rtype: str + """ + return self._exp_parser.get('experiment', 'HPCARCH') + + def set_platform(self, hpc): + """ + Sets main platforms in experiment's config file + + :param hpc: main platforms + :type: str + """ + content = open(self._exp_parser_file).read() + if re.search('HPCARCH =.*', content): + content = content.replace( + re.search('HPCARCH =.*', content).group(0), "HPCARCH = " + hpc) + open(self._exp_parser_file, 'w').write(content) + + def set_version(self, autosubmit_version): + """ + Sets autosubmit's version in autosubmit's config file + + :param autosubmit_version: autosubmit's version + :type autosubmit_version: str + """ + content = open(self._conf_parser_file).read() + if re.search('AUTOSUBMIT_VERSION =.*', content): + content = content.replace(re.search('AUTOSUBMIT_VERSION =.*', content).group(0), + "AUTOSUBMIT_VERSION = " + autosubmit_version) + open(self._conf_parser_file, 'w').write(content) + + def get_version(self): + """ + Returns version number of the current experiment from autosubmit's config file + + :return: version + :rtype: str + """ + return self._conf_parser.get_option('config', 'AUTOSUBMIT_VERSION', 'None') + + def get_total_jobs(self): + """ + Returns max number of running jobs from autosubmit's config file + + :return: max number of running jobs + :rtype: int + """ + return int(self._conf_parser.get('config', 'TOTALJOBS')) + + def get_max_wallclock(self): + """ + Returns max wallclock + + :rtype: str + """ + return self._conf_parser.get_option('config', 'MAX_WALLCLOCK', '') + + def get_max_processors(self): + """ + Returns max processors from autosubmit's config file + + :rtype: str + """ + config_value = self._conf_parser.get_option( + 'config', 'MAX_PROCESSORS', None) + return int(config_value) if config_value is not None else config_value + + def get_max_waiting_jobs(self): + """ + Returns max number of waiting jobs from autosubmit's config file + + :return: main platforms + :rtype: int + """ + return int(self._conf_parser.get_option('config', 'MAXWAITINGJOBS', 10)) + + def get_default_job_type(self): + """ + Returns the default job type from experiment's config file + + :return: default type such as bash, python, r.. + :rtype: str + """ + return self._exp_parser.get_option('project_files', 'JOB_SCRIPTS_TYPE', 'bash') + + def get_safetysleeptime(self): + """ + Returns safety sleep time from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + return int(self._conf_parser.get_option('config', 'SAFETYSLEEPTIME', 10)) + + def set_safetysleeptime(self, sleep_time): + """ + Sets autosubmit's version in autosubmit's config file + + :param sleep_time: value to set + :type sleep_time: int + """ + content = open(self._conf_parser_file).read() + content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0), + "SAFETYSLEEPTIME = %d" % sleep_time) + open(self._conf_parser_file, 'w').write(content) + + def get_retrials(self): + """ + Returns max number of retrials for job from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + return int(self._conf_parser.get('config', 'RETRIALS')) + + def get_notifications(self): + """ + Returns if the user has enabled the notifications from autosubmit's config file + + :return: if notifications + :rtype: string + """ + return self._conf_parser.get_option('mail', 'NOTIFICATIONS', 'false').lower() + + def get_remote_dependencies(self): + """ + Returns if the user has enabled the remote dependencies from autosubmit's config file + + :return: if remote dependencies + :rtype: bool + """ + return self._conf_parser.get_option('wrapper', 'DEPENDENCIES', 'false').lower() == 'true' + + def get_wrapper_type(self): + """ + Returns what kind of wrapper (VERTICAL, MIXED-VERTICAL, HORIZONTAL, HYBRID, NONE) the user has configured in the autosubmit's config + + :return: wrapper type (or none) + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'TYPE', 'None').lower() + + def get_wrapper_jobs(self): + """ + Returns the jobs that should be wrapped, configured in the autosubmit's config + + :return: expression (or none) + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'JOBS_IN_WRAPPER', 'None') + + def get_max_wrapped_jobs(self): + """ + Returns the maximum number of jobs that can be wrapped together as configured in autosubmit's config file + + :return: maximum number of jobs (or total jobs) + :rtype: string + """ + # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs())) + + return int(self._conf_parser.get_option('wrapper', 'MAX_WRAPPED', self.get_total_jobs())) + + def get_wrapper_check_time(self): + """ + Returns time to check the status of jobs in the wrapper + + :return: wrapper check time + :rtype: int + """ + return int(self._conf_parser.get_option('wrapper', 'CHECK_TIME_WRAPPER', self.get_safetysleeptime())) + + def get_wrapper_machinefiles(self): + """ + Returns the strategy for creating the machinefiles in wrapper jobs + + :return: machinefiles function to use + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'MACHINEFILES', '') + + def get_wrapper_queue(self): + """ + Returns the wrapper queue if not defined, will be the one of the first job wrapped + + :return: expression (or none) + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'QUEUE', "") + + def get_jobs_sections(self): + """ + Returns the list of sections defined in the jobs config file + + :return: sections + :rtype: list + """ + return self._jobs_parser.sections() + + def get_copy_remote_logs(self): + """ + Returns if the user has enabled the logs local copy from autosubmit's config file + + :return: if logs local copy + :rtype: bool + """ + return self._conf_parser.get_option('storage', 'COPY_REMOTE_LOGS', 'true').lower() + + def get_mails_to(self): + """ + Returns the address where notifications will be sent from autosubmit's config file + + :return: mail address + :rtype: [str] + """ + return [str(x) for x in self._conf_parser.get_option('mail', 'TO', '').split(' ')] + + def get_communications_library(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + return self._conf_parser.get_option('communications', 'API', 'paramiko').lower() + + def get_storage_type(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower() + + @staticmethod + def is_valid_mail_address(mail_address): + if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): + return True + else: + return False + + def is_valid_communications_library(self): + library = self.get_communications_library() + return library in ['paramiko', 'saga'] + + def is_valid_storage_type(self): + storage_type = self.get_storage_type() + return storage_type in ['pkl', 'db'] + + def is_valid_jobs_in_wrapper(self): + expression = self.get_wrapper_jobs() + if expression != 'None': + parser = self._jobs_parser + sections = parser.sections() + for section in expression.split(" "): + if "&" in section: + for inner_section in section.split("&"): + if inner_section not in sections: + return False + elif section not in sections: + return False + return True + + def is_valid_git_repository(self): + origin_exists = self._exp_parser.check_exists('git', 'PROJECT_ORIGIN') + branch = self.get_git_project_branch() + commit = self.get_git_project_commit() + return origin_exists and (branch is not None or commit is not None) + + @staticmethod + def get_parser(parser_factory, file_path): + # type: (ConfigParserFactory, str) -> ConfigParser + """ + Gets parser for given file + + :param parser_factory: + :param file_path: path to file to be parsed + :type file_path: str + :return: parser + :rtype: SafeConfigParser + """ + parser = parser_factory.create_parser() + parser.optionxform = str + # proj is not required + # print(file_path) + if file_path.find('proj_') > 0: + parser.read(file_path) + else: + with open(file_path) as f: + parser.read(file_path) + return parser diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index c73453711ed51bba574a90a0bd94b2e058320160..ca5cc4bca38fddfe2ce6eb4abe2ea241d77550dc 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -19,20 +19,29 @@ try: # noinspection PyCompatibility from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser + + import os import re import subprocess import json +import logging +import locale from pyparsing import nestedExpr from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser from bscearth.utils.date import parse_date from bscearth.utils.log import Log from ..config.basicConfig import BasicConfig +from ..config.IConfigStrategy import IConfigStrategy +from ..config.ymlConfigStrategy import ymlConfigStrategy +from ..config.confConfigStrategy import confConfigStrategy +logger = logging.getLogger('gunicorn.error') class AutosubmitConfig(object): """ @@ -40,70 +49,39 @@ class AutosubmitConfig(object): :param expid: experiment identifier :type expid: str + :configWrapper: IConfigStrategy -> handling strategy for the type of config files used """ - def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + def __init__(self, expid, basic_config, parser_factory, extension="yml"): # type: (str, BasicConfig, ConfigParserFactory, Extension) -> None - self.expid = expid + self.expid = expid + self._configWrapper = None self.basic_config = basic_config - self.parser_factory = parser_factory - # By default check for .yml files first as it is the new standard for AS 4.0 - - self._conf_parser = None # type: ConfigParser - self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "autosubmit_" + expid + extension) - if os.path.exists(self._conf_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self._exp_parser = None # type: ConfigParser - self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "expdef_" + expid + extension) - if os.path.exists(self._exp_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self._platforms_parser = None # type: ConfigParser - self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + extension) - if os.path.exists(self._platforms_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self._jobs_parser = None # type: ConfigParser - self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "jobs_" + expid + extension) - if os.path.exists(self._jobs_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self._proj_parser = None # type: ConfigParser - self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "proj_" + expid + extension) - if os.path.exists(self._proj_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self.check_proj_file() + # check which type of config files (AS3 or AS4) + platform_conf_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + "." + extension) + if os.path.exists(platform_conf_file): + logger.info("Setting AS4 Config strategy - yml") + self._configWrapper = ymlConfigStrategy(expid, basic_config, parser_factory, ".yml") + else: + logger.info("Setting AS3 Config strategy - conf") + self._configWrapper = confConfigStrategy(expid, basic_config, parser_factory, ".conf") + @property def jobs_parser(self): - return self._jobs_parser + return self._configWrapper.jobs_parser @property def experiment_file(self): """ Returns experiment's config file name """ - return self._exp_parser_file + #return self._exp_parser_file + return self._configWrapper.experiment_file + @property def platforms_parser(self): @@ -113,7 +91,7 @@ class AutosubmitConfig(object): :return: platforms config parser object :rtype: SafeConfigParser """ - return self._platforms_parser + return self._configWrapper.platforms_parser @property def platforms_file(self): @@ -123,27 +101,19 @@ class AutosubmitConfig(object): :return: platforms config file's name :rtype: str """ - return self._platforms_parser_file + return self._configWrapper.platforms_parser_file @property def project_file(self): """ Returns project's config file name """ - return self._proj_parser_file + return self._configWrapper.project_file def check_proj_file(self): """ Add a section header to the project's configuration file (if not exists) """ - # if os.path.exists(self._proj_parser_file): - # with open(self._proj_parser_file, 'w') as f: - # first_line = f.readline() - # if not re.match('[[a-zA-Z0-9]*]', first_line): - # content = f.read() - # f.seek(0, 0) - # f.write('[DEFAULT]'.rstrip('\r\n') + - # '\n' + first_line + content) pass @property @@ -151,45 +121,19 @@ class AutosubmitConfig(object): """ Returns project's jobs file name """ - return self._jobs_parser_file + return self._configWrapper.jobs_file() def get_full_config_as_dict(self): """ Returns full configuration as json object """ - _conf = _exp = _platforms = _jobs = _proj = None - result = {} - - def get_data(parser): - """ - dictionary comprehension to get data from parser - """ - res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ - section for section in parser.sections()]} - return res - # print(self._conf_parser) - result["conf"] = get_data( - self._conf_parser) if self._conf_parser else None - result["exp"] = get_data( - self._exp_parser) if self._exp_parser else None - result["platforms"] = get_data( - self._platforms_parser) if self._platforms_parser else None - result["jobs"] = get_data( - self._jobs_parser) if self._jobs_parser else None - result["proj"] = get_data( - self._proj_parser) if self._proj_parser else None - return result + return self._configWrapper.get_full_config_as_dict() def get_full_config_as_json(self): """ Return config as json object """ - try: - return json.dumps(self.get_full_config_as_dict()) - except Exception as exp: - Log.warning( - "Autosubmit was not able to retrieve and save the configuration into the historical database.") - return "" + return self._configWrapper.get_full_config_as_json() def get_project_dir(self): """ @@ -198,9 +142,8 @@ class AutosubmitConfig(object): :return: experiment's project directory :rtype: str """ - dir_templates = os.path.join(self.basic_config.LOCAL_ROOT_DIR, self.expid, BasicConfig.LOCAL_PROJ_DIR, - self.get_project_destination()) - return dir_templates + return self._configWrapper.get_project_dir() + def get_queue(self, section): """ @@ -210,22 +153,22 @@ class AutosubmitConfig(object): :return: queue :rtype: str """ - return self._jobs_parser.get_option(section, 'QUEUE', '') + return self._configWrapper.get_queue(section) def get_job_platform(self, section): - return self._jobs_parser.get_option(section, 'PLATFORM', '') + return self._configWrapper.get_job_platform(section) def get_platform_queue(self, platform): - return self._platforms_parser.get_option(platform, 'QUEUE', '') + return self._configWrapper.get_platform_queue(platform) def get_platform_serial_queue(self, platform): - return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '') + return self._configWrapper.get_platform_serial_queue(platform) def get_platform_project(self, platform): - return self._platforms_parser.get_option(platform, "PROJECT", "") + return self._configWrapper.get_platform_project(platform) def get_platform_wallclock(self, platform): - return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '') + return self._configWrapper.get_platform_wallclock(platform) def get_wallclock(self, section): """ @@ -235,7 +178,7 @@ class AutosubmitConfig(object): :return: wallclock time :rtype: str """ - return self._jobs_parser.get_option(section, 'WALLCLOCK', '') + return self._configWrapper.get_wallclock(section) def get_synchronize(self, section): """ @@ -245,7 +188,7 @@ class AutosubmitConfig(object): :return: wallclock time :rtype: str """ - return self._jobs_parser.get_option(section, 'SYNCHRONIZE', '') + return self._configWrapper.get_synchronize(section) def get_processors(self, section): """ @@ -255,7 +198,7 @@ class AutosubmitConfig(object): :return: wallclock time :rtype: str """ - return str(self._jobs_parser.get_option(section, 'PROCESSORS', 1)) + return self._configWrapper.get_processors(section) def get_threads(self, section): """ @@ -265,7 +208,7 @@ class AutosubmitConfig(object): :return: threads needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'THREADS', 1)) + return self._configWrapper.get_threads(section) def get_tasks(self, section): """ @@ -275,7 +218,7 @@ class AutosubmitConfig(object): :return: tasks (processes) per host :rtype: str """ - return str(self._jobs_parser.get_option(section, 'TASKS', 0)) + return self._configWrapper.get_tasks(section) def get_scratch_free_space(self, section): """ @@ -285,7 +228,7 @@ class AutosubmitConfig(object): :return: percentage of scratch free space needed :rtype: int """ - return int(self._jobs_parser.get_option(section, 'SCRATCH_FREE_SPACE', 0)) + return self._configWrapper.get_scratch_free_space(section) def get_memory(self, section): """ @@ -295,7 +238,7 @@ class AutosubmitConfig(object): :return: memory needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'MEMORY', '')) + return self._configWrapper.get_memory(section) def get_memory_per_task(self, section): """ @@ -305,7 +248,7 @@ class AutosubmitConfig(object): :return: memory per task needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', '')) + return self._configWrapper.get_memory_per_task(section) def get_migrate_user_to(self, section): """ @@ -314,7 +257,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + return self.get_migrate_user_to(section) def get_current_user(self, section): """ @@ -323,7 +266,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return self._platforms_parser.get_option(section, 'USER', '').lower() + return self._configWrapper.get_current_user(section) def get_current_project(self, section): """ @@ -332,7 +275,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return self._platforms_parser.get_option(section, 'PROJECT', '').lower() + return self._configWrapper.get_current_project(section) def set_new_user(self, section, new_user): """ @@ -341,27 +284,7 @@ class AutosubmitConfig(object): :param section: platform name :type: str """ - with open(self._platforms_parser_file) as p_file: - contentLine = p_file.readline() - contentToMod = "" - content = "" - mod = False - while contentLine: - if re.search(section, contentLine): - mod = True - if mod: - contentToMod += contentLine - else: - content += contentLine - contentLine = p_file.readline() - if mod: - old_user = self.get_current_user(section) - contentToMod = contentToMod.replace(re.search( - r'[^#]\bUSER\b =.*', contentToMod).group(0)[1:], "USER = " + new_user) - contentToMod = contentToMod.replace(re.search( - r'[^#]\bUSER_TO\b =.*', contentToMod).group(0)[1:], "USER_TO = " + old_user) - open(self._platforms_parser_file, 'w').write(content) - open(self._platforms_parser_file, 'a').write(contentToMod) + self._configWrapper.set_new_user(section,new_user) def get_migrate_project_to(self, section): """ @@ -370,7 +293,8 @@ class AutosubmitConfig(object): :return: migrate project to :rtype: str """ - return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() + return self._configWrapper.get_migrate_project_to(section) + def set_new_project(self, section, new_project): """ @@ -379,27 +303,7 @@ class AutosubmitConfig(object): :param section: platform name :type: str """ - with open(self._platforms_parser_file) as p_file: - contentLine = p_file.readline() - contentToMod = "" - content = "" - mod = False - while contentLine: - if re.search(section, contentLine): - mod = True - if mod: - contentToMod += contentLine - else: - content += contentLine - contentLine = p_file.readline() - if mod: - old_project = self.get_current_project(section) - contentToMod = contentToMod.replace(re.search( - r"[^#]\bPROJECT\b =.*", contentToMod).group(0)[1:], "PROJECT = " + new_project) - contentToMod = contentToMod.replace(re.search( - r"[^#]\bPROJECT_TO\b =.*", contentToMod).group(0)[1:], "PROJECT_TO = " + old_project) - open(self._platforms_parser_file, 'w').write(content) - open(self._platforms_parser_file, 'a').write(contentToMod) + self._configWrapper.set_new_project(section,new_project) def get_custom_directives(self, section): """ @@ -409,7 +313,7 @@ class AutosubmitConfig(object): :return: custom directives needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'CUSTOM_DIRECTIVES', '')) + return self._configWrapper.get_custom_directives(section) def check_conf_files(self): """ @@ -419,18 +323,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it finds any error :rtype: bool """ - Log.debug('\nChecking configuration files...') - self.reload() - # result = self.check_platforms_conf() - result = True - result = result and self.check_jobs_conf() - result = result and self.check_autosubmit_conf() - result = result and self.check_expdef_conf() - if result: - Log.debug("Configuration files OK\n") - else: - Log.error("Configuration files invalid\n") - return result + return self._configWrapper.check_conf_files() def check_autosubmit_conf(self): """ @@ -439,42 +332,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - - self._conf_parser.read(self._conf_parser_file) - # result = result and self._conf_parser.check_exists( - # 'config', 'AUTOSUBMIT_VERSION') - # result = result and self._conf_parser.check_is_int( - # 'config', 'MAXWAITINGJOBS', True) - # result = result and self._conf_parser.check_is_int( - # 'config', 'TOTALJOBS', True) - # result = result and self._conf_parser.check_is_int( - # 'config', 'SAFETYSLEEPTIME', True) - # result = result and self._conf_parser.check_is_int( - # 'config', 'RETRIALS', True) - # result = result and self._conf_parser.check_is_boolean( - # 'mail', 'NOTIFICATIONS', False) - result = result and self.is_valid_communications_library() - result = result and self.is_valid_storage_type() - if self.get_wrapper_type() != 'None': - result = result and self.check_wrapper_conf() - - if self.get_notifications() == 'true': - for mail in self.get_mails_to(): - if not self.is_valid_mail_address(mail): - # Log.warning( - # 'One or more of the email addresses configured for the mail notifications are wrong') - break - - if not result: - # Log.critical("{0} is not a valid config file".format( - # os.path.basename(self._conf_parser_file))) - raise Exception("Permission denied for " + - str(os.path.basename(self._conf_parser_file))) - else: - Log.debug('{0} OK'.format( - os.path.basename(self._conf_parser_file))) - return result + return self._configWrapper.check_autosubmit_conf() def check_platforms_conf(self): """ @@ -483,46 +341,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - if len(self._platforms_parser.sections()) == 0: - Log.warning("No remote platforms configured") - - if len(self._platforms_parser.sections()) != len(set(self._platforms_parser.sections())): - Log.error('There are repeated platforms names') - - for section in self._platforms_parser.sections(): - result = result and self._platforms_parser.check_exists( - section, 'TYPE') - platform_type = self._platforms_parser.get_option( - section, 'TYPE', '').lower() - if platform_type != 'ps': - result = result and self._platforms_parser.check_exists( - section, 'PROJECT') - result = result and self._platforms_parser.check_exists( - section, 'USER') - - result = result and self._platforms_parser.check_exists( - section, 'HOST') - result = result and self._platforms_parser.check_exists( - section, 'SCRATCH_DIR') - result = result and self._platforms_parser.check_is_boolean(section, - 'ADD_PROJECT_TO_HOST', False) - result = result and self._platforms_parser.check_is_boolean( - section, 'TEST_SUITE', False) - result = result and self._platforms_parser.check_is_int(section, 'MAX_WAITING_JOBS', - False) - result = result and self._platforms_parser.check_is_int( - section, 'TOTAL_JOBS', False) - - if not result: - Log.critical("{0} is not a valid config file".format( - os.path.basename(self._platforms_parser_file))) - raise Exception("Permission denied for " + - str(os.path.basename(self._platforms_parser_file))) - else: - Log.info('{0} OK'.format( - os.path.basename(self._platforms_parser_file))) - return result + return self._configWrapper.check_platforms_conf() def check_jobs_conf(self): """ @@ -531,65 +350,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - parser = self._jobs_parser - sections = parser.sections() - platforms = self._platforms_parser.sections() - platforms.append('LOCAL') - possible_exception = "" - if len(sections) == 0: - possible_exception += "No remote platforms configured\n" - - # if len(sections) != len(set(sections)): - # Log.error('There are repeated job names') - - for section in sections: - - result = result and parser.check_exists(section, 'FILE') - result = result and parser.check_is_boolean( - section, 'RERUN_ONLY', False) - - if parser.has_option(section, 'PLATFORM'): - # result = result and parser.check_is_choice( - # section, 'PLATFORM', False, platforms) - pass - if parser.has_option(section, 'DEPENDENCIES'): - for dependency in str(parser.get_option(section, 'DEPENDENCIES', '')).split(' '): - if '-' in dependency: - dependency = dependency.split('-')[0] - elif '+' in dependency: - dependency = dependency.split('+')[0] - if '[' in dependency: - dependency = dependency[:dependency.find('[')] - # if dependency not in sections: - # Log.error( - # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, - # dependency)) - - if parser.has_option(section, 'RERUN_DEPENDENCIES'): - for dependency in str(parser.get_option(section, 'RERUN_DEPENDENCIES', - '')).split(' '): - if '-' in dependency: - dependency = dependency.split('-')[0] - if '[' in dependency: - dependency = dependency[:dependency.find('[')] - # if dependency not in sections: - # Log.error( - # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, - # dependency)) - result = result and parser.check_is_choice(section, 'RUNNING', False, - ['once', 'date', 'member', 'chunk']) - - if not result: - # Log.critical("{0} is not a valid config file".format( - # os.path.basename(self._jobs_parser_file))) - raise Exception("Exception while checking jobs_expid.conf " + - str(os.path.basename(self._jobs_parser_file)) + possible_exception) - else: - Log.debug('{0} OK'.format( - os.path.basename(self._jobs_parser_file))) - - return result + return self._configWrapper.check_jobs_conf() def check_expdef_conf(self): """ @@ -598,58 +359,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - parser = self._exp_parser - - # result = result and parser.check_exists('DEFAULT', 'EXPID') - # result = result and parser.check_exists('DEFAULT', 'HPCARCH') - - result = result and parser.check_exists('experiment', 'DATELIST') - result = result and parser.check_exists('experiment', 'MEMBERS') - result = result and parser.check_is_choice('experiment', 'CHUNKSIZEUNIT', True, - ['year', 'month', 'day', 'hour']) - result = result and parser.check_is_int( - 'experiment', 'CHUNKSIZE', True) - result = result and parser.check_is_int( - 'experiment', 'NUMCHUNKS', True) - result = result and parser.check_is_choice('experiment', 'CALENDAR', True, - ['standard', 'noleap']) - - result = result and parser.check_is_boolean('rerun', 'RERUN', True) - - if parser.check_is_choice('project', 'PROJECT_TYPE', True, - ['none', 'git', 'svn', 'local']): - project_type = parser.get_option('project', 'PROJECT_TYPE', '') - - if project_type == 'git': - result = result and parser.check_exists( - 'git', 'PROJECT_ORIGIN') - result = result and parser.check_exists( - 'git', 'PROJECT_BRANCH') - - elif project_type == 'svn': - result = result and parser.check_exists('svn', 'PROJECT_URL') - result = result and parser.check_exists( - 'svn', 'PROJECT_REVISION') - elif project_type == 'local': - result = result and parser.check_exists( - 'local', 'PROJECT_PATH') - - if project_type != 'none': - result = result and parser.check_exists( - 'project_files', 'FILE_PROJECT_CONF') - else: - result = True - - if not result: - # Log.critical("{0} is not a valid config file".format( - # os.path.basename(self._exp_parser_file))) - raise Exception("Permission denied for " + - str(os.path.basename(self._exp_parser_file))) - else: - Log.debug('{0} OK'.format( - os.path.basename(self._exp_parser_file))) - return result + return self._configWrapper.check_expdef_conf() def check_proj(self): """ @@ -658,50 +368,17 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - try: - if self._proj_parser_file == '': - self._proj_parser = None - else: - self._proj_parser = AutosubmitConfig.get_parser( - self.parser_factory, self._proj_parser_file) - return True - except Exception as e: - Log.error('Project conf file error: {0}', e) - return False + return self._configWrapper.check_proj() def check_wrapper_conf(self): - result = True - result = result and self.is_valid_jobs_in_wrapper() - if not result: - Log.error( - "There are sections in JOBS_IN_WRAPPER that are not defined in your jobs.conf file") - - if 'horizontal' in self.get_wrapper_type(): - result = result and self._platforms_parser.check_exists( - self.get_platform(), 'PROCESSORS_PER_NODE') - result = result and self._platforms_parser.check_exists( - self.get_platform(), 'MAX_PROCESSORS') - if 'vertical' in self.get_wrapper_type(): - result = result and self._platforms_parser.check_exists( - self.get_platform(), 'MAX_WALLCLOCK') - return result + return def reload(self): """ Creates parser objects for configuration files """ - if not os.path.exists(self._conf_parser_file): raise IOError("Required file not found {0}".format(self._conf_parser_file)) - if not os.path.exists(self._platforms_parser_file): raise IOError("Required file not found {0}".format(self._platforms_parser_file)) - if not os.path.exists(self._jobs_parser_file): raise IOError("Required file not found {0}".format(self._jobs_parser_file)) - if not os.path.exists(self._exp_parser_file): raise IOError("Required file not found {0}".format(self._exp_parser_file)) - self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file) - self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) - self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file) - self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file) - if self._proj_parser_file == '': - self._proj_parser = None - else: - self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file) + self._configWrapper.reload() + def load_parameters(self): """ @@ -711,23 +388,7 @@ class AutosubmitConfig(object): :return: a dictionary containing tuples [parameter_name, parameter_value] :rtype: dict """ - parameters = dict() - for section in self._exp_parser.sections(): - for option in self._exp_parser.options(section): - parameters[option] = self._exp_parser.get(section, option) - for section in self._conf_parser.sections(): - for option in self._conf_parser.options(section): - parameters[option] = self._conf_parser.get(section, option) - - # project_type = self.get_project_type() - # if project_type != "none" and self._proj_parser is not None: - # # Load project parameters - # Log.debug("Loading project parameters...") - # parameters2 = parameters.copy() - # parameters2.update(self.load_project_parameters()) - # parameters = parameters2 - - return parameters + return self._configWrapper.load_parameters() def load_project_parameters(self): """ @@ -736,17 +397,7 @@ class AutosubmitConfig(object): :return: dictionary containing tuples [parameter_name, parameter_value] :rtype: dict """ - pass - # projdef = [] - # for section in self._proj_parser.sections(): - # print("SEction" + section) - # projdef += self._proj_parser.items(section) - - # parameters = dict() - # for item in projdef: - # parameters[item[0]] = item[1] - - # return parameters + return self._configWrapper.load_project_parameters() def set_expid(self, exp_id): """ @@ -755,18 +406,7 @@ class AutosubmitConfig(object): :param exp_id: experiment identifier to store :type exp_id: str """ - # Experiment conf - content = open(self._exp_parser_file).read() - if re.search('EXPID =.*', content): - content = content.replace( - re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) - open(self._exp_parser_file, 'w').write(content) - - content = open(self._conf_parser_file).read() - if re.search('EXPID =.*', content): - content = content.replace( - re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) - open(self._conf_parser_file, 'w').write(content) + self._configWrapper.set_expid(exp_id) def get_project_type(self): """ @@ -775,7 +415,7 @@ class AutosubmitConfig(object): :return: project type :rtype: str """ - return self._exp_parser.get_option('project', 'PROJECT_TYPE', "NA").lower() + return self._configWrapper.get_project_type() def get_file_project_conf(self): """ @@ -784,7 +424,7 @@ class AutosubmitConfig(object): :return: path to project config file :rtype: str """ - return self._exp_parser.get('project_files', 'FILE_PROJECT_CONF') + return self._configWrapper.get_file_project_conf() def get_file_jobs_conf(self): """ @@ -793,7 +433,8 @@ class AutosubmitConfig(object): :return: path to project config file :rtype: str """ - return self._exp_parser.get_option('project_files', 'FILE_JOBS_CONF', '') + return self._configWrapper.get_file_jobs_conf() + def get_git_project_origin(self): """ @@ -802,7 +443,7 @@ class AutosubmitConfig(object): :return: git origin :rtype: str """ - return self._exp_parser.get_option('git', 'PROJECT_ORIGIN', '') + return self._configWrapper.get_git_project_origin() def get_git_project_branch(self): """ @@ -811,7 +452,7 @@ class AutosubmitConfig(object): :return: git branch :rtype: str """ - return self._exp_parser.get_option('git', 'PROJECT_BRANCH', 'master') + return self._configWrapper.get_git_project_branch() def get_git_project_commit(self): """ @@ -820,7 +461,7 @@ class AutosubmitConfig(object): :return: git commit :rtype: str """ - return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) + return self._configWrapper.get_git_project_commit() def get_submodules_list(self): """ @@ -829,7 +470,7 @@ class AutosubmitConfig(object): :return: submodules to load :rtype: list """ - return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES', '').split()).split() + return self._configWrapper.get_submodules_list() def get_project_destination(self): """ @@ -838,16 +479,7 @@ class AutosubmitConfig(object): :return: git commit :rtype: str """ - value = self._exp_parser.get('project', 'PROJECT_DESTINATION') - if not value: - if self.get_project_type().lower() == "local": - value = os.path.split(self.get_local_project_path())[1] - elif self.get_project_type().lower() == "svn": - value = self.get_svn_project_url().split('/')[-1] - elif self.get_project_type().lower() == "git": - value = self.get_git_project_origin().split( - '/')[-1].split('.')[-2] - return value + return self._configWrapper.get_project_destination() def set_git_project_commit(self, as_conf): """ @@ -855,37 +487,7 @@ class AutosubmitConfig(object): :param as_conf: Configuration class for exteriment :type as_conf: AutosubmitConfig """ - full_project_path = as_conf.get_project_dir() - try: - output = subprocess.check_output("cd {0}; git rev-parse --abbrev-ref HEAD".format(full_project_path), - shell=True) - except subprocess.CalledProcessError: - Log.critical("Failed to retrieve project branch...") - return False - - project_branch = output - Log.debug("Project branch is: " + project_branch) - try: - output = subprocess.check_output( - "cd {0}; git rev-parse HEAD".format(full_project_path), shell=True) - except subprocess.CalledProcessError: - Log.critical("Failed to retrieve project commit SHA...") - return False - project_sha = output - Log.debug("Project commit SHA is: " + project_sha) - - # register changes - content = open(self._exp_parser_file).read() - if re.search('PROJECT_BRANCH =.*', content): - content = content.replace(re.search('PROJECT_BRANCH =.*', content).group(0), - "PROJECT_BRANCH = " + project_branch) - if re.search('PROJECT_COMMIT =.*', content): - content = content.replace(re.search('PROJECT_COMMIT =.*', content).group(0), - "PROJECT_COMMIT = " + project_sha) - open(self._exp_parser_file, 'w').write(content) - Log.debug( - "Project commit SHA succesfully registered to the configuration file.") - return True + return self._configWrapper.get_git_project_commit(as_conf) def get_svn_project_url(self): """ @@ -894,7 +496,7 @@ class AutosubmitConfig(object): :return: subversion project url :rtype: str """ - return self._exp_parser.get_option('svn', 'PROJECT_URL', 'NA') + return self._configWrapper.get_svn_project_url() def get_svn_project_revision(self): """ @@ -903,7 +505,7 @@ class AutosubmitConfig(object): :return: revision for subversion project :rtype: str """ - return self._exp_parser.get('svn', 'PROJECT_REVISION') + return self._configWrapper.get_svn_project_revision() def get_local_project_path(self): """ @@ -912,7 +514,7 @@ class AutosubmitConfig(object): :return: path to local project :rtype: str """ - return self._exp_parser.get('local', 'PROJECT_PATH') + return self._configWrapper.get_local_project_path() def get_date_list(self): """ @@ -921,30 +523,7 @@ class AutosubmitConfig(object): :return: experiment's startdates :rtype: list """ - date_list = list() - string = self._exp_parser.get('experiment', 'DATELIST') - if not string.startswith("["): - string = '[{0}]'.format(string) - split_string = nestedExpr('[', ']').parseString(string).asList() - string_date = None - for split in split_string[0]: - if type(split) is list: - for split_in in split: - if split_in.find("-") != -1: - numbers = split_in.split("-") - for count in range(int(numbers[0]), int(numbers[1]) + 1): - date_list.append(parse_date( - string_date + str(count).zfill(len(numbers[0])))) - else: - date_list.append(parse_date(string_date + split_in)) - string_date = None - else: - if string_date is not None: - date_list.append(parse_date(string_date)) - string_date = split - if string_date is not None: - date_list.append(parse_date(string_date)) - return date_list + return self._configWrapper.get_date_list() def get_num_chunks(self): """ @@ -953,7 +532,7 @@ class AutosubmitConfig(object): :return: number of chunks :rtype: int """ - return int(self._exp_parser.get('experiment', 'NUMCHUNKS')) + return self._configWrapper.get_num_chunks() def get_chunk_ini(self, default=1): """ @@ -963,11 +542,7 @@ class AutosubmitConfig(object): :return: initial chunk :rtype: int """ - chunk_ini = self._exp_parser.get_option( - 'experiment', 'CHUNKINI', default) - if chunk_ini == '': - return default - return int(chunk_ini) + return self._configWrapper.get_chunk_ini(default) def get_chunk_size_unit(self): # type: () -> str @@ -978,12 +553,7 @@ class AutosubmitConfig(object): :rtype: str """ - # try: - # res = self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() - # except Exception as e: - # try: - # res = self.autosubmit_conf.get('experiment') - return self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() + return self._configWrapper.get_chunk_size_unit() def get_chunk_size(self, default=1): # type: (int) -> int @@ -993,16 +563,7 @@ class AutosubmitConfig(object): :return: Chunksize, 1 as default. :rtype: int """ - try: - chunk_size = self._exp_parser.get_option( - 'experiment', 'CHUNKSIZE', default) - except Exception as exp: - print(exp) - chunk_size = '' - pass - if chunk_size == '': - return default - return int(chunk_size) + return self._configWrapper.get_chunk_size(default) def get_member_list(self, run_only=False): """ @@ -1011,31 +572,7 @@ class AutosubmitConfig(object): :return: experiment's members :rtype: list """ - member_list = list() - string = self._exp_parser.get('experiment', 'MEMBERS') if run_only == False else self._exp_parser.get_option( - 'experiment', 'RUN_ONLY_MEMBERS', '') - if not string.startswith("["): - string = '[{0}]'.format(string) - split_string = nestedExpr('[', ']').parseString(string).asList() - string_member = None - for split in split_string[0]: - if type(split) is list: - for split_in in split: - if split_in.find("-") != -1: - numbers = split_in.split("-") - for count in range(int(numbers[0]), int(numbers[1]) + 1): - member_list.append( - string_member + str(count).zfill(len(numbers[0]))) - else: - member_list.append(string_member + split_in) - string_member = None - else: - if string_member is not None: - member_list.append(string_member) - string_member = split - if string_member is not None: - member_list.append(string_member) - return member_list + return self._configWrapper.get_member_list(run_only) def get_rerun(self): """ @@ -1045,7 +582,7 @@ class AutosubmitConfig(object): :rtype: list """ - return self._exp_parser.get('rerun', 'RERUN').lower() + return self._configWrapper.get_rerun() def get_chunk_list(self): """ @@ -1054,7 +591,7 @@ class AutosubmitConfig(object): :return: experiment's chunks :rtype: list """ - return self._exp_parser.get('rerun', 'CHUNKLIST') + return self._configWrapper.get_chunk_list() def get_platform(self): """ @@ -1063,7 +600,7 @@ class AutosubmitConfig(object): :return: main platforms :rtype: str """ - return self._exp_parser.get('experiment', 'HPCARCH') + return self._configWrapper.get_platform() def set_platform(self, hpc): """ @@ -1072,11 +609,7 @@ class AutosubmitConfig(object): :param hpc: main platforms :type: str """ - content = open(self._exp_parser_file).read() - if re.search('HPCARCH =.*', content): - content = content.replace( - re.search('HPCARCH =.*', content).group(0), "HPCARCH = " + hpc) - open(self._exp_parser_file, 'w').write(content) + self._configWrapper.set_platform(hpc) def set_version(self, autosubmit_version): """ @@ -1085,11 +618,7 @@ class AutosubmitConfig(object): :param autosubmit_version: autosubmit's version :type autosubmit_version: str """ - content = open(self._conf_parser_file).read() - if re.search('AUTOSUBMIT_VERSION =.*', content): - content = content.replace(re.search('AUTOSUBMIT_VERSION =.*', content).group(0), - "AUTOSUBMIT_VERSION = " + autosubmit_version) - open(self._conf_parser_file, 'w').write(content) + return self._configWrapper.set_version(autosubmit_version) def get_version(self): """ @@ -1098,7 +627,7 @@ class AutosubmitConfig(object): :return: version :rtype: str """ - return self._conf_parser.get_option('config', 'AUTOSUBMIT_VERSION', 'None') + return self._configWrapper.get_version() def get_total_jobs(self): """ @@ -1107,7 +636,7 @@ class AutosubmitConfig(object): :return: max number of running jobs :rtype: int """ - return int(self._conf_parser.get('config', 'TOTALJOBS')) + return self._configWrapper.get_total_jobs() def get_max_wallclock(self): """ @@ -1115,7 +644,7 @@ class AutosubmitConfig(object): :rtype: str """ - return self._conf_parser.get_option('config', 'MAX_WALLCLOCK', '') + return self._configWrapper.get_max_wallclock() def get_max_processors(self): """ @@ -1123,9 +652,7 @@ class AutosubmitConfig(object): :rtype: str """ - config_value = self._conf_parser.get_option( - 'config', 'MAX_PROCESSORS', None) - return int(config_value) if config_value is not None else config_value + return self._configWrapper.get_max_processors() def get_max_waiting_jobs(self): """ @@ -1134,7 +661,7 @@ class AutosubmitConfig(object): :return: main platforms :rtype: int """ - return int(self._conf_parser.get_option('config', 'MAXWAITINGJOBS', 10)) + return self._configWrapper.get_max_waiting_jobs() def get_default_job_type(self): """ @@ -1143,7 +670,7 @@ class AutosubmitConfig(object): :return: default type such as bash, python, r.. :rtype: str """ - return self._exp_parser.get_option('project_files', 'JOB_SCRIPTS_TYPE', 'bash') + return self._configWrapper.get_default_job_type() def get_safetysleeptime(self): """ @@ -1152,7 +679,7 @@ class AutosubmitConfig(object): :return: safety sleep time :rtype: int """ - return int(self._conf_parser.get_option('config', 'SAFETYSLEEPTIME', 10)) + return self._configWrapper.get_safetysleeptime() def set_safetysleeptime(self, sleep_time): """ @@ -1161,10 +688,7 @@ class AutosubmitConfig(object): :param sleep_time: value to set :type sleep_time: int """ - content = open(self._conf_parser_file).read() - content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0), - "SAFETYSLEEPTIME = %d" % sleep_time) - open(self._conf_parser_file, 'w').write(content) + self._configWrapper.set_safetysleeptime(sleep_time) def get_retrials(self): """ @@ -1173,7 +697,7 @@ class AutosubmitConfig(object): :return: safety sleep time :rtype: int """ - return int(self._conf_parser.get('config', 'RETRIALS')) + return self._configWrapper.get_retrials() def get_notifications(self): """ @@ -1182,7 +706,7 @@ class AutosubmitConfig(object): :return: if notifications :rtype: string """ - return self._conf_parser.get_option('mail', 'NOTIFICATIONS', 'false').lower() + return self._configWrapper.get_notifications() def get_remote_dependencies(self): """ @@ -1191,7 +715,7 @@ class AutosubmitConfig(object): :return: if remote dependencies :rtype: bool """ - return self._conf_parser.get_option('wrapper', 'DEPENDENCIES', 'false').lower() == 'true' + return self._configWrapper.get_remote_dependencies() def get_wrapper_type(self): """ @@ -1200,7 +724,7 @@ class AutosubmitConfig(object): :return: wrapper type (or none) :rtype: string """ - return self._conf_parser.get_option('wrapper', 'TYPE', 'None').lower() + return self._configWrapper.get_wrapper_type() def get_wrapper_jobs(self): """ @@ -1209,7 +733,7 @@ class AutosubmitConfig(object): :return: expression (or none) :rtype: string """ - return self._conf_parser.get_option('wrapper', 'JOBS_IN_WRAPPER', 'None') + return self._configWrapper.get_wrapper_jobs() def get_max_wrapped_jobs(self): """ @@ -1218,9 +742,7 @@ class AutosubmitConfig(object): :return: maximum number of jobs (or total jobs) :rtype: string """ - # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs())) - - return int(self._conf_parser.get_option('wrapper', 'MAX_WRAPPED', self.get_total_jobs())) + return self._configWrapper.get_max_wrapped_jobs() def get_wrapper_check_time(self): """ @@ -1229,7 +751,7 @@ class AutosubmitConfig(object): :return: wrapper check time :rtype: int """ - return int(self._conf_parser.get_option('wrapper', 'CHECK_TIME_WRAPPER', self.get_safetysleeptime())) + return self._configWrapper.get_wrapper_check_time() def get_wrapper_machinefiles(self): """ @@ -1238,7 +760,7 @@ class AutosubmitConfig(object): :return: machinefiles function to use :rtype: string """ - return self._conf_parser.get_option('wrapper', 'MACHINEFILES', '') + return self._configWrapper.get_wrapper_machinefiles() def get_wrapper_queue(self): """ @@ -1247,7 +769,7 @@ class AutosubmitConfig(object): :return: expression (or none) :rtype: string """ - return self._conf_parser.get_option('wrapper', 'QUEUE', "") + return self._configWrapper.get_wrapper_queue() def get_jobs_sections(self): """ @@ -1256,7 +778,7 @@ class AutosubmitConfig(object): :return: sections :rtype: list """ - return self._jobs_parser.sections() + return self._configWrapper.get_jobs_sections() def get_copy_remote_logs(self): """ @@ -1265,7 +787,7 @@ class AutosubmitConfig(object): :return: if logs local copy :rtype: bool """ - return self._conf_parser.get_option('storage', 'COPY_REMOTE_LOGS', 'true').lower() + return self._configWrapper.get_copy_remote_logs() def get_mails_to(self): """ @@ -1274,7 +796,7 @@ class AutosubmitConfig(object): :return: mail address :rtype: [str] """ - return [str(x) for x in self._conf_parser.get_option('mail', 'TO', '').split(' ')] + return self._configWrapper.get_mails_to() def get_communications_library(self): """ @@ -1283,7 +805,7 @@ class AutosubmitConfig(object): :return: communications library :rtype: str """ - return self._conf_parser.get_option('communications', 'API', 'paramiko').lower() + return self._configWrapper.get_communications_library() def get_storage_type(self): """ @@ -1292,45 +814,26 @@ class AutosubmitConfig(object): :return: communications library :rtype: str """ - return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower() + return self._configWrapper.get_storage_type() - @staticmethod - def is_valid_mail_address(mail_address): - if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): - return True - else: - return False + def is_valid_mail_address(self, mail_address): + #TODO: push to parent class as static method + return self._configWrapper.is_valid_mail_address(mail_address) def is_valid_communications_library(self): - library = self.get_communications_library() - return library in ['paramiko', 'saga'] + return self._configWrapper.is_valid_communications_library() def is_valid_storage_type(self): - storage_type = self.get_storage_type() - return storage_type in ['pkl', 'db'] + return self._configWrapper.is_valid_storage_type() def is_valid_jobs_in_wrapper(self): - expression = self.get_wrapper_jobs() - if expression != 'None': - parser = self._jobs_parser - sections = parser.sections() - for section in expression.split(" "): - if "&" in section: - for inner_section in section.split("&"): - if inner_section not in sections: - return False - elif section not in sections: - return False - return True + return self._configWrapper.is_valid_jobs_in_wrapper() def is_valid_git_repository(self): - origin_exists = self._exp_parser.check_exists('git', 'PROJECT_ORIGIN') - branch = self.get_git_project_branch() - commit = self.get_git_project_commit() - return origin_exists and (branch is not None or commit is not None) + return self._configWrapper.is_valid_git_repository() + - @staticmethod - def get_parser(parser_factory, file_path): + def get_parser(self, parser_factory, file_path): # type: (ConfigParserFactory, str) -> ConfigParser """ Gets parser for given file @@ -1341,13 +844,6 @@ class AutosubmitConfig(object): :return: parser :rtype: SafeConfigParser """ - parser = parser_factory.create_parser() - parser.optionxform = str - # proj is not required - # print(file_path) - if file_path.find('proj_') > 0: - parser.read(file_path) - else: - with open(file_path) as f: - parser.read(file_path) - return parser + # TODO: this was static method, check usages + + return self._configWrapper.get_parser(parser_factory, file_path) \ No newline at end of file diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py new file mode 100644 index 0000000000000000000000000000000000000000..ab42d38035ee6fb47cb2a1c3fcc74c500a19cdee --- /dev/null +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python + +# Copyright 2015 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . + +try: + # noinspection PyCompatibility + from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config +except ImportError: + # noinspection PyCompatibility + from configparser import SafeConfigParser + +import os +import re +import subprocess +import json +import logging + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.basicConfig import BasicConfig +from ..config.IConfigStrategy import IConfigStrategy + +logger = logging.getLogger('gunicorn.error') + +class ymlConfigStrategy(IConfigStrategy): + """ + Class to handle experiment configuration coming from file or database + + :param expid: experiment identifier + :type expid: str + """ + def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + logger.info("Creating AS4 Parser !!!!!") + self._conf_parser = Autosubmit4Config(expid) + self._conf_parser.reload(True) + + def jobs_parser(self): + logger.info("Not yet implemented") + pass + + #TODO: at the end of the implementation, check which methods can be moved to the top class for avoid code duplication + @property + def experiment_file(self): + """ + Returns experiment's config file name + """ + return self._exp_parser_file + + def platforms_parser(self): + logger.info("OBSOLOTED - Not yet implemented") + pass + + @property + def platforms_file(self): + """ + Returns experiment's platforms config file name + + :return: platforms config file's name + :rtype: str + """ + return self._platforms_parser_file + + @property + def project_file(self): + """ + Returns project's config file name + """ + return self._proj_parser_file + + @property + def jobs_file(self): + """ + Returns project's jobs file name + """ + return self._jobs_parser_file + + def get_full_config_as_dict(self): + """ + Returns full configuration as json object + """ + _conf = _exp = _platforms = _jobs = _proj = None + result = {} + + def get_data( parser): + """ + dictionary comprehension to get data from parser + """ + logger.info(parser) + #res = {sec: {option: parser[sec][option] for option in parser[sec].keys()} for sec in [ + # section for section in parser.keys()]} + #return res + return parser + + # res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ + # section for section in parser.sections()]} + + + # print(self._conf_parser) + #result["conf"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._conf_parser else None + #result["exp"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._exp_parser else None + result["platforms"] = self._conf_parser.platforms_data if self._conf_parser.platforms_data else None + #result["jobs"] = get_data( self._conf_parser.experiment_data["JOBS"]) if self._conf_parser.experiment_data["JOBS"] else None + #result["proj"] = get_data( self._conf_parser.experiment_data["CONF"] ) if self._proj_parser else None + return result + + + def get_full_config_as_json(self): + return self._conf_parser.get_full_config_as_json() + + def get_project_dir(self): + return self._conf_parser.get_project_dir() + + def get_queue(self, section): + return self._conf_parser.jobs_data[section].get('QUEUE', "") + + def get_job_platform(self, section): + pass + + def get_platform_queue(self, platform): + logger.info("get_platform_queue") + return self._conf_parser.platforms_data[platform]["QUEUE"] + + def get_platform_serial_queue(self, platform): + logger.info("get_platform_serial_queue") + return self._conf_parser.platforms_data[platform]["SERIAL_QUEUE"] + + def get_platform_project(self, platform): + logger.info("get_platform_project") + return self._conf_parser.platforms_data[platform]["PROJECT"] + + def get_platform_wallclock(self, platform): + logger.info("get_platform_wallclock") + return self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', "") + + def get_wallclock(self, section): + return self._conf_parser.jobs_data[section].get('WALLCLOCK', '') + + + def get_synchronize(self, section): + return self._conf_parser.get_synchronize(section) + + def get_processors(self, section): + return self._conf_parser.get_processors(section) + + def get_threads(self, section): + return self._conf_parser.get_threads(section) + + def get_tasks(self, section): + return self._conf_parser.get_tasks(section) + + def get_scratch_free_space(self, section): + return self._conf_parser.get_scratch_free_space(section) + + def get_memory(self, section): + return self._conf_parser.get_memory(section) + + def get_memory_per_task(self, section): + return self._conf_parser.get_memory_per_task(section) + + def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + :return: migrate user to + :rtype: str + """ + return self._conf_parser.get_migrate_user_to(section) + + def get_current_user(self, section): + return self._conf_parser.get_current_user(section) + + def get_current_project(self, section): + return self._conf_parser.get_current_project(section) + + def set_new_user(self, section, new_user): + self._conf_parser.set_new_user(section, new_user) + + def get_migrate_project_to(self, section): + return self._conf_parser.get_migrate_project_to(section) + + def set_new_project(self, section, new_project): + self._conf_parser.set_new_project(section, new_project) + + def get_custom_directives(self, section): + """ + Gets custom directives needed for the given job type + :param section: job type + :type section: str + :return: custom directives needed + :rtype: str + """ + return str(self._conf_parser.jobs_data.get(section, {}).get('CUSTOM_DIRECTIVES', "")) + + + def check_conf_files(self): + return self._conf_parser.check_conf_files() + + def check_autosubmit_conf(self): + return self._conf_parser.check_autosubmit_conf() + + def check_platforms_conf(self): + return self._conf_parser.check_platforms_conf() + + def check_jobs_conf(self): + return self._conf_parser.check_jobs_conf() + + def check_expdef_conf(self): + return self._conf_parser.check_expdef_conf() + + def check_proj(self): + return self._conf_parser.check_proj() + + def check_wrapper_conf(self): + self._conf_parser.check_wrapper_conf() + + def reload(self): + self._conf_parser.reload() + + def load_parameters(self): + return self._conf_parser.load_parameters() + + def load_project_parameters(self): + return self._conf_parser.load_project_parameters() + + def set_expid(self, exp_id): + self._conf_parser.set_expid(exp_id) + + def get_project_type(self): + return self._conf_parser.get_project_type() + + def get_file_project_conf(self): + return self._conf_parser.get_file_project_conf() + + def get_file_jobs_conf(self): + return self._conf_parser.get_file_jobs_conf() + + def get_git_project_origin(self): + return self._conf_parser.get_git_project_origin() + + def get_git_project_branch(self): + return self._conf_parser.get_git_project_branch() + + def get_git_project_commit(self): + return self._conf_parser.get_git_project_commit() + + def get_submodules_list(self): + return self._conf_parser.get_submodules_list() + + def get_project_destination(self): + return self._conf_parser.get_project_destination() + + def set_git_project_commit(self, as_conf): + self._conf_parser.set_git_project_commit(as_conf) + + def get_svn_project_url(self): + return self._conf_parser.get_svn_project_url() + + def get_svn_project_revision(self): + return self._conf_parser.get_svn_project_revision() + + def get_local_project_path(self): + return self._conf_parser.get_local_project_path() + + def get_date_list(self): + return self._conf_parser.get_date_list() + + def get_num_chunks(self): + return self._conf_parser.get_num_chunks() + + def get_chunk_ini(self, default=1): + return self._conf_parser.get_chunk_ini(default) + + def get_chunk_size_unit(self): + """ + Unit for the chunk length + + :return: Unit for the chunk length Options: {hour, day, month, year} + :rtype: str + """ + return self._conf_parser.get_chunk_size_unit() + + + def get_chunk_size(self, default=1): + try: + chunk_size = self._conf_parser.get_chunk_size(default) + except Exception as exp: + print(exp) + chunk_size = '' + pass + if chunk_size == '': + return default + return int(chunk_size) + + def get_member_list(self, run_only=False): + #return self._conf_parser.get_member_list(run_only) + """ + Returns members list from experiment's config file + + :return: experiment's members + :rtype: list + """ + return self._conf_parser.get_member_list(run_only) + + def get_rerun(self): + return self._conf_parser.get_rerun() + + def get_chunk_list(self): + return self._conf_parser.get_chunk_list() + + def get_platform(self): + return self._conf_parser.get_platform() + + def set_platform(self, hpc): + self._conf_parser.set_platform(hpc) + + def set_version(self, autosubmit_version): + self._conf_parser.set_version(autosubmit_version) + + def get_version(self): + return self._conf_parser.get_version() + + def get_total_jobs(self): + return self._conf_parser.get_total_jobs() + + def get_max_wallclock(self): + return self._conf_parser.get_max_wallclock() + + def get_max_processors(self): + return self._conf_parser.get_max_processors() + + def get_max_waiting_jobs(self): + return self._conf_parser.get_max_waiting_jobs() + + def get_default_job_type(self): + return self._conf_parser.get_default_job_type() + + def get_safetysleeptime(self): + return self._conf_parser.get_safetysleeptime() + + def set_safetysleeptime(self, sleep_time): + self._conf_parser.set_safetysleeptime(sleep_time) + + def get_retrials(self): + return self._conf_parser.get_retrials() + + def get_notifications(self): + return self._conf_parser.get_notifications() + + def get_remote_dependencies(self): + return self._conf_parser.get_remote_dependencies() + + def get_wrapper_type(self): + return self._conf_parser.get_wrapper_type() + + def get_wrapper_jobs(self): + return self._conf_parser.get_wrapper_jobs() + + def get_max_wrapped_jobs(self): + return self._conf_parser.get_max_wrapped_jobs() + + def get_wrapper_check_time(self): + return self._conf_parser.get_wrapper_check_time() + + def get_wrapper_machinefiles(self): + return self._conf_parser.get_wrapper_machinefiles() + + def get_wrapper_queue(self): + return self._conf_parser.get_wrapper_queue() + + def get_jobs_sections(self): + return self._conf_parser.get_jobs_sections() + + def get_copy_remote_logs(self): + return self._conf_parser.get_copy_remote_logs() + + def get_mails_to(self): + return self._conf_parser.get_mails_to() + + def get_communications_library(self): + return self._conf_parser.get_communications_library() + + def get_storage_type(self): + return self._conf_parser.get_storage_type() + + @staticmethod + def is_valid_mail_address(mail_address): + return self._conf_parser.is_valid_mail_address(mail_address) + + @classmethod + def is_valid_communications_library(self): + return self._conf_parser.is_valid_communications_library() + + @classmethod + def is_valid_storage_type(self): + return self._conf_parser.is_valid_storage_type() + + def is_valid_jobs_in_wrapper(self): + self._conf_parser.is_valid_jobs_in_wrapper() + + def is_valid_git_repository(self): + self._conf_parser.is_valid_git_repository() + + @staticmethod + def get_parser(parser_factory, file_path): + return Autosubmit4Config.get_parser(parser_factory, file_path) + + diff --git a/autosubmit_api/database/db_common.py b/autosubmit_api/database/db_common.py index bde23d0eca9c3f62c0745b9dbdd202cb7209eea3..e58ef8cbd41f21e2629190b25a889e8bee91f288 100644 --- a/autosubmit_api/database/db_common.py +++ b/autosubmit_api/database/db_common.py @@ -365,8 +365,6 @@ def search_experiment_by_id(searchString, typeExp=None, onlyActive=None, owner=N wrapper = autosubmit_config_facade.get_wrapper_type() last_modified_pkl_datetime = autosubmit_config_facade.get_pkl_last_modified_time_as_datetime() except Exception as exp: - version = "Unknown" - wrapper = None last_modified_pkl_datetime = None pass status = experiment_status.get(expid, "NOT RUNNING") @@ -454,8 +452,6 @@ def get_current_running_exp(): wrapper = autosubmit_config_facade.get_wrapper_type() last_modified_pkl_datetime = autosubmit_config_facade.get_pkl_last_modified_time_as_datetime() except Exception as exp: - version = "Unknown" - wrapper = None last_modified_pkl_datetime = None pass if (expid in experiment_times): diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index dc5584ed310ad30740a8a119ffa3d64ad84d7b6d..b75d58aeda54e7e11270cbebc480008dca68b9b2 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -28,6 +28,8 @@ import datetime import json import multiprocessing import subprocess +import logging + from collections import deque from ..autosubmit_legacy.autosubmit import Autosubmit from ..database import db_common as db_common @@ -1121,6 +1123,7 @@ def get_auto_conf_data(expid): max_wrapped = as_conf.get_max_wrapped_jobs() return (wrapper_type, max_wrapped) except Exception as ex: + logger.info(traceback.format_exc()) print(("Couldn't retrieve conf data (wrapper info) from {0}. Exception {1}.".format(expid, str(ex)))) return ("None", 0) @@ -1382,6 +1385,7 @@ def get_current_configuration_by_expid(expid, valid_user, log): warning = True warning_message = "The filesystem system configuration can't be retrieved because '{}'".format( exp) + logger.info(traceback.format_exc()) currentFileSystemConfig["contains_nones"] = True log.info(warning_message) pass diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 0000000000000000000000000000000000000000..16206472223cff374eb87ed7c82e85bae5be354e --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,2 @@ +conda env create -f conda_environment.yml +conda activate autosubmit_api \ No newline at end of file diff --git a/deployment/conda_environment.yaml b/deployment/conda_environment.yaml index 61c5ed8a954088ed6904be924789cd353f0f8d07..cc9fedcb9593e92f5f4b5a61de0e3f061a61a334 100644 --- a/deployment/conda_environment.yaml +++ b/deployment/conda_environment.yaml @@ -75,3 +75,4 @@ dependencies: - urllib3==1.26.14 - werkzeug==2.2.2 - zipp==3.12.0 + - autosubmitconfigparser==1.0.27 diff --git a/deployment/update_launch_autosubmit_API_conda.sh b/deployment/update_launch_autosubmit_API_conda.sh new file mode 100644 index 0000000000000000000000000000000000000000..f36f8f6b0f655f5fdfe44e268ab04cd97570cc93 --- /dev/null +++ b/deployment/update_launch_autosubmit_API_conda.sh @@ -0,0 +1,38 @@ +#!/bin/bash +. ~/.bashrc + +LOG_PATH=${PLOG} +UPDATE=false + +while getopts ":e:u" opt; do + case "$opt" in + u) UPDATE=true; + ;; + esac +done + + +# Stop current instance of unicorn +pstree -ap | grep gunicorn | awk -F',' '{print $2}' | awk -F' ' '{print $1}' | head -n 1 | xargs -r kill +pstree -ap | grep gunicorn + +# activate conda environment +conda activate autosubmit_api + +# if update to a new version we install it from pip +if [ "${UPDATE}" = true ]; then + pip install autosubmit_api --upgrade +fi + +# prepare to launch +echo "Set SECRET KEY" +export SECRET_KEY='c&X=