From 7c7076a797c72d88acb9a304e7e91d8215e9967f Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Fri, 10 Mar 2023 16:03:42 -0300 Subject: [PATCH 01/25] added import to use the new library for managing the config files - #12 --- autosubmit_api/config/basicConfig.py | 1 + autosubmit_api/config/config_common.py | 1 + deployment/README.md | 2 + .../update_launch_autosubmit_API_conda.sh | 38 +++++++++++++++++++ 4 files changed, 42 insertions(+) create mode 100644 deployment/README.md create mode 100644 deployment/update_launch_autosubmit_API_conda.sh diff --git a/autosubmit_api/config/basicConfig.py b/autosubmit_api/config/basicConfig.py index 3c436866..38a25576 100644 --- a/autosubmit_api/config/basicConfig.py +++ b/autosubmit_api/config/basicConfig.py @@ -19,6 +19,7 @@ try: # noinspection PyCompatibility from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index c7345371..3a9ab05c 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -19,6 +19,7 @@ try: # noinspection PyCompatibility from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser diff --git a/deployment/README.md b/deployment/README.md new file mode 100644 index 00000000..16206472 --- /dev/null +++ b/deployment/README.md @@ -0,0 +1,2 @@ +conda env create -f conda_environment.yml +conda activate autosubmit_api \ No newline at end of file diff --git a/deployment/update_launch_autosubmit_API_conda.sh b/deployment/update_launch_autosubmit_API_conda.sh new file mode 100644 index 00000000..f36f8f6b --- /dev/null +++ b/deployment/update_launch_autosubmit_API_conda.sh @@ -0,0 +1,38 @@ +#!/bin/bash +. ~/.bashrc + +LOG_PATH=${PLOG} +UPDATE=false + +while getopts ":e:u" opt; do + case "$opt" in + u) UPDATE=true; + ;; + esac +done + + +# Stop current instance of unicorn +pstree -ap | grep gunicorn | awk -F',' '{print $2}' | awk -F' ' '{print $1}' | head -n 1 | xargs -r kill +pstree -ap | grep gunicorn + +# activate conda environment +conda activate autosubmit_api + +# if update to a new version we install it from pip +if [ "${UPDATE}" = true ]; then + pip install autosubmit_api --upgrade +fi + +# prepare to launch +echo "Set SECRET KEY" +export SECRET_KEY='c&X= Date: Wed, 26 Apr 2023 15:25:49 +0200 Subject: [PATCH 02/25] Initial test on the library that will provide parsing for the yaml files of AS4 experiments - #12 --- autosubmit_api/config/config_common.py | 42 ++++++++++++++++++-------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 3a9ab05c..1d166d2e 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -19,14 +19,18 @@ try: # noinspection PyCompatibility from configparser import SafeConfigParser - from autosubmitconfigparser.config.configcommon import AutosubmitConfig + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser + + import os import re import subprocess import json +import logging +import locale from pyparsing import nestedExpr from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser @@ -34,6 +38,7 @@ from bscearth.utils.date import parse_date from bscearth.utils.log import Log from ..config.basicConfig import BasicConfig +logger = logging.getLogger('gunicorn.error') class AutosubmitConfig(object): """ @@ -691,20 +696,31 @@ class AutosubmitConfig(object): """ Creates parser objects for configuration files """ - if not os.path.exists(self._conf_parser_file): raise IOError("Required file not found {0}".format(self._conf_parser_file)) - if not os.path.exists(self._platforms_parser_file): raise IOError("Required file not found {0}".format(self._platforms_parser_file)) - if not os.path.exists(self._jobs_parser_file): raise IOError("Required file not found {0}".format(self._jobs_parser_file)) - if not os.path.exists(self._exp_parser_file): raise IOError("Required file not found {0}".format(self._exp_parser_file)) - self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file) - self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) - self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file) - self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file) - if self._proj_parser_file == '': - self._proj_parser = None + logger.info(str(self._conf_parser_file)) + is_as4 = self._conf_parser_file.endswith('.yml') or self._conf_parser_file.endswith('.yaml') + if is_as4: + logger.info("LOADING Configuration - for AS4 !!!!!") + as_conf = Autosubmit4Config(self.expid) + as_conf.reload(True) + # log the content + logger.info(as_conf.platforms_data) else: - self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file) + logger.info("LOADING Configuration - for AS3 !!!!!") + if not os.path.exists(self._conf_parser_file): raise IOError( "Required file not found {0}".format(self._conf_parser_file)) + if not os.path.exists(self._platforms_parser_file): raise IOError( "Required file not found {0}".format(self._platforms_parser_file)) + if not os.path.exists(self._jobs_parser_file): raise IOError( "Required file not found {0}".format(self._jobs_parser_file)) + if not os.path.exists(self._exp_parser_file): raise IOError( "Required file not found {0}".format(self._exp_parser_file)) + self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file) + self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) + self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file) + self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file) + if self._proj_parser_file == '': + self._proj_parser = None + else: + self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file) + - def load_parameters(self): +def load_parameters(self): """ Load parameters from experiment and autosubmit config files. If experiment's type is not none, also load parameters from model's config file -- GitLab From 252ddef5efd1ba65f17b3492b0eaaa22fdf871ce Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Wed, 26 Apr 2023 17:00:12 +0200 Subject: [PATCH 03/25] initial testing changes, for getting the data from yaml files - #12 --- autosubmit_api/autosubmit_legacy/job/job_dict.py | 3 +++ autosubmit_api/config/config_common.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/autosubmit_api/autosubmit_legacy/job/job_dict.py b/autosubmit_api/autosubmit_legacy/job/job_dict.py index 0f833cc3..f67f7748 100644 --- a/autosubmit_api/autosubmit_legacy/job/job_dict.py +++ b/autosubmit_api/autosubmit_legacy/job/job_dict.py @@ -371,6 +371,9 @@ class DicJobs: :param default: value to return if not defined in configuration file :type default: object """ + #todo: here should be implemented some of the changes for supporting yaml files + + if self._parser.has_option(section, option): return self._parser.get(section, option) else: diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 1d166d2e..1fefca0f 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -701,9 +701,17 @@ class AutosubmitConfig(object): if is_as4: logger.info("LOADING Configuration - for AS4 !!!!!") as_conf = Autosubmit4Config(self.expid) + # this loads all configurations as_conf.reload(True) # log the content logger.info(as_conf.platforms_data) + logger.info(as_conf.experiment_data) + logger.info(as_conf.jobs_data) + # testing getting host property from platform config file + # todo: here should be implemented some of the changes for supporting yaml files + logger.info(as_conf.platforms_data["MARENOSTRUM4"]["HOST"]) + #sample + #self.load_config_file(non_minimal_conf, Path(filename))) else: logger.info("LOADING Configuration - for AS3 !!!!!") if not os.path.exists(self._conf_parser_file): raise IOError( "Required file not found {0}".format(self._conf_parser_file)) -- GitLab From bc475b7403ce6d90cf968a3e7fd91e45d446dacf Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Tue, 2 May 2023 17:12:13 +0200 Subject: [PATCH 04/25] initial set of changes for testing the AS4 autosubmitconfigparser for platform config file - #12 --- autosubmit_api/config/config_common.py | 91 ++++++++++++++------------ 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 1fefca0f..2c376d2d 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -178,8 +178,7 @@ class AutosubmitConfig(object): self._conf_parser) if self._conf_parser else None result["exp"] = get_data( self._exp_parser) if self._exp_parser else None - result["platforms"] = get_data( - self._platforms_parser) if self._platforms_parser else None + result["platforms"] = self._platforms_data if self._platforms_data else None result["jobs"] = get_data( self._jobs_parser) if self._jobs_parser else None result["proj"] = get_data( @@ -222,16 +221,20 @@ class AutosubmitConfig(object): return self._jobs_parser.get_option(section, 'PLATFORM', '') def get_platform_queue(self, platform): - return self._platforms_parser.get_option(platform, 'QUEUE', '') + return self._platforms_data[platform]["QUEUE"] + # return self._platforms_parser.get_option(platform, 'QUEUE', '') def get_platform_serial_queue(self, platform): - return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '') + return self._platforms_data[platform]["SERIAL_QUEUE"] + #return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '') def get_platform_project(self, platform): - return self._platforms_parser.get_option(platform, "PROJECT", "") + return self._platforms_data[platform]["PROJECT"] + #return self._platforms_parser.get_option(platform, "PROJECT", "") def get_platform_wallclock(self, platform): - return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '') + return self._platforms_data[platform]["MAX_WALLCLOCK"] + #return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '') def get_wallclock(self, section): """ @@ -320,7 +323,8 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + #return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + return str(self._platforms_data[section]["USER_TO"]).lower() def get_current_user(self, section): """ @@ -329,7 +333,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return self._platforms_parser.get_option(section, 'USER', '').lower() + return str(self._platforms_data[section]["USER"]).lower() def get_current_project(self, section): """ @@ -338,7 +342,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return self._platforms_parser.get_option(section, 'PROJECT', '').lower() + return str(self._platforms_data[section]["USER"]).lower() def set_new_user(self, section, new_user): """ @@ -376,7 +380,9 @@ class AutosubmitConfig(object): :return: migrate project to :rtype: str """ - return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() + #return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() + return str(self._platforms_data[section]["PROJECT_TO"]).lower() + def set_new_project(self, section, new_project): """ @@ -490,35 +496,27 @@ class AutosubmitConfig(object): :rtype: bool """ result = True - if len(self._platforms_parser.sections()) == 0: + # self._platforms_data.keys() == 0 + #if len(self._platforms_parser.sections()) == 0: + if len(self._platforms_data.keys())== 0: Log.warning("No remote platforms configured") - if len(self._platforms_parser.sections()) != len(set(self._platforms_parser.sections())): + if len(self._platforms_data.keys()) != len(set(self._platforms_data.keys())): Log.error('There are repeated platforms names') - for section in self._platforms_parser.sections(): - result = result and self._platforms_parser.check_exists( - section, 'TYPE') - platform_type = self._platforms_parser.get_option( - section, 'TYPE', '').lower() + for section in self._platforms_data.keys(): + # get the current platform for checking their values + platform = self._platforms_data[section] + + result = result and platform.get('TYPE') is not None + platform_type = platform.get('TYPE').lower() if platform_type != 'ps': - result = result and self._platforms_parser.check_exists( - section, 'PROJECT') - result = result and self._platforms_parser.check_exists( - section, 'USER') - - result = result and self._platforms_parser.check_exists( - section, 'HOST') - result = result and self._platforms_parser.check_exists( - section, 'SCRATCH_DIR') - result = result and self._platforms_parser.check_is_boolean(section, - 'ADD_PROJECT_TO_HOST', False) - result = result and self._platforms_parser.check_is_boolean( - section, 'TEST_SUITE', False) - result = result and self._platforms_parser.check_is_int(section, 'MAX_WAITING_JOBS', - False) - result = result and self._platforms_parser.check_is_int( - section, 'TOTAL_JOBS', False) + result = result and platform.get('PROJECT') is not None + result = result and platform.get('USER') is not None + result = result and platform.get('HOST') is not None + result = result and platform.get('SCRATCH_DIR') is not None + result = result and platform.get('ADD_PROJECT_TO_HOST').lower() in ['false', 'true'] + result = result and platform.get('TEST_SUITE').lower() in ['false', 'true'] if not result: Log.critical("{0} is not a valid config file".format( @@ -540,7 +538,8 @@ class AutosubmitConfig(object): result = True parser = self._jobs_parser sections = parser.sections() - platforms = self._platforms_parser.sections() + #platforms = self._platforms_parser.sections() + platforms = self._platforms_data.keys() platforms.append('LOCAL') possible_exception = "" if len(sections) == 0: @@ -682,14 +681,14 @@ class AutosubmitConfig(object): Log.error( "There are sections in JOBS_IN_WRAPPER that are not defined in your jobs.conf file") + platform = self._platforms_data[self.get_platform()] if 'horizontal' in self.get_wrapper_type(): - result = result and self._platforms_parser.check_exists( - self.get_platform(), 'PROCESSORS_PER_NODE') - result = result and self._platforms_parser.check_exists( - self.get_platform(), 'MAX_PROCESSORS') + result = result and platform.get('PROCESSORS_PER_NODE') is not None + result = result and platform.get('MAX_PROCESSORS') is not None + if 'vertical' in self.get_wrapper_type(): - result = result and self._platforms_parser.check_exists( - self.get_platform(), 'MAX_WALLCLOCK') + result = result and platform.get('MAX_WALLCLOCK') is not None + return result def reload(self): @@ -698,6 +697,10 @@ class AutosubmitConfig(object): """ logger.info(str(self._conf_parser_file)) is_as4 = self._conf_parser_file.endswith('.yml') or self._conf_parser_file.endswith('.yaml') + + #todo: remove internal parser object and work directly with the data loaded + # try first with only one conf type so we check how it goes for both + if is_as4: logger.info("LOADING Configuration - for AS4 !!!!!") as_conf = Autosubmit4Config(self.expid) @@ -707,6 +710,8 @@ class AutosubmitConfig(object): logger.info(as_conf.platforms_data) logger.info(as_conf.experiment_data) logger.info(as_conf.jobs_data) + + self._platforms_data = as_conf.platforms_data # testing getting host property from platform config file # todo: here should be implemented some of the changes for supporting yaml files logger.info(as_conf.platforms_data["MARENOSTRUM4"]["HOST"]) @@ -714,12 +719,14 @@ class AutosubmitConfig(object): #self.load_config_file(non_minimal_conf, Path(filename))) else: logger.info("LOADING Configuration - for AS3 !!!!!") + # checking existence of config files if not os.path.exists(self._conf_parser_file): raise IOError( "Required file not found {0}".format(self._conf_parser_file)) if not os.path.exists(self._platforms_parser_file): raise IOError( "Required file not found {0}".format(self._platforms_parser_file)) if not os.path.exists(self._jobs_parser_file): raise IOError( "Required file not found {0}".format(self._jobs_parser_file)) if not os.path.exists(self._exp_parser_file): raise IOError( "Required file not found {0}".format(self._exp_parser_file)) + self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file) - self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) + #self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file) self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file) if self._proj_parser_file == '': -- GitLab From a810c88fdba4b87fa9d5b4941c1e10707789645f Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Tue, 2 May 2023 17:22:48 +0200 Subject: [PATCH 05/25] fix indentation error - #12 --- autosubmit_api/config/config_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 2c376d2d..d4e275b2 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -735,7 +735,7 @@ class AutosubmitConfig(object): self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file) -def load_parameters(self): + def load_parameters(self): """ Load parameters from experiment and autosubmit config files. If experiment's type is not none, also load parameters from model's config file -- GitLab From 6ebd7744d9960d4dd50d61fb12240522e8b405a5 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Tue, 2 May 2023 17:36:17 +0200 Subject: [PATCH 06/25] add AS4 config parser to full load method - #12 --- autosubmit_api/config/config_common.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index d4e275b2..92bab3a7 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -166,14 +166,20 @@ class AutosubmitConfig(object): _conf = _exp = _platforms = _jobs = _proj = None result = {} - def get_data(parser): - """ - dictionary comprehension to get data from parser - """ - res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ - section for section in parser.sections()]} - return res + #def get_data(parser): + # """ + # dictionary comprehension to get data from parser + # """ + # res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ + # section for section in parser.sections()]} + # return res + + as_conf = Autosubmit4Config(self.expid) + # this loads all configurations + as_conf.reload(True) + self._platforms_data = as_conf.platforms_data # print(self._conf_parser) + result["conf"] = get_data( self._conf_parser) if self._conf_parser else None result["exp"] = get_data( -- GitLab From e5d544ceb89448225b2c6c7432b2cc1f020a8ecc Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Thu, 4 May 2023 10:06:47 +0200 Subject: [PATCH 07/25] constructor fixes - #12 --- autosubmit_api/config/config_common.py | 9 ++++++++- autosubmit_api/experiment/common_requests.py | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 92bab3a7..af99fe64 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -78,10 +78,17 @@ class AutosubmitConfig(object): self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + extension) if os.path.exists(self._platforms_parser_file) == False: if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") + #self.__init__(expid, basic_config, parser_factory, ".conf") + # todo: this should be moved at the beginning + as_conf = Autosubmit4Config(self.expid) + # this loads all configurations + as_conf.reload(True) + self._platforms_data = as_conf.platforms_data elif extension == ".conf": return None + + self._jobs_parser = None # type: ConfigParser self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "jobs_" + expid + extension) if os.path.exists(self._jobs_parser_file) == False: diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index dc5584ed..eb0198f0 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -1382,6 +1382,7 @@ def get_current_configuration_by_expid(expid, valid_user, log): warning = True warning_message = "The filesystem system configuration can't be retrieved because '{}'".format( exp) + log.info((traceback.format_exc())) currentFileSystemConfig["contains_nones"] = True log.info(warning_message) pass -- GitLab From 9aedb30f947f2ab1d1216af38deb77c048d337bd Mon Sep 17 00:00:00 2001 From: jberlin Date: Thu, 4 May 2023 13:04:48 +0200 Subject: [PATCH 08/25] add AS4 config parser class to keep original interface in config_common, add also the new library to the Conda recipe - #12 --- autosubmit_api/config/config_common.py | 2 + autosubmit_api/config/ymlConfigParser.py | 250 +++++++++++++++++++++++ deployment/conda_environment.yaml | 1 + 3 files changed, 253 insertions(+) create mode 100644 autosubmit_api/config/ymlConfigParser.py diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index af99fe64..e62fa86b 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -1386,6 +1386,8 @@ class AutosubmitConfig(object): :return: parser :rtype: SafeConfigParser """ + # todo: add here the yml parser depending of the file type + parser = parser_factory.create_parser() parser.optionxform = str # proj is not required diff --git a/autosubmit_api/config/ymlConfigParser.py b/autosubmit_api/config/ymlConfigParser.py new file mode 100644 index 00000000..dc443590 --- /dev/null +++ b/autosubmit_api/config/ymlConfigParser.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python + +# Copyright 2023 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit , Autosubmot API. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . + +""" +Module containing functions to manage autosubmit 4 config files that are in yml format. +""" +import os +import re +import subprocess +import json +import logging +import locale + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.basicConfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config + +logger = logging.getLogger('gunicorn.error') + + +class ymlConfigParser(object): + + def __init__(self, conf_type): + logger.info("Creating AS4 Parser !!!!!") + self.conf_parser = Autosubmit4Config(self.expid) + self.conf_parser.reload(True) + # set the type of config file (platforms, jobs, exp, etc) + self.conf_type = conf_type + # set the data object + self.conf_data = self.conf_parser[conf_type] + + + def has_option(self,section,option): + return self.conf_data[section].get('TYPE') is not None + + def get_option(self, section, option, default=None): + """ + Gets an option from given parser + + :param self: parser to use + :type self: SafeConfigParser + :param section: section that contains the option + :type section: str + :param option: option to get + :type option: str + :param default: value to be returned if option is not present + :type default: object + :return: option value + :rtype: str + """ + if self.has_option(section, option): + return self.get(section, option) + else: + return default + + + def get_bool_option(self, section, option, default): + """ + Gets a boolean option from given parser + + :param self: parser to use + :type self: SafeConfigParser + :param section: section that contains the option + :type section: str + :param option: option to get + :type option: str + :param default: value to be returned if option is not present + :type default: bool + :return: option value + :rtype: bool + """ + if self.has_option(section, option): + return self.get(section, option).lower().strip() == 'true' + else: + return default + + def get_int_option(self, section, option, default=0): + """ + Gets an integer option + + :param section: section that contains the option + :type section: str + :param option: option to get + :type option: str + :param default: value to be returned if option is not present + :type default: int + :return: option value + :rtype: int + """ + return int(self.get_option(section, option, default)) + + def get_float_option(self, section, option, default=0.0): + """ + Gets a float option + + :param section: section that contains the option + :type section: str + :param option: option to get + :type option: str + :param default: value to be returned if option is not present + :type default: float + :return: option value + :rtype: float + """ + return float(self.get_option(section, option, default)) + + def get_choice_option(self, section, option, choices, default=None, ignore_case=False): + """ + Gets a boolean option + + :param ignore_case: if True, + :param choices: available choices + :type choices: [str] + :param section: section that contains the option + :type section: str + :param option: option to get + :type option: str + :param default: value to be returned if option is not present + :type default: str + :return: option value + :rtype: str + """ + + if self.has_option(section, option): + value = self.get_option(section, option, choices[0]) + if ignore_case: + value = value.lower() + for choice in choices: + if value == choice.lower(): + return choice + else: + if value in choices: + return value + raise ConfigError('Value {2} in option {0} in section {1} is not a valid choice'.format(option, section, + value)) + else: + if default: + return default + raise ConfigError('Option {0} in section {1} is not present and there is not a default value'.format(option, + section)) + + def check_exists(self, section, option): + """ + Checks if an option exists in given parser + + :param section: section that contains the option + :type section: str + :param option: option to check + :type option: str + :return: True if option exists, False otherwise + :rtype: bool + """ + if self.has_option(section, option): + return True + else: + Log.error('Option {0} in section {1} not found'.format(option, section)) + return False + + def check_is_boolean(self, section, option, must_exist): + """ + Checks if an option is a boolean value in given parser + + :param section: section that contains the option + :type section: str + :param option: option to check + :type option: str + :param must_exist: if True, option must exist + :type must_exist: bool + :return: True if option value is boolean, False otherwise + :rtype: bool + """ + if must_exist and not self.check_exists(section, option): + Log.error('Option {0} in section {1} must exist'.format(option, section)) + return False + if self.get_option(section, option, 'false').lower() not in ['false', 'true']: + Log.error('Option {0} in section {1} must be true or false'.format(option, section)) + return False + return True + + def check_is_choice(self, section, option, must_exist, choices): + """ + Checks if an option is a valid choice in given parser + + :param self: parser to use + :type self: SafeConfigParser + :param section: section that contains the option + :type section: str + :param option: option to check + :type option: str + :param must_exist: if True, option must exist + :type must_exist: bool + :param choices: valid choices + :type choices: list + :return: True if option value is a valid choice, False otherwise + :rtype: bool + """ + if must_exist and not self.check_exists(section, option): + return False + value = self.get_option(section, option, choices[0]) + if value not in choices: + Log.error('Value {2} in option {0} in section {1} is not a valid choice'.format(option, section, value)) + return False + return True + + def check_is_int(self, section, option, must_exist): + """ + Checks if an option is an integer value in given parser + + :param self: parser to use + :type self: SafeConfigParser + :param section: section that contains the option + :type section: str + :param option: option to check + :type option: str + :param must_exist: if True, option must exist + :type must_exist: bool + :return: True if option value is integer, False otherwise + :rtype: bool + """ + if must_exist and not self.check_exists(section, option): + return False + value = self.get_option(section, option, '1') + try: + int(value) + except ValueError: + Log.error('Option {0} in section {1} is not valid an integer'.format(option, section)) + return False + return True + + + diff --git a/deployment/conda_environment.yaml b/deployment/conda_environment.yaml index 61c5ed8a..cc9fedcb 100644 --- a/deployment/conda_environment.yaml +++ b/deployment/conda_environment.yaml @@ -75,3 +75,4 @@ dependencies: - urllib3==1.26.14 - werkzeug==2.2.2 - zipp==3.12.0 + - autosubmitconfigparser==1.0.27 -- GitLab From b2f46bedf42bdf63b7586afb45c9697e0b22780c Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Thu, 4 May 2023 17:36:08 +0200 Subject: [PATCH 09/25] add the new library to the config_common file - #12 --- autosubmit_api/config/config_common.py | 22 ++++++++++++++-------- autosubmit_api/config/ymlConfigParser.py | 8 +++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index e62fa86b..f91eea99 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -37,6 +37,7 @@ from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser from bscearth.utils.date import parse_date from bscearth.utils.log import Log from ..config.basicConfig import BasicConfig +from ..config.ymlConfigParser import ymlConfigParser logger = logging.getLogger('gunicorn.error') @@ -1387,14 +1388,19 @@ class AutosubmitConfig(object): :rtype: SafeConfigParser """ # todo: add here the yml parser depending of the file type - - parser = parser_factory.create_parser() - parser.optionxform = str - # proj is not required - # print(file_path) - if file_path.find('proj_') > 0: - parser.read(file_path) + is_as4 = file_path.endswith('.yml') or file_path.endswith('.yaml') + if is_as4: + # load as4 parser + parser = ymlConfigParser() else: - with open(file_path) as f: + # load as3 parser + parser = parser_factory.create_parser() + parser.optionxform = str + # proj is not required + # print(file_path) + if file_path.find('proj_') > 0: parser.read(file_path) + else: + with open(file_path) as f: + parser.read(file_path) return parser diff --git a/autosubmit_api/config/ymlConfigParser.py b/autosubmit_api/config/ymlConfigParser.py index dc443590..2df801d5 100644 --- a/autosubmit_api/config/ymlConfigParser.py +++ b/autosubmit_api/config/ymlConfigParser.py @@ -19,6 +19,12 @@ """ Module containing functions to manage autosubmit 4 config files that are in yml format. +Sample files for a given experiment: + /esarchive/autosubmit/EXPID/conf/autosubmit_EXPID.yml + /esarchive/autosubmit/EXPID/conf/expdef_EXPID.yml + /esarchive/autosubmit/EXPID/conf/jobs_EXPID.yml + /esarchive/autosubmit/EXPID/conf/platforms_EXPID.yml + /esarchive/autosubmit/EXPID/conf/proj_EXPID.yml """ import os import re @@ -68,7 +74,7 @@ class ymlConfigParser(object): :rtype: str """ if self.has_option(section, option): - return self.get(section, option) + return self.conf_data[section].get(option) else: return default -- GitLab From 80422595f2e74748cf844713ea9995233a4d9a3a Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Tue, 9 May 2023 17:38:52 +0200 Subject: [PATCH 10/25] Define initial files to achieve the implementation of the strategy pattern for Autosubmit config files - #12 --- autosubmit_api/config/IConfigStrategy.py | 826 ++++++++++++++++++++ autosubmit_api/config/confConfigStrategy.py | 66 ++ autosubmit_api/config/ymlConfigStrategy.py | 65 ++ 3 files changed, 957 insertions(+) create mode 100644 autosubmit_api/config/IConfigStrategy.py create mode 100644 autosubmit_api/config/confConfigStrategy.py create mode 100644 autosubmit_api/config/ymlConfigStrategy.py diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py new file mode 100644 index 00000000..062e966c --- /dev/null +++ b/autosubmit_api/config/IConfigStrategy.py @@ -0,0 +1,826 @@ +#!/usr/bin/env python + +# Copyright 2015 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . + +try: + # noinspection PyCompatibility + from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config +except ImportError: + # noinspection PyCompatibility + from configparser import SafeConfigParser + + +import os +import re +import subprocess +import json +import logging +import locale + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.basicConfig import BasicConfig +from ..config.ymlConfigParser import ymlConfigParser +from abc import ABCMeta, abstractmethod + + +class IConfigStrategy: + """ + Public interface for Autosubmit config files + + :param expid: experiment identifier + :type expid: str + """ + + __metaclass__ = ABCMeta + + @abstractmethod + def jobs_parser(self): + raise NotImplementedError + + @abstractmethod + def experiment_file(self): + """ + Returns experiment's config file name + """ + return self._exp_parser_file + + @abstractmethod + def platforms_parser(self): + """ + Returns experiment's platforms parser object + + :return: platforms config parser object + :rtype: SafeConfigParser + """ + pass + + @abstractmethod + def platforms_file(self): + """ + Returns experiment's platforms config file name + + :return: platforms config file's name + :rtype: str + """ + pass + + @abstractmethod + def project_file(self): + """ + Returns project's config file name + """ + pass + + @classmethod + def check_proj_file(self): + """ + Add a section header to the project's configuration file (if not exists) + """ + pass + + @abstractmethod + def jobs_file(self): + """ + Returns project's jobs file name + """ + pass + + @abstractmethod + def get_full_config_as_dict(self): + """ + Returns full configuration as json object + """ + pass + + @abstractmethod + def get_full_config_as_json(self): + """ + Return config as json object + """ + pass + + @abstractmethod + def get_project_dir(self): + """ + Returns experiment's project directory + + :return: experiment's project directory + :rtype: str + """ + pass + + @abstractmethod + def get_queue(self, section): + """ + Get queue for the given job type + :param section: job type + :type section: str + :return: queue + :rtype: str + """ + pass + + @abstractmethod + def get_job_platform(self, section): + pass + + @abstractmethod + def get_platform_queue(self, platform): + pass + + @abstractmethod + def get_platform_serial_queue(self, platform): + pass + + @abstractmethod + def get_platform_project(self, platform): + pass + + @abstractmethod + def get_platform_wallclock(self, platform): + pass + + def get_wallclock(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + pass + + def get_synchronize(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + pass + + def get_processors(self, section): + """ + Gets processors needed for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + pass + + def get_threads(self, section): + """ + Gets threads needed for the given job type + :param section: job type + :type section: str + :return: threads needed + :rtype: str + """ + pass + + def get_tasks(self, section): + """ + Gets tasks needed for the given job type + :param section: job type + :type section: str + :return: tasks (processes) per host + :rtype: str + """ + pass + + def get_scratch_free_space(self, section): + """ + Gets scratch free space needed for the given job type + :param section: job type + :type section: str + :return: percentage of scratch free space needed + :rtype: int + """ + pass + + def get_memory(self, section): + """ + Gets memory needed for the given job type + :param section: job type + :type section: str + :return: memory needed + :rtype: str + """ + pass + + def get_memory_per_task(self, section): + """ + Gets memory per task needed for the given job type + :param section: job type + :type section: str + :return: memory per task needed + :rtype: str + """ + pass + + def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + + :return: migrate user to + :rtype: str + """ + #return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + pass + + def get_current_user(self, section): + """ + Returns the user to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + pass + + def get_current_project(self, section): + """ + Returns the project to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + return str(self._platforms_data[section]["USER"]).lower() + + def set_new_user(self, section, new_user): + """ + Sets new user for given platform + :param new_user: + :param section: platform name + :type: str + """ + + + def get_migrate_project_to(self, section): + """ + Returns the project to change to from platform config file. + + :return: migrate project to + :rtype: str + """ + + + + def set_new_project(self, section, new_project): + """ + Sets new project for given platform + :param new_project: + :param section: platform name + :type: str + """ + + + def get_custom_directives(self, section): + """ + Gets custom directives needed for the given job type + :param section: job type + :type section: str + :return: custom directives needed + :rtype: str + """ + + + def check_conf_files(self): + """ + Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing + required options. Prints results in log + + :return: True if everything is correct, False if it finds any error + :rtype: bool + """ + + + def check_autosubmit_conf(self): + """ + Checks experiment's autosubmit configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + + + def check_platforms_conf(self): + """ + Checks experiment's queues configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + + + def check_jobs_conf(self): + """ + Checks experiment's jobs configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + + + def check_expdef_conf(self): + """ + Checks experiment's experiment configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + + + def check_proj(self): + """ + Checks project config file + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + + + def check_wrapper_conf(self): + + + def reload(self): + """ + Creates parser objects for configuration files + """ + + def load_parameters(self): + """ + Load parameters from experiment and autosubmit config files. If experiment's type is not none, + also load parameters from model's config file + + :return: a dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + + + def load_project_parameters(self): + """ + Loads parameters from model config file + + :return: dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + pass + + + def set_expid(self, exp_id): + """ + Set experiment identifier in autosubmit and experiment config files + + :param exp_id: experiment identifier to store + :type exp_id: str + """ + + + def get_project_type(self): + """ + Returns project type from experiment config file + + :return: project type + :rtype: str + """ + pass + + def get_file_project_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + pass + + def get_file_jobs_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + + + def get_git_project_origin(self): + """ + Returns git origin from experiment config file + + :return: git origin + :rtype: str + """ + + def get_git_project_branch(self): + """ + Returns git branch from experiment's config file + + :return: git branch + :rtype: str + """ + + def get_git_project_commit(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + + def get_submodules_list(self): + """ + Returns submodules list from experiment's config file + Default is --recursive + :return: submodules to load + :rtype: list + """ + + def get_project_destination(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + + + def set_git_project_commit(self, as_conf): + """ + Function to register in the configuration the commit SHA of the git project version. + :param as_conf: Configuration class for exteriment + :type as_conf: AutosubmitConfig + """ + + def get_svn_project_url(self): + """ + Gets subversion project url + + :return: subversion project url + :rtype: str + """ + + def get_svn_project_revision(self): + """ + Get revision for subversion project + + :return: revision for subversion project + :rtype: str + """ + + def get_local_project_path(self): + """ + Gets path to origin for local project + + :return: path to local project + :rtype: str + """ + + def get_date_list(self): + """ + Returns startdates list from experiment's config file + + :return: experiment's startdates + :rtype: list + """ + + + def get_num_chunks(self): + """ + Returns number of chunks to run for each member + + :return: number of chunks + :rtype: int + """ + + def get_chunk_ini(self, default=1): + """ + Returns the first chunk from where the experiment will start + + :param default: + :return: initial chunk + :rtype: int + """ + + + def get_chunk_size_unit(self): + # type: () -> str + """ + Unit for the chunk length + + :return: Unit for the chunk length Options: {hour, day, month, year} + :rtype: str + """ + + pass + + def get_chunk_size(self, default=1): + # type: (int) -> int + """ + Chunk Size as defined in the expdef file. + + :return: Chunksize, 1 as default. + :rtype: int + """ + pass + + def get_member_list(self, run_only=False): + """ + Returns members list from experiment's config file + + :return: experiment's members + :rtype: list + """ + pass + + def get_rerun(self): + """ + Returns startdates list from experiment's config file + + :return: rerurn value + :rtype: list + """ + + pass + + def get_chunk_list(self): + """ + Returns chunk list from experiment's config file + + :return: experiment's chunks + :rtype: list + """ + pass + + def get_platform(self): + """ + Returns main platforms from experiment's config file + + :return: main platforms + :rtype: str + """ + pass + + def set_platform(self, hpc): + """ + Sets main platforms in experiment's config file + + :param hpc: main platforms + :type: str + """ + pass + + def set_version(self, autosubmit_version): + """ + Sets autosubmit's version in autosubmit's config file + + :param autosubmit_version: autosubmit's version + :type autosubmit_version: str + """ + pass + + def get_version(self): + """ + Returns version number of the current experiment from autosubmit's config file + + :return: version + :rtype: str + """ + pass + + def get_total_jobs(self): + """ + Returns max number of running jobs from autosubmit's config file + + :return: max number of running jobs + :rtype: int + """ + pass + + def get_max_wallclock(self): + """ + Returns max wallclock + + :rtype: str + """ + pass + + def get_max_processors(self): + """ + Returns max processors from autosubmit's config file + + :rtype: str + """ + pass + + def get_max_waiting_jobs(self): + """ + Returns max number of waiting jobs from autosubmit's config file + + :return: main platforms + :rtype: int + """ + pass + + def get_default_job_type(self): + """ + Returns the default job type from experiment's config file + + :return: default type such as bash, python, r.. + :rtype: str + """ + pass + + def get_safetysleeptime(self): + """ + Returns safety sleep time from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + pass + + def set_safetysleeptime(self, sleep_time): + """ + Sets autosubmit's version in autosubmit's config file + + :param sleep_time: value to set + :type sleep_time: int + """ + pass + + def get_retrials(self): + """ + Returns max number of retrials for job from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + pass + + def get_notifications(self): + """ + Returns if the user has enabled the notifications from autosubmit's config file + + :return: if notifications + :rtype: string + """ + pass + + def get_remote_dependencies(self): + """ + Returns if the user has enabled the remote dependencies from autosubmit's config file + + :return: if remote dependencies + :rtype: bool + """ + pass + + def get_wrapper_type(self): + """ + Returns what kind of wrapper (VERTICAL, MIXED-VERTICAL, HORIZONTAL, HYBRID, NONE) the user has configured in the autosubmit's config + + :return: wrapper type (or none) + :rtype: string + """ + pass + + def get_wrapper_jobs(self): + """ + Returns the jobs that should be wrapped, configured in the autosubmit's config + + :return: expression (or none) + :rtype: string + """ + pass + + def get_max_wrapped_jobs(self): + """ + Returns the maximum number of jobs that can be wrapped together as configured in autosubmit's config file + + :return: maximum number of jobs (or total jobs) + :rtype: string + """ + # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs())) + + pass + + def get_wrapper_check_time(self): + """ + Returns time to check the status of jobs in the wrapper + + :return: wrapper check time + :rtype: int + """ + pass + + def get_wrapper_machinefiles(self): + """ + Returns the strategy for creating the machinefiles in wrapper jobs + + :return: machinefiles function to use + :rtype: string + """ + pass + + def get_wrapper_queue(self): + """ + Returns the wrapper queue if not defined, will be the one of the first job wrapped + + :return: expression (or none) + :rtype: string + """ + pass + + def get_jobs_sections(self): + """ + Returns the list of sections defined in the jobs config file + + :return: sections + :rtype: list + """ + return self._jobs_parser.sections() + + def get_copy_remote_logs(self): + """ + Returns if the user has enabled the logs local copy from autosubmit's config file + + :return: if logs local copy + :rtype: bool + """ + pass + + def get_mails_to(self): + """ + Returns the address where notifications will be sent from autosubmit's config file + + :return: mail address + :rtype: [str] + """ + pass + + def get_communications_library(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + pass + + def get_storage_type(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + pass + + @staticmethod + def is_valid_mail_address(mail_address): + if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): + return True + else: + return False + @classmethod + def is_valid_communications_library(self): + library = self.get_communications_library() + return library in ['paramiko', 'saga'] + + @classmethod + def is_valid_storage_type(self): + storage_type = self.get_storage_type() + return storage_type in ['pkl', 'db'] + + + def is_valid_jobs_in_wrapper(self): + pass + + def is_valid_git_repository(self): + pass + + @staticmethod + def get_parser(parser_factory, file_path): + # type: (ConfigParserFactory, str) -> ConfigParser + pass \ No newline at end of file diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py new file mode 100644 index 00000000..103e5f2f --- /dev/null +++ b/autosubmit_api/config/confConfigStrategy.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# Copyright 2015 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . + +try: + # noinspection PyCompatibility + from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config +except ImportError: + # noinspection PyCompatibility + from configparser import SafeConfigParser + + +import os +import re +import subprocess +import json +import logging +import locale + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.IConfigStrategy import IConfigStrategy + + + +logger = logging.getLogger('gunicorn.error') + +class confConfigStrategy(IConfigStrategy): + """ + Class to handle experiment configuration coming from file or database + + :param expid: experiment identifier + :type expid: str + """ + def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + + @classmethod + def jobs_parser(self): + raise NotImplementedError + + @classmethod + def experiment_file(self): + """ + Returns experiment's config file name + """ + return self._exp_parser_file + + @classmethod + def platforms_parser(self): \ No newline at end of file diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py new file mode 100644 index 00000000..b20fa63c --- /dev/null +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +# Copyright 2015 Earth Sciences Department, BSC-CNS + +# This file is part of Autosubmit. + +# Autosubmit is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# Autosubmit is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . + +try: + # noinspection PyCompatibility + from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config +except ImportError: + # noinspection PyCompatibility + from configparser import SafeConfigParser + + +import os +import re +import subprocess +import json +import logging +import locale + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.IConfigStrategy import IConfigStrategy + +logger = logging.getLogger('gunicorn.error') + +class ymlConfigStrategy(IConfigStrategy): + """ + Class to handle experiment configuration coming from file or database + + :param expid: experiment identifier + :type expid: str + """ + def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + + @classmethod + def jobs_parser(self): + raise NotImplementedError + + @classmethod + def experiment_file(self): + """ + Returns experiment's config file name + """ + return self._exp_parser_file + + @classmethod + def platforms_parser(self): \ No newline at end of file -- GitLab From 46b15a315a512ec73a53572bbcf75743c8260af0 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Wed, 10 May 2023 17:18:06 +0200 Subject: [PATCH 11/25] Implement the strategy for AS3 and above config files, adapt config_common to use the config wrappers- #12 --- autosubmit_api/config/IConfigStrategy.py | 15 +- autosubmit_api/config/confConfigStrategy.py | 1327 ++++++++++++++++++- autosubmit_api/config/config_common.py | 782 ++--------- autosubmit_api/config/ymlConfigStrategy.py | 299 ++++- 4 files changed, 1712 insertions(+), 711 deletions(-) diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py index 062e966c..d568fe75 100644 --- a/autosubmit_api/config/IConfigStrategy.py +++ b/autosubmit_api/config/IConfigStrategy.py @@ -39,10 +39,10 @@ from bscearth.utils.date import parse_date from bscearth.utils.log import Log from ..config.basicConfig import BasicConfig from ..config.ymlConfigParser import ymlConfigParser -from abc import ABCMeta, abstractmethod +from abc import ABC, abstractmethod -class IConfigStrategy: +class IConfigStrategy(ABC): """ Public interface for Autosubmit config files @@ -50,8 +50,6 @@ class IConfigStrategy: :type expid: str """ - __metaclass__ = ABCMeta - @abstractmethod def jobs_parser(self): raise NotImplementedError @@ -71,7 +69,7 @@ class IConfigStrategy: :return: platforms config parser object :rtype: SafeConfigParser """ - pass + pass @abstractmethod def platforms_file(self): @@ -111,13 +109,6 @@ class IConfigStrategy: """ pass - @abstractmethod - def get_full_config_as_json(self): - """ - Return config as json object - """ - pass - @abstractmethod def get_project_dir(self): """ diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py index 103e5f2f..0a74d12c 100644 --- a/autosubmit_api/config/confConfigStrategy.py +++ b/autosubmit_api/config/confConfigStrategy.py @@ -1,4 +1,6 @@ -#!/usr/bin/env python + +# !/usr/bin/env python + # Copyright 2015 Earth Sciences Department, BSC-CNS # This file is part of Autosubmit. @@ -15,31 +17,25 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . - try: # noinspection PyCompatibility from configparser import SafeConfigParser - from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser - - import os import re import subprocess import json import logging -import locale from pyparsing import nestedExpr from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser from bscearth.utils.date import parse_date from bscearth.utils.log import Log +from ..config.basicConfig import BasicConfig from ..config.IConfigStrategy import IConfigStrategy - - logger = logging.getLogger('gunicorn.error') class confConfigStrategy(IConfigStrategy): @@ -49,18 +45,1323 @@ class confConfigStrategy(IConfigStrategy): :param expid: experiment identifier :type expid: str """ + def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + # type: (str, BasicConfig, ConfigParserFactory, Extension) -> None + + self.expid = expid + self.basic_config = basic_config + self.parser_factory = parser_factory + + # By default check for .yml files first as it is the new standard for AS 4.0 + self._conf_parser = None # type: ConfigParser + self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "autosubmit_" + expid + extension) + if os.path.exists(self._conf_parser_file) == False: + if extension == ".yml": + self.__init__(expid, basic_config, parser_factory, ".conf") + elif extension == ".conf": + return None + + self._exp_parser = None # type: ConfigParser + self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "expdef_" + expid + extension) + if os.path.exists(self._exp_parser_file) == False: + if extension == ".yml": + self.__init__(expid, basic_config, parser_factory, ".conf") + elif extension == ".conf": + return None + + self._platforms_parser = None # type: ConfigParser + self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "platforms_" + expid + extension) + if os.path.exists(self._platforms_parser_file) == False: + if extension == ".yml": + self.__init__(expid, basic_config, parser_factory, ".conf") + elif extension == ".conf": + return None - @classmethod + self._jobs_parser = None # type: ConfigParser + self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "jobs_" + expid + extension) + if os.path.exists(self._jobs_parser_file) == False: + if extension == ".yml": + self.__init__(expid, basic_config, parser_factory, ".conf") + elif extension == ".conf": + return None + + self._proj_parser = None # type: ConfigParser + self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", + "proj_" + expid + extension) + if os.path.exists(self._proj_parser_file) == False: + if extension == ".yml": + self.__init__(expid, basic_config, parser_factory, ".conf") + elif extension == ".conf": + return None + + self.check_proj_file() + + @property def jobs_parser(self): - raise NotImplementedError + return self._jobs_parser - @classmethod + @property def experiment_file(self): """ Returns experiment's config file name """ return self._exp_parser_file - @classmethod - def platforms_parser(self): \ No newline at end of file + @property + def platforms_parser(self): + """ + Returns experiment's platforms parser object + + :return: platforms config parser object + :rtype: SafeConfigParser + """ + return self._platforms_parser + + @property + def platforms_file(self): + """ + Returns experiment's platforms config file name + + :return: platforms config file's name + :rtype: str + """ + return self._platforms_parser_file + + @property + def project_file(self): + """ + Returns project's config file name + """ + return self._proj_parser_file + + def check_proj_file(self): + """ + Add a section header to the project's configuration file (if not exists) + """ + # if os.path.exists(self._proj_parser_file): + # with open(self._proj_parser_file, 'w') as f: + # first_line = f.readline() + # if not re.match('[[a-zA-Z0-9]*]', first_line): + # content = f.read() + # f.seek(0, 0) + # f.write('[DEFAULT]'.rstrip('\r\n') + + # '\n' + first_line + content) + pass + + @property + def jobs_file(self): + """ + Returns project's jobs file name + """ + return self._jobs_parser_file + + def get_full_config_as_dict(self): + """ + Returns full configuration as json object + """ + _conf = _exp = _platforms = _jobs = _proj = None + result = {} + + def get_data(parser): + """ + dictionary comprehension to get data from parser + """ + res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ + section for section in parser.sections()]} + return res + + # print(self._conf_parser) + result["conf"] = get_data( + self._conf_parser) if self._conf_parser else None + result["exp"] = get_data( + self._exp_parser) if self._exp_parser else None + result["platforms"] = get_data( + self._platforms_parser) if self._platforms_parser else None + result["jobs"] = get_data( + self._jobs_parser) if self._jobs_parser else None + result["proj"] = get_data( + self._proj_parser) if self._proj_parser else None + return result + + def get_full_config_as_json(self): + """ + Return config as json object + """ + try: + return json.dumps(self.get_full_config_as_dict()) + except Exception as exp: + Log.warning( + "Autosubmit was not able to retrieve and save the configuration into the historical database.") + return "" + + def get_project_dir(self): + """ + Returns experiment's project directory + + :return: experiment's project directory + :rtype: str + """ + dir_templates = os.path.join(self.basic_config.LOCAL_ROOT_DIR, self.expid, BasicConfig.LOCAL_PROJ_DIR, + self.get_project_destination()) + return dir_templates + + def get_queue(self, section): + """ + Get queue for the given job type + :param section: job type + :type section: str + :return: queue + :rtype: str + """ + return self._jobs_parser.get_option(section, 'QUEUE', '') + + def get_job_platform(self, section): + return self._jobs_parser.get_option(section, 'PLATFORM', '') + + def get_platform_queue(self, platform): + return self._platforms_parser.get_option(platform, 'QUEUE', '') + + def get_platform_serial_queue(self, platform): + return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '') + + def get_platform_project(self, platform): + return self._platforms_parser.get_option(platform, "PROJECT", "") + + def get_platform_wallclock(self, platform): + return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '') + + def get_wallclock(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + return self._jobs_parser.get_option(section, 'WALLCLOCK', '') + + def get_synchronize(self, section): + """ + Gets wallclock for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + return self._jobs_parser.get_option(section, 'SYNCHRONIZE', '') + + def get_processors(self, section): + """ + Gets processors needed for the given job type + :param section: job type + :type section: str + :return: wallclock time + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'PROCESSORS', 1)) + + def get_threads(self, section): + """ + Gets threads needed for the given job type + :param section: job type + :type section: str + :return: threads needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'THREADS', 1)) + + def get_tasks(self, section): + """ + Gets tasks needed for the given job type + :param section: job type + :type section: str + :return: tasks (processes) per host + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'TASKS', 0)) + + def get_scratch_free_space(self, section): + """ + Gets scratch free space needed for the given job type + :param section: job type + :type section: str + :return: percentage of scratch free space needed + :rtype: int + """ + return int(self._jobs_parser.get_option(section, 'SCRATCH_FREE_SPACE', 0)) + + def get_memory(self, section): + """ + Gets memory needed for the given job type + :param section: job type + :type section: str + :return: memory needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'MEMORY', '')) + + def get_memory_per_task(self, section): + """ + Gets memory per task needed for the given job type + :param section: job type + :type section: str + :return: memory per task needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', '')) + + def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'USER_TO', '').lower() + + def get_current_user(self, section): + """ + Returns the user to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'USER', '').lower() + + def get_current_project(self, section): + """ + Returns the project to be changed from platform config file. + + :return: migrate user to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'PROJECT', '').lower() + + def set_new_user(self, section, new_user): + """ + Sets new user for given platform + :param new_user: + :param section: platform name + :type: str + """ + with open(self._platforms_parser_file) as p_file: + contentLine = p_file.readline() + contentToMod = "" + content = "" + mod = False + while contentLine: + if re.search(section, contentLine): + mod = True + if mod: + contentToMod += contentLine + else: + content += contentLine + contentLine = p_file.readline() + if mod: + old_user = self.get_current_user(section) + contentToMod = contentToMod.replace(re.search( + r'[^#]\bUSER\b =.*', contentToMod).group(0)[1:], "USER = " + new_user) + contentToMod = contentToMod.replace(re.search( + r'[^#]\bUSER_TO\b =.*', contentToMod).group(0)[1:], "USER_TO = " + old_user) + open(self._platforms_parser_file, 'w').write(content) + open(self._platforms_parser_file, 'a').write(contentToMod) + + def get_migrate_project_to(self, section): + """ + Returns the project to change to from platform config file. + + :return: migrate project to + :rtype: str + """ + return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() + + def set_new_project(self, section, new_project): + """ + Sets new project for given platform + :param new_project: + :param section: platform name + :type: str + """ + with open(self._platforms_parser_file) as p_file: + contentLine = p_file.readline() + contentToMod = "" + content = "" + mod = False + while contentLine: + if re.search(section, contentLine): + mod = True + if mod: + contentToMod += contentLine + else: + content += contentLine + contentLine = p_file.readline() + if mod: + old_project = self.get_current_project(section) + contentToMod = contentToMod.replace(re.search( + r"[^#]\bPROJECT\b =.*", contentToMod).group(0)[1:], "PROJECT = " + new_project) + contentToMod = contentToMod.replace(re.search( + r"[^#]\bPROJECT_TO\b =.*", contentToMod).group(0)[1:], "PROJECT_TO = " + old_project) + open(self._platforms_parser_file, 'w').write(content) + open(self._platforms_parser_file, 'a').write(contentToMod) + + def get_custom_directives(self, section): + """ + Gets custom directives needed for the given job type + :param section: job type + :type section: str + :return: custom directives needed + :rtype: str + """ + return str(self._jobs_parser.get_option(section, 'CUSTOM_DIRECTIVES', '')) + + def check_conf_files(self): + """ + Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing + required options. Prints results in log + + :return: True if everything is correct, False if it finds any error + :rtype: bool + """ + Log.debug('\nChecking configuration files...') + self.reload() + # result = self.check_platforms_conf() + result = True + result = result and self.check_jobs_conf() + result = result and self.check_autosubmit_conf() + result = result and self.check_expdef_conf() + if result: + Log.debug("Configuration files OK\n") + else: + Log.error("Configuration files invalid\n") + return result + + def check_autosubmit_conf(self): + """ + Checks experiment's autosubmit configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + + self._conf_parser.read(self._conf_parser_file) + # result = result and self._conf_parser.check_exists( + # 'config', 'AUTOSUBMIT_VERSION') + # result = result and self._conf_parser.check_is_int( + # 'config', 'MAXWAITINGJOBS', True) + # result = result and self._conf_parser.check_is_int( + # 'config', 'TOTALJOBS', True) + # result = result and self._conf_parser.check_is_int( + # 'config', 'SAFETYSLEEPTIME', True) + # result = result and self._conf_parser.check_is_int( + # 'config', 'RETRIALS', True) + # result = result and self._conf_parser.check_is_boolean( + # 'mail', 'NOTIFICATIONS', False) + result = result and self.is_valid_communications_library() + result = result and self.is_valid_storage_type() + if self.get_wrapper_type() != 'None': + result = result and self.check_wrapper_conf() + + if self.get_notifications() == 'true': + for mail in self.get_mails_to(): + if not self.is_valid_mail_address(mail): + # Log.warning( + # 'One or more of the email addresses configured for the mail notifications are wrong') + break + + if not result: + # Log.critical("{0} is not a valid config file".format( + # os.path.basename(self._conf_parser_file))) + raise Exception("Permission denied for " + + str(os.path.basename(self._conf_parser_file))) + else: + Log.debug('{0} OK'.format( + os.path.basename(self._conf_parser_file))) + return result + + def check_platforms_conf(self): + """ + Checks experiment's queues configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + if len(self._platforms_parser.sections()) == 0: + Log.warning("No remote platforms configured") + + if len(self._platforms_parser.sections()) != len(set(self._platforms_parser.sections())): + Log.error('There are repeated platforms names') + + for section in self._platforms_parser.sections(): + result = result and self._platforms_parser.check_exists( + section, 'TYPE') + platform_type = self._platforms_parser.get_option( + section, 'TYPE', '').lower() + if platform_type != 'ps': + result = result and self._platforms_parser.check_exists( + section, 'PROJECT') + result = result and self._platforms_parser.check_exists( + section, 'USER') + + result = result and self._platforms_parser.check_exists( + section, 'HOST') + result = result and self._platforms_parser.check_exists( + section, 'SCRATCH_DIR') + result = result and self._platforms_parser.check_is_boolean(section, + 'ADD_PROJECT_TO_HOST', False) + result = result and self._platforms_parser.check_is_boolean( + section, 'TEST_SUITE', False) + result = result and self._platforms_parser.check_is_int(section, 'MAX_WAITING_JOBS', + False) + result = result and self._platforms_parser.check_is_int( + section, 'TOTAL_JOBS', False) + + if not result: + Log.critical("{0} is not a valid config file".format( + os.path.basename(self._platforms_parser_file))) + raise Exception("Permission denied for " + + str(os.path.basename(self._platforms_parser_file))) + else: + Log.info('{0} OK'.format( + os.path.basename(self._platforms_parser_file))) + return result + + def check_jobs_conf(self): + """ + Checks experiment's jobs configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + parser = self._jobs_parser + sections = parser.sections() + platforms = self._platforms_parser.sections() + platforms.append('LOCAL') + possible_exception = "" + if len(sections) == 0: + possible_exception += "No remote platforms configured\n" + + # if len(sections) != len(set(sections)): + # Log.error('There are repeated job names') + + for section in sections: + + result = result and parser.check_exists(section, 'FILE') + result = result and parser.check_is_boolean( + section, 'RERUN_ONLY', False) + + if parser.has_option(section, 'PLATFORM'): + # result = result and parser.check_is_choice( + # section, 'PLATFORM', False, platforms) + pass + if parser.has_option(section, 'DEPENDENCIES'): + for dependency in str(parser.get_option(section, 'DEPENDENCIES', '')).split(' '): + if '-' in dependency: + dependency = dependency.split('-')[0] + elif '+' in dependency: + dependency = dependency.split('+')[0] + if '[' in dependency: + dependency = dependency[:dependency.find('[')] + # if dependency not in sections: + # Log.error( + # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, + # dependency)) + + if parser.has_option(section, 'RERUN_DEPENDENCIES'): + for dependency in str(parser.get_option(section, 'RERUN_DEPENDENCIES', + '')).split(' '): + if '-' in dependency: + dependency = dependency.split('-')[0] + if '[' in dependency: + dependency = dependency[:dependency.find('[')] + # if dependency not in sections: + # Log.error( + # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, + # dependency)) + result = result and parser.check_is_choice(section, 'RUNNING', False, + ['once', 'date', 'member', 'chunk']) + + if not result: + # Log.critical("{0} is not a valid config file".format( + # os.path.basename(self._jobs_parser_file))) + raise Exception("Exception while checking jobs_expid.conf " + + str(os.path.basename(self._jobs_parser_file)) + possible_exception) + else: + Log.debug('{0} OK'.format( + os.path.basename(self._jobs_parser_file))) + + return result + + def check_expdef_conf(self): + """ + Checks experiment's experiment configuration file. + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + result = True + parser = self._exp_parser + + # result = result and parser.check_exists('DEFAULT', 'EXPID') + # result = result and parser.check_exists('DEFAULT', 'HPCARCH') + + result = result and parser.check_exists('experiment', 'DATELIST') + result = result and parser.check_exists('experiment', 'MEMBERS') + result = result and parser.check_is_choice('experiment', 'CHUNKSIZEUNIT', True, + ['year', 'month', 'day', 'hour']) + result = result and parser.check_is_int( + 'experiment', 'CHUNKSIZE', True) + result = result and parser.check_is_int( + 'experiment', 'NUMCHUNKS', True) + result = result and parser.check_is_choice('experiment', 'CALENDAR', True, + ['standard', 'noleap']) + + result = result and parser.check_is_boolean('rerun', 'RERUN', True) + + if parser.check_is_choice('project', 'PROJECT_TYPE', True, + ['none', 'git', 'svn', 'local']): + project_type = parser.get_option('project', 'PROJECT_TYPE', '') + + if project_type == 'git': + result = result and parser.check_exists( + 'git', 'PROJECT_ORIGIN') + result = result and parser.check_exists( + 'git', 'PROJECT_BRANCH') + + elif project_type == 'svn': + result = result and parser.check_exists('svn', 'PROJECT_URL') + result = result and parser.check_exists( + 'svn', 'PROJECT_REVISION') + elif project_type == 'local': + result = result and parser.check_exists( + 'local', 'PROJECT_PATH') + + if project_type != 'none': + result = result and parser.check_exists( + 'project_files', 'FILE_PROJECT_CONF') + else: + result = True + + if not result: + # Log.critical("{0} is not a valid config file".format( + # os.path.basename(self._exp_parser_file))) + raise Exception("Permission denied for " + + str(os.path.basename(self._exp_parser_file))) + else: + Log.debug('{0} OK'.format( + os.path.basename(self._exp_parser_file))) + return result + + def check_proj(self): + """ + Checks project config file + + :return: True if everything is correct, False if it founds any error + :rtype: bool + """ + try: + if self._proj_parser_file == '': + self._proj_parser = None + else: + self._proj_parser = AutosubmitConfig.get_parser( + self.parser_factory, self._proj_parser_file) + return True + except Exception as e: + Log.error('Project conf file error: {0}', e) + return False + + def check_wrapper_conf(self): + result = True + result = result and self.is_valid_jobs_in_wrapper() + if not result: + Log.error( + "There are sections in JOBS_IN_WRAPPER that are not defined in your jobs.conf file") + + if 'horizontal' in self.get_wrapper_type(): + result = result and self._platforms_parser.check_exists( + self.get_platform(), 'PROCESSORS_PER_NODE') + result = result and self._platforms_parser.check_exists( + self.get_platform(), 'MAX_PROCESSORS') + if 'vertical' in self.get_wrapper_type(): + result = result and self._platforms_parser.check_exists( + self.get_platform(), 'MAX_WALLCLOCK') + return result + + def reload(self): + """ + Creates parser objects for configuration files + """ + if not os.path.exists(self._conf_parser_file): raise IOError( + "Required file not found {0}".format(self._conf_parser_file)) + if not os.path.exists(self._platforms_parser_file): raise IOError( + "Required file not found {0}".format(self._platforms_parser_file)) + if not os.path.exists(self._jobs_parser_file): raise IOError( + "Required file not found {0}".format(self._jobs_parser_file)) + if not os.path.exists(self._exp_parser_file): raise IOError( + "Required file not found {0}".format(self._exp_parser_file)) + self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file) + self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) + self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file) + self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file) + if self._proj_parser_file == '': + self._proj_parser = None + else: + self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file) + + def load_parameters(self): + """ + Load parameters from experiment and autosubmit config files. If experiment's type is not none, + also load parameters from model's config file + + :return: a dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + parameters = dict() + for section in self._exp_parser.sections(): + for option in self._exp_parser.options(section): + parameters[option] = self._exp_parser.get(section, option) + for section in self._conf_parser.sections(): + for option in self._conf_parser.options(section): + parameters[option] = self._conf_parser.get(section, option) + + # project_type = self.get_project_type() + # if project_type != "none" and self._proj_parser is not None: + # # Load project parameters + # Log.debug("Loading project parameters...") + # parameters2 = parameters.copy() + # parameters2.update(self.load_project_parameters()) + # parameters = parameters2 + + return parameters + + def load_project_parameters(self): + """ + Loads parameters from model config file + + :return: dictionary containing tuples [parameter_name, parameter_value] + :rtype: dict + """ + pass + # projdef = [] + # for section in self._proj_parser.sections(): + # print("SEction" + section) + # projdef += self._proj_parser.items(section) + + # parameters = dict() + # for item in projdef: + # parameters[item[0]] = item[1] + + # return parameters + + def set_expid(self, exp_id): + """ + Set experiment identifier in autosubmit and experiment config files + + :param exp_id: experiment identifier to store + :type exp_id: str + """ + # Experiment conf + content = open(self._exp_parser_file).read() + if re.search('EXPID =.*', content): + content = content.replace( + re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) + open(self._exp_parser_file, 'w').write(content) + + content = open(self._conf_parser_file).read() + if re.search('EXPID =.*', content): + content = content.replace( + re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) + open(self._conf_parser_file, 'w').write(content) + + def get_project_type(self): + """ + Returns project type from experiment config file + + :return: project type + :rtype: str + """ + return self._exp_parser.get_option('project', 'PROJECT_TYPE', "NA").lower() + + def get_file_project_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + return self._exp_parser.get('project_files', 'FILE_PROJECT_CONF') + + def get_file_jobs_conf(self): + """ + Returns path to project config file from experiment config file + + :return: path to project config file + :rtype: str + """ + return self._exp_parser.get_option('project_files', 'FILE_JOBS_CONF', '') + + def get_git_project_origin(self): + """ + Returns git origin from experiment config file + + :return: git origin + :rtype: str + """ + return self._exp_parser.get_option('git', 'PROJECT_ORIGIN', '') + + def get_git_project_branch(self): + """ + Returns git branch from experiment's config file + + :return: git branch + :rtype: str + """ + return self._exp_parser.get_option('git', 'PROJECT_BRANCH', 'master') + + def get_git_project_commit(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) + + def get_submodules_list(self): + """ + Returns submodules list from experiment's config file + Default is --recursive + :return: submodules to load + :rtype: list + """ + return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES', '').split()).split() + + def get_project_destination(self): + """ + Returns git commit from experiment's config file + + :return: git commit + :rtype: str + """ + value = self._exp_parser.get('project', 'PROJECT_DESTINATION') + if not value: + if self.get_project_type().lower() == "local": + value = os.path.split(self.get_local_project_path())[1] + elif self.get_project_type().lower() == "svn": + value = self.get_svn_project_url().split('/')[-1] + elif self.get_project_type().lower() == "git": + value = self.get_git_project_origin().split( + '/')[-1].split('.')[-2] + return value + + def set_git_project_commit(self, as_conf): + """ + Function to register in the configuration the commit SHA of the git project version. + :param as_conf: Configuration class for exteriment + :type as_conf: AutosubmitConfig + """ + full_project_path = as_conf.get_project_dir() + try: + output = subprocess.check_output( + "cd {0}; git rev-parse --abbrev-ref HEAD".format(full_project_path), + shell=True) + except subprocess.CalledProcessError: + Log.critical("Failed to retrieve project branch...") + return False + + project_branch = output + Log.debug("Project branch is: " + project_branch) + try: + output = subprocess.check_output( + "cd {0}; git rev-parse HEAD".format(full_project_path), shell=True) + except subprocess.CalledProcessError: + Log.critical("Failed to retrieve project commit SHA...") + return False + project_sha = output + Log.debug("Project commit SHA is: " + project_sha) + + # register changes + content = open(self._exp_parser_file).read() + if re.search('PROJECT_BRANCH =.*', content): + content = content.replace(re.search('PROJECT_BRANCH =.*', content).group(0), + "PROJECT_BRANCH = " + project_branch) + if re.search('PROJECT_COMMIT =.*', content): + content = content.replace(re.search('PROJECT_COMMIT =.*', content).group(0), + "PROJECT_COMMIT = " + project_sha) + open(self._exp_parser_file, 'w').write(content) + Log.debug( + "Project commit SHA succesfully registered to the configuration file.") + return True + + def get_svn_project_url(self): + """ + Gets subversion project url + + :return: subversion project url + :rtype: str + """ + return self._exp_parser.get_option('svn', 'PROJECT_URL', 'NA') + + def get_svn_project_revision(self): + """ + Get revision for subversion project + + :return: revision for subversion project + :rtype: str + """ + return self._exp_parser.get('svn', 'PROJECT_REVISION') + + def get_local_project_path(self): + """ + Gets path to origin for local project + + :return: path to local project + :rtype: str + """ + return self._exp_parser.get('local', 'PROJECT_PATH') + + def get_date_list(self): + """ + Returns startdates list from experiment's config file + + :return: experiment's startdates + :rtype: list + """ + date_list = list() + string = self._exp_parser.get('experiment', 'DATELIST') + if not string.startswith("["): + string = '[{0}]'.format(string) + split_string = nestedExpr('[', ']').parseString(string).asList() + string_date = None + for split in split_string[0]: + if type(split) is list: + for split_in in split: + if split_in.find("-") != -1: + numbers = split_in.split("-") + for count in range(int(numbers[0]), int(numbers[1]) + 1): + date_list.append(parse_date( + string_date + str(count).zfill(len(numbers[0])))) + else: + date_list.append(parse_date(string_date + split_in)) + string_date = None + else: + if string_date is not None: + date_list.append(parse_date(string_date)) + string_date = split + if string_date is not None: + date_list.append(parse_date(string_date)) + return date_list + + def get_num_chunks(self): + """ + Returns number of chunks to run for each member + + :return: number of chunks + :rtype: int + """ + return int(self._exp_parser.get('experiment', 'NUMCHUNKS')) + + def get_chunk_ini(self, default=1): + """ + Returns the first chunk from where the experiment will start + + :param default: + :return: initial chunk + :rtype: int + """ + chunk_ini = self._exp_parser.get_option( + 'experiment', 'CHUNKINI', default) + if chunk_ini == '': + return default + return int(chunk_ini) + + def get_chunk_size_unit(self): + # type: () -> str + """ + Unit for the chunk length + + :return: Unit for the chunk length Options: {hour, day, month, year} + :rtype: str + """ + + # try: + # res = self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() + # except Exception as e: + # try: + # res = self.autosubmit_conf.get('experiment') + return self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() + + def get_chunk_size(self, default=1): + # type: (int) -> int + """ + Chunk Size as defined in the expdef file. + + :return: Chunksize, 1 as default. + :rtype: int + """ + try: + chunk_size = self._exp_parser.get_option( + 'experiment', 'CHUNKSIZE', default) + except Exception as exp: + print(exp) + chunk_size = '' + pass + if chunk_size == '': + return default + return int(chunk_size) + + def get_member_list(self, run_only=False): + """ + Returns members list from experiment's config file + + :return: experiment's members + :rtype: list + """ + member_list = list() + string = self._exp_parser.get('experiment', + 'MEMBERS') if run_only == False else self._exp_parser.get_option( + 'experiment', 'RUN_ONLY_MEMBERS', '') + if not string.startswith("["): + string = '[{0}]'.format(string) + split_string = nestedExpr('[', ']').parseString(string).asList() + string_member = None + for split in split_string[0]: + if type(split) is list: + for split_in in split: + if split_in.find("-") != -1: + numbers = split_in.split("-") + for count in range(int(numbers[0]), int(numbers[1]) + 1): + member_list.append( + string_member + str(count).zfill(len(numbers[0]))) + else: + member_list.append(string_member + split_in) + string_member = None + else: + if string_member is not None: + member_list.append(string_member) + string_member = split + if string_member is not None: + member_list.append(string_member) + return member_list + + def get_rerun(self): + """ + Returns startdates list from experiment's config file + + :return: rerurn value + :rtype: list + """ + + return self._exp_parser.get('rerun', 'RERUN').lower() + + def get_chunk_list(self): + """ + Returns chunk list from experiment's config file + + :return: experiment's chunks + :rtype: list + """ + return self._exp_parser.get('rerun', 'CHUNKLIST') + + def get_platform(self): + """ + Returns main platforms from experiment's config file + + :return: main platforms + :rtype: str + """ + return self._exp_parser.get('experiment', 'HPCARCH') + + def set_platform(self, hpc): + """ + Sets main platforms in experiment's config file + + :param hpc: main platforms + :type: str + """ + content = open(self._exp_parser_file).read() + if re.search('HPCARCH =.*', content): + content = content.replace( + re.search('HPCARCH =.*', content).group(0), "HPCARCH = " + hpc) + open(self._exp_parser_file, 'w').write(content) + + def set_version(self, autosubmit_version): + """ + Sets autosubmit's version in autosubmit's config file + + :param autosubmit_version: autosubmit's version + :type autosubmit_version: str + """ + content = open(self._conf_parser_file).read() + if re.search('AUTOSUBMIT_VERSION =.*', content): + content = content.replace(re.search('AUTOSUBMIT_VERSION =.*', content).group(0), + "AUTOSUBMIT_VERSION = " + autosubmit_version) + open(self._conf_parser_file, 'w').write(content) + + def get_version(self): + """ + Returns version number of the current experiment from autosubmit's config file + + :return: version + :rtype: str + """ + return self._conf_parser.get_option('config', 'AUTOSUBMIT_VERSION', 'None') + + def get_total_jobs(self): + """ + Returns max number of running jobs from autosubmit's config file + + :return: max number of running jobs + :rtype: int + """ + return int(self._conf_parser.get('config', 'TOTALJOBS')) + + def get_max_wallclock(self): + """ + Returns max wallclock + + :rtype: str + """ + return self._conf_parser.get_option('config', 'MAX_WALLCLOCK', '') + + def get_max_processors(self): + """ + Returns max processors from autosubmit's config file + + :rtype: str + """ + config_value = self._conf_parser.get_option( + 'config', 'MAX_PROCESSORS', None) + return int(config_value) if config_value is not None else config_value + + def get_max_waiting_jobs(self): + """ + Returns max number of waiting jobs from autosubmit's config file + + :return: main platforms + :rtype: int + """ + return int(self._conf_parser.get_option('config', 'MAXWAITINGJOBS', 10)) + + def get_default_job_type(self): + """ + Returns the default job type from experiment's config file + + :return: default type such as bash, python, r.. + :rtype: str + """ + return self._exp_parser.get_option('project_files', 'JOB_SCRIPTS_TYPE', 'bash') + + def get_safetysleeptime(self): + """ + Returns safety sleep time from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + return int(self._conf_parser.get_option('config', 'SAFETYSLEEPTIME', 10)) + + def set_safetysleeptime(self, sleep_time): + """ + Sets autosubmit's version in autosubmit's config file + + :param sleep_time: value to set + :type sleep_time: int + """ + content = open(self._conf_parser_file).read() + content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0), + "SAFETYSLEEPTIME = %d" % sleep_time) + open(self._conf_parser_file, 'w').write(content) + + def get_retrials(self): + """ + Returns max number of retrials for job from autosubmit's config file + + :return: safety sleep time + :rtype: int + """ + return int(self._conf_parser.get('config', 'RETRIALS')) + + def get_notifications(self): + """ + Returns if the user has enabled the notifications from autosubmit's config file + + :return: if notifications + :rtype: string + """ + return self._conf_parser.get_option('mail', 'NOTIFICATIONS', 'false').lower() + + def get_remote_dependencies(self): + """ + Returns if the user has enabled the remote dependencies from autosubmit's config file + + :return: if remote dependencies + :rtype: bool + """ + return self._conf_parser.get_option('wrapper', 'DEPENDENCIES', 'false').lower() == 'true' + + def get_wrapper_type(self): + """ + Returns what kind of wrapper (VERTICAL, MIXED-VERTICAL, HORIZONTAL, HYBRID, NONE) the user has configured in the autosubmit's config + + :return: wrapper type (or none) + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'TYPE', 'None').lower() + + def get_wrapper_jobs(self): + """ + Returns the jobs that should be wrapped, configured in the autosubmit's config + + :return: expression (or none) + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'JOBS_IN_WRAPPER', 'None') + + def get_max_wrapped_jobs(self): + """ + Returns the maximum number of jobs that can be wrapped together as configured in autosubmit's config file + + :return: maximum number of jobs (or total jobs) + :rtype: string + """ + # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs())) + + return int(self._conf_parser.get_option('wrapper', 'MAX_WRAPPED', self.get_total_jobs())) + + def get_wrapper_check_time(self): + """ + Returns time to check the status of jobs in the wrapper + + :return: wrapper check time + :rtype: int + """ + return int(self._conf_parser.get_option('wrapper', 'CHECK_TIME_WRAPPER', self.get_safetysleeptime())) + + def get_wrapper_machinefiles(self): + """ + Returns the strategy for creating the machinefiles in wrapper jobs + + :return: machinefiles function to use + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'MACHINEFILES', '') + + def get_wrapper_queue(self): + """ + Returns the wrapper queue if not defined, will be the one of the first job wrapped + + :return: expression (or none) + :rtype: string + """ + return self._conf_parser.get_option('wrapper', 'QUEUE', "") + + def get_jobs_sections(self): + """ + Returns the list of sections defined in the jobs config file + + :return: sections + :rtype: list + """ + return self._jobs_parser.sections() + + def get_copy_remote_logs(self): + """ + Returns if the user has enabled the logs local copy from autosubmit's config file + + :return: if logs local copy + :rtype: bool + """ + return self._conf_parser.get_option('storage', 'COPY_REMOTE_LOGS', 'true').lower() + + def get_mails_to(self): + """ + Returns the address where notifications will be sent from autosubmit's config file + + :return: mail address + :rtype: [str] + """ + return [str(x) for x in self._conf_parser.get_option('mail', 'TO', '').split(' ')] + + def get_communications_library(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + return self._conf_parser.get_option('communications', 'API', 'paramiko').lower() + + def get_storage_type(self): + """ + Returns the communications library from autosubmit's config file. Paramiko by default. + + :return: communications library + :rtype: str + """ + return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower() + + @staticmethod + def is_valid_mail_address(mail_address): + if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): + return True + else: + return False + + def is_valid_communications_library(self): + library = self.get_communications_library() + return library in ['paramiko', 'saga'] + + def is_valid_storage_type(self): + storage_type = self.get_storage_type() + return storage_type in ['pkl', 'db'] + + def is_valid_jobs_in_wrapper(self): + expression = self.get_wrapper_jobs() + if expression != 'None': + parser = self._jobs_parser + sections = parser.sections() + for section in expression.split(" "): + if "&" in section: + for inner_section in section.split("&"): + if inner_section not in sections: + return False + elif section not in sections: + return False + return True + + def is_valid_git_repository(self): + origin_exists = self._exp_parser.check_exists('git', 'PROJECT_ORIGIN') + branch = self.get_git_project_branch() + commit = self.get_git_project_commit() + return origin_exists and (branch is not None or commit is not None) + + @staticmethod + def get_parser(parser_factory, file_path): + # type: (ConfigParserFactory, str) -> ConfigParser + """ + Gets parser for given file + + :param parser_factory: + :param file_path: path to file to be parsed + :type file_path: str + :return: parser + :rtype: SafeConfigParser + """ + parser = parser_factory.create_parser() + parser.optionxform = str + # proj is not required + # print(file_path) + if file_path.find('proj_') > 0: + parser.read(file_path) + else: + with open(file_path) as f: + parser.read(file_path) + return parser diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index f91eea99..c70a75d3 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -38,6 +38,9 @@ from bscearth.utils.date import parse_date from bscearth.utils.log import Log from ..config.basicConfig import BasicConfig from ..config.ymlConfigParser import ymlConfigParser +from ..config.IConfigStrategy import IConfigStrategy +from ..config.ymlConfigStrategy import ymlConfigStrategy +from ..config.confConfigStrategy import confConfigStrategy logger = logging.getLogger('gunicorn.error') @@ -47,77 +50,33 @@ class AutosubmitConfig(object): :param expid: experiment identifier :type expid: str + :configWrapper: IConfigStrategy -> handling strategy for the type of config files used """ def __init__(self, expid, basic_config, parser_factory, extension=".yml"): # type: (str, BasicConfig, ConfigParserFactory, Extension) -> None - self.expid = expid - - self.basic_config = basic_config - - self.parser_factory = parser_factory + self.expid = expid + self._configWrapper = None # By default check for .yml files first as it is the new standard for AS 4.0 + if extension == ".yml": + self._configWrapper = ymlConfigStrategy(expid, basic_config, parser_factory, ".yml") + elif extension == ".conf": + self._configWrapper = confConfigStrategy(expid, basic_config, parser_factory, ".conf") - self._conf_parser = None # type: ConfigParser - self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "autosubmit_" + expid + extension) - if os.path.exists(self._conf_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self._exp_parser = None # type: ConfigParser - self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "expdef_" + expid + extension) - if os.path.exists(self._exp_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self._platforms_parser = None # type: ConfigParser - self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + extension) - if os.path.exists(self._platforms_parser_file) == False: - if extension == ".yml": - #self.__init__(expid, basic_config, parser_factory, ".conf") - # todo: this should be moved at the beginning - as_conf = Autosubmit4Config(self.expid) - # this loads all configurations - as_conf.reload(True) - self._platforms_data = as_conf.platforms_data - elif extension == ".conf": - return None - - - - self._jobs_parser = None # type: ConfigParser - self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "jobs_" + expid + extension) - if os.path.exists(self._jobs_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self._proj_parser = None # type: ConfigParser - self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "proj_" + expid + extension) - if os.path.exists(self._proj_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None - - self.check_proj_file() @property def jobs_parser(self): - return self._jobs_parser + return self._configWrapper.jobs_parser() @property def experiment_file(self): """ Returns experiment's config file name """ - return self._exp_parser_file + #return self._exp_parser_file + return self._configWrapper.experiment_file() + @property def platforms_parser(self): @@ -144,20 +103,12 @@ class AutosubmitConfig(object): """ Returns project's config file name """ - return self._proj_parser_file + return self._configWrapper.project_file() def check_proj_file(self): """ Add a section header to the project's configuration file (if not exists) """ - # if os.path.exists(self._proj_parser_file): - # with open(self._proj_parser_file, 'w') as f: - # first_line = f.readline() - # if not re.match('[[a-zA-Z0-9]*]', first_line): - # content = f.read() - # f.seek(0, 0) - # f.write('[DEFAULT]'.rstrip('\r\n') + - # '\n' + first_line + content) pass @property @@ -165,50 +116,19 @@ class AutosubmitConfig(object): """ Returns project's jobs file name """ - return self._jobs_parser_file + return self._configWrapper.jobs_file() def get_full_config_as_dict(self): """ Returns full configuration as json object """ - _conf = _exp = _platforms = _jobs = _proj = None - result = {} - - #def get_data(parser): - # """ - # dictionary comprehension to get data from parser - # """ - # res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ - # section for section in parser.sections()]} - # return res - - as_conf = Autosubmit4Config(self.expid) - # this loads all configurations - as_conf.reload(True) - self._platforms_data = as_conf.platforms_data - # print(self._conf_parser) - - result["conf"] = get_data( - self._conf_parser) if self._conf_parser else None - result["exp"] = get_data( - self._exp_parser) if self._exp_parser else None - result["platforms"] = self._platforms_data if self._platforms_data else None - result["jobs"] = get_data( - self._jobs_parser) if self._jobs_parser else None - result["proj"] = get_data( - self._proj_parser) if self._proj_parser else None - return result + return self._configWrapper.get_full_config_as_dict() def get_full_config_as_json(self): """ Return config as json object """ - try: - return json.dumps(self.get_full_config_as_dict()) - except Exception as exp: - Log.warning( - "Autosubmit was not able to retrieve and save the configuration into the historical database.") - return "" + return self._configWrapper.get_full_config_as_json() def get_project_dir(self): """ @@ -217,9 +137,8 @@ class AutosubmitConfig(object): :return: experiment's project directory :rtype: str """ - dir_templates = os.path.join(self.basic_config.LOCAL_ROOT_DIR, self.expid, BasicConfig.LOCAL_PROJ_DIR, - self.get_project_destination()) - return dir_templates + return self._configWrapper.get_project_dir() + def get_queue(self, section): """ @@ -229,26 +148,22 @@ class AutosubmitConfig(object): :return: queue :rtype: str """ - return self._jobs_parser.get_option(section, 'QUEUE', '') + return self._configWrapper.get_queue(section) def get_job_platform(self, section): - return self._jobs_parser.get_option(section, 'PLATFORM', '') + return self._configWrapper.get_job_platform(section) def get_platform_queue(self, platform): - return self._platforms_data[platform]["QUEUE"] - # return self._platforms_parser.get_option(platform, 'QUEUE', '') + return self._configWrapper.get_platform_queue(platform) def get_platform_serial_queue(self, platform): - return self._platforms_data[platform]["SERIAL_QUEUE"] - #return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '') + return self._configWrapper.get_platform_serial_queue(platform) def get_platform_project(self, platform): - return self._platforms_data[platform]["PROJECT"] - #return self._platforms_parser.get_option(platform, "PROJECT", "") + return self._configWrapper.get_platform_project(platform) def get_platform_wallclock(self, platform): - return self._platforms_data[platform]["MAX_WALLCLOCK"] - #return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '') + return self._configWrapper.get_platform_wallclock(platform) def get_wallclock(self, section): """ @@ -258,7 +173,7 @@ class AutosubmitConfig(object): :return: wallclock time :rtype: str """ - return self._jobs_parser.get_option(section, 'WALLCLOCK', '') + return self._configWrapper.get_wallclock(section) def get_synchronize(self, section): """ @@ -268,7 +183,7 @@ class AutosubmitConfig(object): :return: wallclock time :rtype: str """ - return self._jobs_parser.get_option(section, 'SYNCHRONIZE', '') + return self._configWrapper.get_synchronize(section) def get_processors(self, section): """ @@ -278,7 +193,7 @@ class AutosubmitConfig(object): :return: wallclock time :rtype: str """ - return str(self._jobs_parser.get_option(section, 'PROCESSORS', 1)) + return self._configWrapper.get_processors(section) def get_threads(self, section): """ @@ -288,7 +203,7 @@ class AutosubmitConfig(object): :return: threads needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'THREADS', 1)) + return self._configWrapper.get_threads(section) def get_tasks(self, section): """ @@ -298,7 +213,7 @@ class AutosubmitConfig(object): :return: tasks (processes) per host :rtype: str """ - return str(self._jobs_parser.get_option(section, 'TASKS', 0)) + return self._configWrapper.get_tasks(section) def get_scratch_free_space(self, section): """ @@ -308,7 +223,7 @@ class AutosubmitConfig(object): :return: percentage of scratch free space needed :rtype: int """ - return int(self._jobs_parser.get_option(section, 'SCRATCH_FREE_SPACE', 0)) + return self._configWrapper.get_scratch_free_space(section) def get_memory(self, section): """ @@ -318,7 +233,7 @@ class AutosubmitConfig(object): :return: memory needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'MEMORY', '')) + return self._configWrapper.get_memory(section) def get_memory_per_task(self, section): """ @@ -328,7 +243,7 @@ class AutosubmitConfig(object): :return: memory per task needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', '')) + return self._configWrapper.get_memory_per_task(section) def get_migrate_user_to(self, section): """ @@ -337,8 +252,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - #return self._platforms_parser.get_option(section, 'USER_TO', '').lower() - return str(self._platforms_data[section]["USER_TO"]).lower() + return self.get_migrate_user_to(section) def get_current_user(self, section): """ @@ -347,7 +261,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return str(self._platforms_data[section]["USER"]).lower() + return self._configWrapper.get_current_user(section) def get_current_project(self, section): """ @@ -356,7 +270,7 @@ class AutosubmitConfig(object): :return: migrate user to :rtype: str """ - return str(self._platforms_data[section]["USER"]).lower() + return self._configWrapper.get_current_project(section) def set_new_user(self, section, new_user): """ @@ -365,27 +279,7 @@ class AutosubmitConfig(object): :param section: platform name :type: str """ - with open(self._platforms_parser_file) as p_file: - contentLine = p_file.readline() - contentToMod = "" - content = "" - mod = False - while contentLine: - if re.search(section, contentLine): - mod = True - if mod: - contentToMod += contentLine - else: - content += contentLine - contentLine = p_file.readline() - if mod: - old_user = self.get_current_user(section) - contentToMod = contentToMod.replace(re.search( - r'[^#]\bUSER\b =.*', contentToMod).group(0)[1:], "USER = " + new_user) - contentToMod = contentToMod.replace(re.search( - r'[^#]\bUSER_TO\b =.*', contentToMod).group(0)[1:], "USER_TO = " + old_user) - open(self._platforms_parser_file, 'w').write(content) - open(self._platforms_parser_file, 'a').write(contentToMod) + self._configWrapper.set_new_user(section,new_user) def get_migrate_project_to(self, section): """ @@ -394,8 +288,7 @@ class AutosubmitConfig(object): :return: migrate project to :rtype: str """ - #return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower() - return str(self._platforms_data[section]["PROJECT_TO"]).lower() + return self._configWrapper.get_migrate_project_to(section) def set_new_project(self, section, new_project): @@ -405,27 +298,7 @@ class AutosubmitConfig(object): :param section: platform name :type: str """ - with open(self._platforms_parser_file) as p_file: - contentLine = p_file.readline() - contentToMod = "" - content = "" - mod = False - while contentLine: - if re.search(section, contentLine): - mod = True - if mod: - contentToMod += contentLine - else: - content += contentLine - contentLine = p_file.readline() - if mod: - old_project = self.get_current_project(section) - contentToMod = contentToMod.replace(re.search( - r"[^#]\bPROJECT\b =.*", contentToMod).group(0)[1:], "PROJECT = " + new_project) - contentToMod = contentToMod.replace(re.search( - r"[^#]\bPROJECT_TO\b =.*", contentToMod).group(0)[1:], "PROJECT_TO = " + old_project) - open(self._platforms_parser_file, 'w').write(content) - open(self._platforms_parser_file, 'a').write(contentToMod) + self._configWrapper.set_new_project(section,new_project) def get_custom_directives(self, section): """ @@ -435,7 +308,7 @@ class AutosubmitConfig(object): :return: custom directives needed :rtype: str """ - return str(self._jobs_parser.get_option(section, 'CUSTOM_DIRECTIVES', '')) + return self._configWrapper.get_custom_directives(section) def check_conf_files(self): """ @@ -445,18 +318,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it finds any error :rtype: bool """ - Log.debug('\nChecking configuration files...') - self.reload() - # result = self.check_platforms_conf() - result = True - result = result and self.check_jobs_conf() - result = result and self.check_autosubmit_conf() - result = result and self.check_expdef_conf() - if result: - Log.debug("Configuration files OK\n") - else: - Log.error("Configuration files invalid\n") - return result + return self._configWrapper.check_conf_files() def check_autosubmit_conf(self): """ @@ -465,42 +327,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - - self._conf_parser.read(self._conf_parser_file) - # result = result and self._conf_parser.check_exists( - # 'config', 'AUTOSUBMIT_VERSION') - # result = result and self._conf_parser.check_is_int( - # 'config', 'MAXWAITINGJOBS', True) - # result = result and self._conf_parser.check_is_int( - # 'config', 'TOTALJOBS', True) - # result = result and self._conf_parser.check_is_int( - # 'config', 'SAFETYSLEEPTIME', True) - # result = result and self._conf_parser.check_is_int( - # 'config', 'RETRIALS', True) - # result = result and self._conf_parser.check_is_boolean( - # 'mail', 'NOTIFICATIONS', False) - result = result and self.is_valid_communications_library() - result = result and self.is_valid_storage_type() - if self.get_wrapper_type() != 'None': - result = result and self.check_wrapper_conf() - - if self.get_notifications() == 'true': - for mail in self.get_mails_to(): - if not self.is_valid_mail_address(mail): - # Log.warning( - # 'One or more of the email addresses configured for the mail notifications are wrong') - break - - if not result: - # Log.critical("{0} is not a valid config file".format( - # os.path.basename(self._conf_parser_file))) - raise Exception("Permission denied for " + - str(os.path.basename(self._conf_parser_file))) - else: - Log.debug('{0} OK'.format( - os.path.basename(self._conf_parser_file))) - return result + return self._configWrapper.check_autosubmit_conf() def check_platforms_conf(self): """ @@ -509,38 +336,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - # self._platforms_data.keys() == 0 - #if len(self._platforms_parser.sections()) == 0: - if len(self._platforms_data.keys())== 0: - Log.warning("No remote platforms configured") - - if len(self._platforms_data.keys()) != len(set(self._platforms_data.keys())): - Log.error('There are repeated platforms names') - - for section in self._platforms_data.keys(): - # get the current platform for checking their values - platform = self._platforms_data[section] - - result = result and platform.get('TYPE') is not None - platform_type = platform.get('TYPE').lower() - if platform_type != 'ps': - result = result and platform.get('PROJECT') is not None - result = result and platform.get('USER') is not None - result = result and platform.get('HOST') is not None - result = result and platform.get('SCRATCH_DIR') is not None - result = result and platform.get('ADD_PROJECT_TO_HOST').lower() in ['false', 'true'] - result = result and platform.get('TEST_SUITE').lower() in ['false', 'true'] - - if not result: - Log.critical("{0} is not a valid config file".format( - os.path.basename(self._platforms_parser_file))) - raise Exception("Permission denied for " + - str(os.path.basename(self._platforms_parser_file))) - else: - Log.info('{0} OK'.format( - os.path.basename(self._platforms_parser_file))) - return result + return self._configWrapper.check_platforms_conf() def check_jobs_conf(self): """ @@ -549,66 +345,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - parser = self._jobs_parser - sections = parser.sections() - #platforms = self._platforms_parser.sections() - platforms = self._platforms_data.keys() - platforms.append('LOCAL') - possible_exception = "" - if len(sections) == 0: - possible_exception += "No remote platforms configured\n" - - # if len(sections) != len(set(sections)): - # Log.error('There are repeated job names') - - for section in sections: - - result = result and parser.check_exists(section, 'FILE') - result = result and parser.check_is_boolean( - section, 'RERUN_ONLY', False) - - if parser.has_option(section, 'PLATFORM'): - # result = result and parser.check_is_choice( - # section, 'PLATFORM', False, platforms) - pass - if parser.has_option(section, 'DEPENDENCIES'): - for dependency in str(parser.get_option(section, 'DEPENDENCIES', '')).split(' '): - if '-' in dependency: - dependency = dependency.split('-')[0] - elif '+' in dependency: - dependency = dependency.split('+')[0] - if '[' in dependency: - dependency = dependency[:dependency.find('[')] - # if dependency not in sections: - # Log.error( - # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, - # dependency)) - - if parser.has_option(section, 'RERUN_DEPENDENCIES'): - for dependency in str(parser.get_option(section, 'RERUN_DEPENDENCIES', - '')).split(' '): - if '-' in dependency: - dependency = dependency.split('-')[0] - if '[' in dependency: - dependency = dependency[:dependency.find('[')] - # if dependency not in sections: - # Log.error( - # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section, - # dependency)) - result = result and parser.check_is_choice(section, 'RUNNING', False, - ['once', 'date', 'member', 'chunk']) - - if not result: - # Log.critical("{0} is not a valid config file".format( - # os.path.basename(self._jobs_parser_file))) - raise Exception("Exception while checking jobs_expid.conf " + - str(os.path.basename(self._jobs_parser_file)) + possible_exception) - else: - Log.debug('{0} OK'.format( - os.path.basename(self._jobs_parser_file))) - - return result + return self._configWrapper.check_jobs_conf() def check_expdef_conf(self): """ @@ -617,58 +354,7 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - result = True - parser = self._exp_parser - - # result = result and parser.check_exists('DEFAULT', 'EXPID') - # result = result and parser.check_exists('DEFAULT', 'HPCARCH') - - result = result and parser.check_exists('experiment', 'DATELIST') - result = result and parser.check_exists('experiment', 'MEMBERS') - result = result and parser.check_is_choice('experiment', 'CHUNKSIZEUNIT', True, - ['year', 'month', 'day', 'hour']) - result = result and parser.check_is_int( - 'experiment', 'CHUNKSIZE', True) - result = result and parser.check_is_int( - 'experiment', 'NUMCHUNKS', True) - result = result and parser.check_is_choice('experiment', 'CALENDAR', True, - ['standard', 'noleap']) - - result = result and parser.check_is_boolean('rerun', 'RERUN', True) - - if parser.check_is_choice('project', 'PROJECT_TYPE', True, - ['none', 'git', 'svn', 'local']): - project_type = parser.get_option('project', 'PROJECT_TYPE', '') - - if project_type == 'git': - result = result and parser.check_exists( - 'git', 'PROJECT_ORIGIN') - result = result and parser.check_exists( - 'git', 'PROJECT_BRANCH') - - elif project_type == 'svn': - result = result and parser.check_exists('svn', 'PROJECT_URL') - result = result and parser.check_exists( - 'svn', 'PROJECT_REVISION') - elif project_type == 'local': - result = result and parser.check_exists( - 'local', 'PROJECT_PATH') - - if project_type != 'none': - result = result and parser.check_exists( - 'project_files', 'FILE_PROJECT_CONF') - else: - result = True - - if not result: - # Log.critical("{0} is not a valid config file".format( - # os.path.basename(self._exp_parser_file))) - raise Exception("Permission denied for " + - str(os.path.basename(self._exp_parser_file))) - else: - Log.debug('{0} OK'.format( - os.path.basename(self._exp_parser_file))) - return result + return self._configWrapper.check_expdef_conf() def check_proj(self): """ @@ -677,76 +363,16 @@ class AutosubmitConfig(object): :return: True if everything is correct, False if it founds any error :rtype: bool """ - try: - if self._proj_parser_file == '': - self._proj_parser = None - else: - self._proj_parser = AutosubmitConfig.get_parser( - self.parser_factory, self._proj_parser_file) - return True - except Exception as e: - Log.error('Project conf file error: {0}', e) - return False + return self._configWrapper.check_proj() def check_wrapper_conf(self): - result = True - result = result and self.is_valid_jobs_in_wrapper() - if not result: - Log.error( - "There are sections in JOBS_IN_WRAPPER that are not defined in your jobs.conf file") - - platform = self._platforms_data[self.get_platform()] - if 'horizontal' in self.get_wrapper_type(): - result = result and platform.get('PROCESSORS_PER_NODE') is not None - result = result and platform.get('MAX_PROCESSORS') is not None - - if 'vertical' in self.get_wrapper_type(): - result = result and platform.get('MAX_WALLCLOCK') is not None - - return result + return def reload(self): """ Creates parser objects for configuration files """ - logger.info(str(self._conf_parser_file)) - is_as4 = self._conf_parser_file.endswith('.yml') or self._conf_parser_file.endswith('.yaml') - - #todo: remove internal parser object and work directly with the data loaded - # try first with only one conf type so we check how it goes for both - - if is_as4: - logger.info("LOADING Configuration - for AS4 !!!!!") - as_conf = Autosubmit4Config(self.expid) - # this loads all configurations - as_conf.reload(True) - # log the content - logger.info(as_conf.platforms_data) - logger.info(as_conf.experiment_data) - logger.info(as_conf.jobs_data) - - self._platforms_data = as_conf.platforms_data - # testing getting host property from platform config file - # todo: here should be implemented some of the changes for supporting yaml files - logger.info(as_conf.platforms_data["MARENOSTRUM4"]["HOST"]) - #sample - #self.load_config_file(non_minimal_conf, Path(filename))) - else: - logger.info("LOADING Configuration - for AS3 !!!!!") - # checking existence of config files - if not os.path.exists(self._conf_parser_file): raise IOError( "Required file not found {0}".format(self._conf_parser_file)) - if not os.path.exists(self._platforms_parser_file): raise IOError( "Required file not found {0}".format(self._platforms_parser_file)) - if not os.path.exists(self._jobs_parser_file): raise IOError( "Required file not found {0}".format(self._jobs_parser_file)) - if not os.path.exists(self._exp_parser_file): raise IOError( "Required file not found {0}".format(self._exp_parser_file)) - - self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file) - #self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) - self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file) - self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file) - if self._proj_parser_file == '': - self._proj_parser = None - else: - self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file) + self._configWrapper.reload() def load_parameters(self): @@ -757,23 +383,7 @@ class AutosubmitConfig(object): :return: a dictionary containing tuples [parameter_name, parameter_value] :rtype: dict """ - parameters = dict() - for section in self._exp_parser.sections(): - for option in self._exp_parser.options(section): - parameters[option] = self._exp_parser.get(section, option) - for section in self._conf_parser.sections(): - for option in self._conf_parser.options(section): - parameters[option] = self._conf_parser.get(section, option) - - # project_type = self.get_project_type() - # if project_type != "none" and self._proj_parser is not None: - # # Load project parameters - # Log.debug("Loading project parameters...") - # parameters2 = parameters.copy() - # parameters2.update(self.load_project_parameters()) - # parameters = parameters2 - - return parameters + return self._configWrapper.load_parameters() def load_project_parameters(self): """ @@ -782,17 +392,7 @@ class AutosubmitConfig(object): :return: dictionary containing tuples [parameter_name, parameter_value] :rtype: dict """ - pass - # projdef = [] - # for section in self._proj_parser.sections(): - # print("SEction" + section) - # projdef += self._proj_parser.items(section) - - # parameters = dict() - # for item in projdef: - # parameters[item[0]] = item[1] - - # return parameters + return self._configWrapper.load_project_parameters() def set_expid(self, exp_id): """ @@ -801,18 +401,7 @@ class AutosubmitConfig(object): :param exp_id: experiment identifier to store :type exp_id: str """ - # Experiment conf - content = open(self._exp_parser_file).read() - if re.search('EXPID =.*', content): - content = content.replace( - re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) - open(self._exp_parser_file, 'w').write(content) - - content = open(self._conf_parser_file).read() - if re.search('EXPID =.*', content): - content = content.replace( - re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id) - open(self._conf_parser_file, 'w').write(content) + self._configWrapper.set_expid(exp_id) def get_project_type(self): """ @@ -821,7 +410,7 @@ class AutosubmitConfig(object): :return: project type :rtype: str """ - return self._exp_parser.get_option('project', 'PROJECT_TYPE', "NA").lower() + return self._configWrapper.get_project_type() def get_file_project_conf(self): """ @@ -830,7 +419,7 @@ class AutosubmitConfig(object): :return: path to project config file :rtype: str """ - return self._exp_parser.get('project_files', 'FILE_PROJECT_CONF') + return self._configWrapper.get_file_project_conf() def get_file_jobs_conf(self): """ @@ -839,7 +428,8 @@ class AutosubmitConfig(object): :return: path to project config file :rtype: str """ - return self._exp_parser.get_option('project_files', 'FILE_JOBS_CONF', '') + return self._configWrapper.get_file_jobs_conf() + def get_git_project_origin(self): """ @@ -848,7 +438,7 @@ class AutosubmitConfig(object): :return: git origin :rtype: str """ - return self._exp_parser.get_option('git', 'PROJECT_ORIGIN', '') + return self._configWrapper.get_git_project_origin() def get_git_project_branch(self): """ @@ -857,7 +447,7 @@ class AutosubmitConfig(object): :return: git branch :rtype: str """ - return self._exp_parser.get_option('git', 'PROJECT_BRANCH', 'master') + return self._configWrapper.get_git_project_branch() def get_git_project_commit(self): """ @@ -866,7 +456,7 @@ class AutosubmitConfig(object): :return: git commit :rtype: str """ - return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None) + return self._configWrapper.get_git_project_commit() def get_submodules_list(self): """ @@ -875,7 +465,7 @@ class AutosubmitConfig(object): :return: submodules to load :rtype: list """ - return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES', '').split()).split() + return self._configWrapper.get_submodules_list() def get_project_destination(self): """ @@ -884,16 +474,7 @@ class AutosubmitConfig(object): :return: git commit :rtype: str """ - value = self._exp_parser.get('project', 'PROJECT_DESTINATION') - if not value: - if self.get_project_type().lower() == "local": - value = os.path.split(self.get_local_project_path())[1] - elif self.get_project_type().lower() == "svn": - value = self.get_svn_project_url().split('/')[-1] - elif self.get_project_type().lower() == "git": - value = self.get_git_project_origin().split( - '/')[-1].split('.')[-2] - return value + return self._configWrapper.get_project_destination() def set_git_project_commit(self, as_conf): """ @@ -901,37 +482,7 @@ class AutosubmitConfig(object): :param as_conf: Configuration class for exteriment :type as_conf: AutosubmitConfig """ - full_project_path = as_conf.get_project_dir() - try: - output = subprocess.check_output("cd {0}; git rev-parse --abbrev-ref HEAD".format(full_project_path), - shell=True) - except subprocess.CalledProcessError: - Log.critical("Failed to retrieve project branch...") - return False - - project_branch = output - Log.debug("Project branch is: " + project_branch) - try: - output = subprocess.check_output( - "cd {0}; git rev-parse HEAD".format(full_project_path), shell=True) - except subprocess.CalledProcessError: - Log.critical("Failed to retrieve project commit SHA...") - return False - project_sha = output - Log.debug("Project commit SHA is: " + project_sha) - - # register changes - content = open(self._exp_parser_file).read() - if re.search('PROJECT_BRANCH =.*', content): - content = content.replace(re.search('PROJECT_BRANCH =.*', content).group(0), - "PROJECT_BRANCH = " + project_branch) - if re.search('PROJECT_COMMIT =.*', content): - content = content.replace(re.search('PROJECT_COMMIT =.*', content).group(0), - "PROJECT_COMMIT = " + project_sha) - open(self._exp_parser_file, 'w').write(content) - Log.debug( - "Project commit SHA succesfully registered to the configuration file.") - return True + return self._configWrapper.get_git_project_commit(as_conf) def get_svn_project_url(self): """ @@ -940,7 +491,7 @@ class AutosubmitConfig(object): :return: subversion project url :rtype: str """ - return self._exp_parser.get_option('svn', 'PROJECT_URL', 'NA') + return self._configWrapper.get_svn_project_url() def get_svn_project_revision(self): """ @@ -949,7 +500,7 @@ class AutosubmitConfig(object): :return: revision for subversion project :rtype: str """ - return self._exp_parser.get('svn', 'PROJECT_REVISION') + return self._configWrapper.get_svn_project_revision() def get_local_project_path(self): """ @@ -958,7 +509,7 @@ class AutosubmitConfig(object): :return: path to local project :rtype: str """ - return self._exp_parser.get('local', 'PROJECT_PATH') + return self._configWrapper.get_local_project_path() def get_date_list(self): """ @@ -967,30 +518,7 @@ class AutosubmitConfig(object): :return: experiment's startdates :rtype: list """ - date_list = list() - string = self._exp_parser.get('experiment', 'DATELIST') - if not string.startswith("["): - string = '[{0}]'.format(string) - split_string = nestedExpr('[', ']').parseString(string).asList() - string_date = None - for split in split_string[0]: - if type(split) is list: - for split_in in split: - if split_in.find("-") != -1: - numbers = split_in.split("-") - for count in range(int(numbers[0]), int(numbers[1]) + 1): - date_list.append(parse_date( - string_date + str(count).zfill(len(numbers[0])))) - else: - date_list.append(parse_date(string_date + split_in)) - string_date = None - else: - if string_date is not None: - date_list.append(parse_date(string_date)) - string_date = split - if string_date is not None: - date_list.append(parse_date(string_date)) - return date_list + return self._configWrapper.get_date_list() def get_num_chunks(self): """ @@ -999,7 +527,7 @@ class AutosubmitConfig(object): :return: number of chunks :rtype: int """ - return int(self._exp_parser.get('experiment', 'NUMCHUNKS')) + return self._configWrapper.get_num_chunks() def get_chunk_ini(self, default=1): """ @@ -1009,11 +537,7 @@ class AutosubmitConfig(object): :return: initial chunk :rtype: int """ - chunk_ini = self._exp_parser.get_option( - 'experiment', 'CHUNKINI', default) - if chunk_ini == '': - return default - return int(chunk_ini) + return self._configWrapper.get_chunk_ini(default) def get_chunk_size_unit(self): # type: () -> str @@ -1024,12 +548,7 @@ class AutosubmitConfig(object): :rtype: str """ - # try: - # res = self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() - # except Exception as e: - # try: - # res = self.autosubmit_conf.get('experiment') - return self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower() + return self._configWrapper.get_chunk_size_unit() def get_chunk_size(self, default=1): # type: (int) -> int @@ -1039,16 +558,7 @@ class AutosubmitConfig(object): :return: Chunksize, 1 as default. :rtype: int """ - try: - chunk_size = self._exp_parser.get_option( - 'experiment', 'CHUNKSIZE', default) - except Exception as exp: - print(exp) - chunk_size = '' - pass - if chunk_size == '': - return default - return int(chunk_size) + return self._configWrapper.get_chunk_size(default) def get_member_list(self, run_only=False): """ @@ -1057,31 +567,7 @@ class AutosubmitConfig(object): :return: experiment's members :rtype: list """ - member_list = list() - string = self._exp_parser.get('experiment', 'MEMBERS') if run_only == False else self._exp_parser.get_option( - 'experiment', 'RUN_ONLY_MEMBERS', '') - if not string.startswith("["): - string = '[{0}]'.format(string) - split_string = nestedExpr('[', ']').parseString(string).asList() - string_member = None - for split in split_string[0]: - if type(split) is list: - for split_in in split: - if split_in.find("-") != -1: - numbers = split_in.split("-") - for count in range(int(numbers[0]), int(numbers[1]) + 1): - member_list.append( - string_member + str(count).zfill(len(numbers[0]))) - else: - member_list.append(string_member + split_in) - string_member = None - else: - if string_member is not None: - member_list.append(string_member) - string_member = split - if string_member is not None: - member_list.append(string_member) - return member_list + return self._configWrapper.get_member_list(run_only) def get_rerun(self): """ @@ -1091,7 +577,7 @@ class AutosubmitConfig(object): :rtype: list """ - return self._exp_parser.get('rerun', 'RERUN').lower() + return self._configWrapper.get_rerun() def get_chunk_list(self): """ @@ -1100,7 +586,7 @@ class AutosubmitConfig(object): :return: experiment's chunks :rtype: list """ - return self._exp_parser.get('rerun', 'CHUNKLIST') + return self._configWrapper.get_chunk_list() def get_platform(self): """ @@ -1109,7 +595,7 @@ class AutosubmitConfig(object): :return: main platforms :rtype: str """ - return self._exp_parser.get('experiment', 'HPCARCH') + return self._configWrapper.get_platform() def set_platform(self, hpc): """ @@ -1118,11 +604,7 @@ class AutosubmitConfig(object): :param hpc: main platforms :type: str """ - content = open(self._exp_parser_file).read() - if re.search('HPCARCH =.*', content): - content = content.replace( - re.search('HPCARCH =.*', content).group(0), "HPCARCH = " + hpc) - open(self._exp_parser_file, 'w').write(content) + self._configWrapper.set_platform(hpc) def set_version(self, autosubmit_version): """ @@ -1131,11 +613,7 @@ class AutosubmitConfig(object): :param autosubmit_version: autosubmit's version :type autosubmit_version: str """ - content = open(self._conf_parser_file).read() - if re.search('AUTOSUBMIT_VERSION =.*', content): - content = content.replace(re.search('AUTOSUBMIT_VERSION =.*', content).group(0), - "AUTOSUBMIT_VERSION = " + autosubmit_version) - open(self._conf_parser_file, 'w').write(content) + return self._configWrapper.set_version(autosubmit_version) def get_version(self): """ @@ -1144,7 +622,7 @@ class AutosubmitConfig(object): :return: version :rtype: str """ - return self._conf_parser.get_option('config', 'AUTOSUBMIT_VERSION', 'None') + return self._configWrapper.get_version() def get_total_jobs(self): """ @@ -1153,7 +631,7 @@ class AutosubmitConfig(object): :return: max number of running jobs :rtype: int """ - return int(self._conf_parser.get('config', 'TOTALJOBS')) + return self._configWrapper.get_total_jobs() def get_max_wallclock(self): """ @@ -1161,7 +639,7 @@ class AutosubmitConfig(object): :rtype: str """ - return self._conf_parser.get_option('config', 'MAX_WALLCLOCK', '') + return self._configWrapper.get_max_wallclock() def get_max_processors(self): """ @@ -1169,9 +647,7 @@ class AutosubmitConfig(object): :rtype: str """ - config_value = self._conf_parser.get_option( - 'config', 'MAX_PROCESSORS', None) - return int(config_value) if config_value is not None else config_value + return self._configWrapper.get_max_processors() def get_max_waiting_jobs(self): """ @@ -1180,7 +656,7 @@ class AutosubmitConfig(object): :return: main platforms :rtype: int """ - return int(self._conf_parser.get_option('config', 'MAXWAITINGJOBS', 10)) + return self._configWrapper.get_max_waiting_jobs(self) def get_default_job_type(self): """ @@ -1189,7 +665,7 @@ class AutosubmitConfig(object): :return: default type such as bash, python, r.. :rtype: str """ - return self._exp_parser.get_option('project_files', 'JOB_SCRIPTS_TYPE', 'bash') + return self._configWrapper.get_default_job_type() def get_safetysleeptime(self): """ @@ -1198,7 +674,7 @@ class AutosubmitConfig(object): :return: safety sleep time :rtype: int """ - return int(self._conf_parser.get_option('config', 'SAFETYSLEEPTIME', 10)) + return self._configWrapper.get_safetysleeptime() def set_safetysleeptime(self, sleep_time): """ @@ -1207,10 +683,7 @@ class AutosubmitConfig(object): :param sleep_time: value to set :type sleep_time: int """ - content = open(self._conf_parser_file).read() - content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0), - "SAFETYSLEEPTIME = %d" % sleep_time) - open(self._conf_parser_file, 'w').write(content) + self._configWrapper.set_safetysleeptime(sleep_time) def get_retrials(self): """ @@ -1219,7 +692,7 @@ class AutosubmitConfig(object): :return: safety sleep time :rtype: int """ - return int(self._conf_parser.get('config', 'RETRIALS')) + return self._configWrapper.get_retrials() def get_notifications(self): """ @@ -1228,7 +701,7 @@ class AutosubmitConfig(object): :return: if notifications :rtype: string """ - return self._conf_parser.get_option('mail', 'NOTIFICATIONS', 'false').lower() + return self._configWrapper.get_notifications() def get_remote_dependencies(self): """ @@ -1237,7 +710,7 @@ class AutosubmitConfig(object): :return: if remote dependencies :rtype: bool """ - return self._conf_parser.get_option('wrapper', 'DEPENDENCIES', 'false').lower() == 'true' + return self._configWrapper.get_remote_dependencies() def get_wrapper_type(self): """ @@ -1246,7 +719,7 @@ class AutosubmitConfig(object): :return: wrapper type (or none) :rtype: string """ - return self._conf_parser.get_option('wrapper', 'TYPE', 'None').lower() + return self._configWrapper.get_wrapper_type() def get_wrapper_jobs(self): """ @@ -1255,7 +728,7 @@ class AutosubmitConfig(object): :return: expression (or none) :rtype: string """ - return self._conf_parser.get_option('wrapper', 'JOBS_IN_WRAPPER', 'None') + return self._configWrapper.get_wrapper_jobs() def get_max_wrapped_jobs(self): """ @@ -1264,9 +737,7 @@ class AutosubmitConfig(object): :return: maximum number of jobs (or total jobs) :rtype: string """ - # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs())) - - return int(self._conf_parser.get_option('wrapper', 'MAX_WRAPPED', self.get_total_jobs())) + return self._configWrapper.get_max_wrapped_jobs() def get_wrapper_check_time(self): """ @@ -1275,7 +746,7 @@ class AutosubmitConfig(object): :return: wrapper check time :rtype: int """ - return int(self._conf_parser.get_option('wrapper', 'CHECK_TIME_WRAPPER', self.get_safetysleeptime())) + return self._configWrapper.get_wrapper_check_time() def get_wrapper_machinefiles(self): """ @@ -1284,7 +755,7 @@ class AutosubmitConfig(object): :return: machinefiles function to use :rtype: string """ - return self._conf_parser.get_option('wrapper', 'MACHINEFILES', '') + return self._configWrapper.get_wrapper_machinefiles() def get_wrapper_queue(self): """ @@ -1293,7 +764,7 @@ class AutosubmitConfig(object): :return: expression (or none) :rtype: string """ - return self._conf_parser.get_option('wrapper', 'QUEUE', "") + return self._configWrapper.get_wrapper_queue() def get_jobs_sections(self): """ @@ -1302,7 +773,7 @@ class AutosubmitConfig(object): :return: sections :rtype: list """ - return self._jobs_parser.sections() + return self._configWrapper.get_jobs_sections() def get_copy_remote_logs(self): """ @@ -1311,7 +782,7 @@ class AutosubmitConfig(object): :return: if logs local copy :rtype: bool """ - return self._conf_parser.get_option('storage', 'COPY_REMOTE_LOGS', 'true').lower() + return self._configWrapper.get_copy_remote_logs() def get_mails_to(self): """ @@ -1320,7 +791,7 @@ class AutosubmitConfig(object): :return: mail address :rtype: [str] """ - return [str(x) for x in self._conf_parser.get_option('mail', 'TO', '').split(' ')] + return self._configWrapper.get_mails_to() def get_communications_library(self): """ @@ -1329,7 +800,7 @@ class AutosubmitConfig(object): :return: communications library :rtype: str """ - return self._conf_parser.get_option('communications', 'API', 'paramiko').lower() + return self._configWrapper.get_communications_library() def get_storage_type(self): """ @@ -1338,45 +809,26 @@ class AutosubmitConfig(object): :return: communications library :rtype: str """ - return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower() + return self._configWrapper.get_storage_type() - @staticmethod - def is_valid_mail_address(mail_address): - if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address): - return True - else: - return False + def is_valid_mail_address(self, mail_address): + #TODO: push to parent class as static method + return self._configWrapper.is_valid_mail_address(mail_address) def is_valid_communications_library(self): - library = self.get_communications_library() - return library in ['paramiko', 'saga'] + return self._configWrapper.is_valid_communications_library() def is_valid_storage_type(self): - storage_type = self.get_storage_type() - return storage_type in ['pkl', 'db'] + return self._configWrapper.is_valid_storage_type() def is_valid_jobs_in_wrapper(self): - expression = self.get_wrapper_jobs() - if expression != 'None': - parser = self._jobs_parser - sections = parser.sections() - for section in expression.split(" "): - if "&" in section: - for inner_section in section.split("&"): - if inner_section not in sections: - return False - elif section not in sections: - return False - return True + return self._configWrapper.is_valid_jobs_in_wrapper() def is_valid_git_repository(self): - origin_exists = self._exp_parser.check_exists('git', 'PROJECT_ORIGIN') - branch = self.get_git_project_branch() - commit = self.get_git_project_commit() - return origin_exists and (branch is not None or commit is not None) + return self._configWrapper.is_valid_git_repository() - @staticmethod - def get_parser(parser_factory, file_path): + + def get_parser(self, parser_factory, file_path): # type: (ConfigParserFactory, str) -> ConfigParser """ Gets parser for given file @@ -1387,20 +839,6 @@ class AutosubmitConfig(object): :return: parser :rtype: SafeConfigParser """ - # todo: add here the yml parser depending of the file type - is_as4 = file_path.endswith('.yml') or file_path.endswith('.yaml') - if is_as4: - # load as4 parser - parser = ymlConfigParser() - else: - # load as3 parser - parser = parser_factory.create_parser() - parser.optionxform = str - # proj is not required - # print(file_path) - if file_path.find('proj_') > 0: - parser.read(file_path) - else: - with open(file_path) as f: - parser.read(file_path) - return parser + # TODO: this was static method, check usages + + return self._configWrapper.get_parser(parser_factory, file_path) \ No newline at end of file diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index b20fa63c..0d6b30a7 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . +from ..config.IConfigStrategy import IConfigStrategy try: # noinspection PyCompatibility from configparser import SafeConfigParser @@ -33,12 +34,6 @@ import json import logging import locale -from pyparsing import nestedExpr -from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser -from bscearth.utils.date import parse_date -from bscearth.utils.log import Log -from ..config.IConfigStrategy import IConfigStrategy - logger = logging.getLogger('gunicorn.error') class ymlConfigStrategy(IConfigStrategy): @@ -49,17 +44,293 @@ class ymlConfigStrategy(IConfigStrategy): :type expid: str """ def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + logger.info("Creating AS4 Parser !!!!!") + self.conf_parser = Autosubmit4Config(expid) + self.conf_parser.reload(True) + - @classmethod def jobs_parser(self): - raise NotImplementedError + pass - @classmethod def experiment_file(self): - """ - Returns experiment's config file name - """ - return self._exp_parser_file + pass + + def platforms_parser(self): + pass + + def platforms_file(self): + pass + + def project_file(self): + pass + + @classmethod + def check_proj_file(self): + super().check_proj_file() + + def jobs_file(self): + pass + + def get_full_config_as_dict(self): + pass + + def get_full_config_as_json(self): + pass + + def get_project_dir(self): + pass + + def get_queue(self, section): + pass + + def get_job_platform(self, section): + pass + + def get_platform_queue(self, platform): + pass + + def get_platform_serial_queue(self, platform): + pass + + def get_platform_project(self, platform): + pass + + def get_platform_wallclock(self, platform): + pass + + def get_wallclock(self, section): + return super().get_wallclock(section) + + def get_synchronize(self, section): + return super().get_synchronize(section) + + def get_processors(self, section): + return super().get_processors(section) + + def get_threads(self, section): + return super().get_threads(section) + + def get_tasks(self, section): + return super().get_tasks(section) + + def get_scratch_free_space(self, section): + return super().get_scratch_free_space(section) + + def get_memory(self, section): + return super().get_memory(section) + + def get_memory_per_task(self, section): + return super().get_memory_per_task(section) + + def get_migrate_user_to(self, section): + return super().get_migrate_user_to(section) + + def get_current_user(self, section): + return super().get_current_user(section) + + def get_current_project(self, section): + return super().get_current_project(section) + + def set_new_user(self, section, new_user): + super().set_new_user(section, new_user) + + def get_migrate_project_to(self, section): + return super().get_migrate_project_to(section) + + def set_new_project(self, section, new_project): + super().set_new_project(section, new_project) + + def get_custom_directives(self, section): + return super().get_custom_directives(section) + + def check_conf_files(self): + return super().check_conf_files() + + def check_autosubmit_conf(self): + return super().check_autosubmit_conf() + + def check_platforms_conf(self): + return super().check_platforms_conf() + + def check_jobs_conf(self): + return super().check_jobs_conf() + + def check_expdef_conf(self): + return super().check_expdef_conf() + + def check_proj(self): + return super().check_proj() + + def check_wrapper_conf(self): + super().check_wrapper_conf() + + def reload(self): + super().reload() + + def load_parameters(self): + return super().load_parameters() + + def load_project_parameters(self): + return super().load_project_parameters() + + def set_expid(self, exp_id): + super().set_expid(exp_id) + + def get_project_type(self): + return super().get_project_type() + + def get_file_project_conf(self): + return super().get_file_project_conf() + + def get_file_jobs_conf(self): + return super().get_file_jobs_conf() + + def get_git_project_origin(self): + return super().get_git_project_origin() + + def get_git_project_branch(self): + return super().get_git_project_branch() + + def get_git_project_commit(self): + return super().get_git_project_commit() + + def get_submodules_list(self): + return super().get_submodules_list() + + def get_project_destination(self): + return super().get_project_destination() + + def set_git_project_commit(self, as_conf): + super().set_git_project_commit(as_conf) + + def get_svn_project_url(self): + return super().get_svn_project_url() + + def get_svn_project_revision(self): + return super().get_svn_project_revision() + + def get_local_project_path(self): + return super().get_local_project_path() + + def get_date_list(self): + return super().get_date_list() + + def get_num_chunks(self): + return super().get_num_chunks() + + def get_chunk_ini(self, default=1): + return super().get_chunk_ini(default) + + def get_chunk_size_unit(self): + return super().get_chunk_size_unit() + + def get_chunk_size(self, default=1): + return super().get_chunk_size(default) + + def get_member_list(self, run_only=False): + return super().get_member_list(run_only) + + def get_rerun(self): + return super().get_rerun() + + def get_chunk_list(self): + return super().get_chunk_list() + + def get_platform(self): + return super().get_platform() + + def set_platform(self, hpc): + super().set_platform(hpc) + + def set_version(self, autosubmit_version): + super().set_version(autosubmit_version) + + def get_version(self): + return super().get_version() + + def get_total_jobs(self): + return super().get_total_jobs() + + def get_max_wallclock(self): + return super().get_max_wallclock() + + def get_max_processors(self): + return super().get_max_processors() + + def get_max_waiting_jobs(self): + return super().get_max_waiting_jobs() + + def get_default_job_type(self): + return super().get_default_job_type() + + def get_safetysleeptime(self): + return super().get_safetysleeptime() + + def set_safetysleeptime(self, sleep_time): + super().set_safetysleeptime(sleep_time) + + def get_retrials(self): + return super().get_retrials() + + def get_notifications(self): + return super().get_notifications() + + def get_remote_dependencies(self): + return super().get_remote_dependencies() + + def get_wrapper_type(self): + return super().get_wrapper_type() + + def get_wrapper_jobs(self): + return super().get_wrapper_jobs() + + def get_max_wrapped_jobs(self): + return super().get_max_wrapped_jobs() + + def get_wrapper_check_time(self): + return super().get_wrapper_check_time() + + def get_wrapper_machinefiles(self): + return super().get_wrapper_machinefiles() + + def get_wrapper_queue(self): + return super().get_wrapper_queue() + + def get_jobs_sections(self): + return super().get_jobs_sections() + + def get_copy_remote_logs(self): + return super().get_copy_remote_logs() + + def get_mails_to(self): + return super().get_mails_to() + + def get_communications_library(self): + return super().get_communications_library() + + def get_storage_type(self): + return super().get_storage_type() + + @staticmethod + def is_valid_mail_address(mail_address): + return super().is_valid_mail_address(mail_address) + + @classmethod + def is_valid_communications_library(self): + return super().is_valid_communications_library() @classmethod - def platforms_parser(self): \ No newline at end of file + def is_valid_storage_type(self): + return super().is_valid_storage_type() + + def is_valid_jobs_in_wrapper(self): + super().is_valid_jobs_in_wrapper() + + def is_valid_git_repository(self): + super().is_valid_git_repository() + + @staticmethod + def get_parser(parser_factory, file_path): + return super().get_parser(parser_factory, file_path) + + -- GitLab From 9437edfcced8f26c295320f5af5e1351a682da9c Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Thu, 11 May 2023 17:41:54 +0200 Subject: [PATCH 12/25] Some fixes after initial round of tests of the new approach - #12 --- autosubmit_api/config/IConfigStrategy.py | 33 +++++++++++++-------- autosubmit_api/config/confConfigStrategy.py | 3 ++ autosubmit_api/config/ymlConfigStrategy.py | 10 +++++-- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py index d568fe75..7d22fe2d 100644 --- a/autosubmit_api/config/IConfigStrategy.py +++ b/autosubmit_api/config/IConfigStrategy.py @@ -38,7 +38,6 @@ from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser from bscearth.utils.date import parse_date from bscearth.utils.log import Log from ..config.basicConfig import BasicConfig -from ..config.ymlConfigParser import ymlConfigParser from abc import ABC, abstractmethod @@ -86,14 +85,14 @@ class IConfigStrategy(ABC): """ Returns project's config file name """ - pass + pass @classmethod def check_proj_file(self): """ Add a section header to the project's configuration file (if not exists) """ - pass + pass @abstractmethod def jobs_file(self): @@ -128,7 +127,7 @@ class IConfigStrategy(ABC): :return: queue :rtype: str """ - pass + pass @abstractmethod def get_job_platform(self, section): @@ -265,7 +264,7 @@ class IConfigStrategy(ABC): :param section: platform name :type: str """ - + pass def get_migrate_project_to(self, section): """ @@ -274,6 +273,7 @@ class IConfigStrategy(ABC): :return: migrate project to :rtype: str """ + pass @@ -284,6 +284,7 @@ class IConfigStrategy(ABC): :param section: platform name :type: str """ + pass def get_custom_directives(self, section): @@ -294,6 +295,7 @@ class IConfigStrategy(ABC): :return: custom directives needed :rtype: str """ + pass def check_conf_files(self): @@ -304,7 +306,7 @@ class IConfigStrategy(ABC): :return: True if everything is correct, False if it finds any error :rtype: bool """ - + pass def check_autosubmit_conf(self): """ @@ -313,7 +315,7 @@ class IConfigStrategy(ABC): :return: True if everything is correct, False if it founds any error :rtype: bool """ - + pass def check_platforms_conf(self): """ @@ -331,6 +333,7 @@ class IConfigStrategy(ABC): :return: True if everything is correct, False if it founds any error :rtype: bool """ + pass def check_expdef_conf(self): @@ -340,6 +343,7 @@ class IConfigStrategy(ABC): :return: True if everything is correct, False if it founds any error :rtype: bool """ + pass def check_proj(self): @@ -349,15 +353,18 @@ class IConfigStrategy(ABC): :return: True if everything is correct, False if it founds any error :rtype: bool """ + pass def check_wrapper_conf(self): + pass def reload(self): """ Creates parser objects for configuration files """ + pass def load_parameters(self): """ @@ -717,21 +724,21 @@ class IConfigStrategy(ABC): pass def get_wrapper_check_time(self): - """ + """ Returns time to check the status of jobs in the wrapper :return: wrapper check time :rtype: int """ - pass + pass def get_wrapper_machinefiles(self): """ - Returns the strategy for creating the machinefiles in wrapper jobs + Returns the strategy for creating the machinefiles in wrapper jobs - :return: machinefiles function to use - :rtype: string - """ + :return: machinefiles function to use + :rtype: string + """ pass def get_wrapper_queue(self): diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py index 0a74d12c..7c38a233 100644 --- a/autosubmit_api/config/confConfigStrategy.py +++ b/autosubmit_api/config/confConfigStrategy.py @@ -20,9 +20,12 @@ try: # noinspection PyCompatibility from configparser import SafeConfigParser + from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser + + import os import re import subprocess diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 0d6b30a7..9696d65b 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -from ..config.IConfigStrategy import IConfigStrategy try: # noinspection PyCompatibility from configparser import SafeConfigParser @@ -26,13 +25,18 @@ except ImportError: # noinspection PyCompatibility from configparser import SafeConfigParser - import os import re import subprocess import json import logging -import locale + +from pyparsing import nestedExpr +from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser +from bscearth.utils.date import parse_date +from bscearth.utils.log import Log +from ..config.basicConfig import BasicConfig +from ..config.IConfigStrategy import IConfigStrategy logger = logging.getLogger('gunicorn.error') -- GitLab From 2afaf8bce48e8aa62e1e8a188d4b43a17d5cd991 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Fri, 12 May 2023 14:46:34 +0200 Subject: [PATCH 13/25] Add some fixes after debugging in the development VM - #12 --- autosubmit_api/config/confConfigStrategy.py | 10 +++++----- autosubmit_api/config/config_common.py | 11 ++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py index 0a74d12c..968181a5 100644 --- a/autosubmit_api/config/confConfigStrategy.py +++ b/autosubmit_api/config/confConfigStrategy.py @@ -706,14 +706,14 @@ class confConfigStrategy(IConfigStrategy): "Required file not found {0}".format(self._jobs_parser_file)) if not os.path.exists(self._exp_parser_file): raise IOError( "Required file not found {0}".format(self._exp_parser_file)) - self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file) - self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file) - self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file) - self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file) + self._conf_parser = confConfigStrategy.get_parser(self.parser_factory, self._conf_parser_file) + self._platforms_parser = confConfigStrategy.get_parser(self.parser_factory, self._platforms_parser_file) + self._jobs_parser = confConfigStrategy.get_parser(self.parser_factory, self._jobs_parser_file) + self._exp_parser = confConfigStrategy.get_parser(self.parser_factory, self._exp_parser_file) if self._proj_parser_file == '': self._proj_parser = None else: - self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file) + self._proj_parser = confConfigStrategy.get_parser(self.parser_factory, self._proj_parser_file) def load_parameters(self): """ diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index c70a75d3..0f7bcbb1 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -53,16 +53,17 @@ class AutosubmitConfig(object): :configWrapper: IConfigStrategy -> handling strategy for the type of config files used """ - def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + def __init__(self, expid, basic_config, parser_factory, extension="yml"): # type: (str, BasicConfig, ConfigParserFactory, Extension) -> None self.expid = expid self._configWrapper = None - # By default check for .yml files first as it is the new standard for AS 4.0 - if extension == ".yml": - self._configWrapper = ymlConfigStrategy(expid, basic_config, parser_factory, ".yml") - elif extension == ".conf": + # check which type of config files (AS3 or AS4) + platform_conf_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + extension) + if os.path.exists(platform_conf_file): self._configWrapper = confConfigStrategy(expid, basic_config, parser_factory, ".conf") + elif extension == "yml": + self._configWrapper = ymlConfigStrategy(expid, basic_config, parser_factory, ".yml") @property -- GitLab From 789a529c0bcc3f3d338b888757e579af0361b210 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Mon, 15 May 2023 17:06:27 +0200 Subject: [PATCH 14/25] More fixes - #12 --- autosubmit_api/config/config_common.py | 11 ++++++++--- autosubmit_api/config/ymlConfigStrategy.py | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 0f7bcbb1..2fa05897 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -58,12 +58,17 @@ class AutosubmitConfig(object): self.expid = expid self._configWrapper = None + self.basic_config = basic_config + self.parser_factory = parser_factory + # check which type of config files (AS3 or AS4) - platform_conf_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + extension) + platform_conf_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + "." + extension) if os.path.exists(platform_conf_file): - self._configWrapper = confConfigStrategy(expid, basic_config, parser_factory, ".conf") - elif extension == "yml": + logger.info("Setting AS4 Config strategy - yml") self._configWrapper = ymlConfigStrategy(expid, basic_config, parser_factory, ".yml") + else: + logger.info("Setting AS3 Config strategy - conf") + self._configWrapper = confConfigStrategy(expid, basic_config, parser_factory, ".conf") @property diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 9696d65b..3397c2dc 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -54,18 +54,23 @@ class ymlConfigStrategy(IConfigStrategy): def jobs_parser(self): + logger.info("Not yet implemented") pass def experiment_file(self): + logger.info("Not yet implemented") pass def platforms_parser(self): + logger.info("OBSOLOTED - Not yet implemented") pass def platforms_file(self): + logger.info("Not yet implemented") pass def project_file(self): + logger.info("Not yet implemented") pass @classmethod -- GitLab From ae4b753d1188f052267993da123c02b61618d639 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Mon, 15 May 2023 17:33:24 +0200 Subject: [PATCH 15/25] Add basic getters for platform part for yml strategy - #12 --- autosubmit_api/config/ymlConfigStrategy.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 3397c2dc..d7236221 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -49,8 +49,8 @@ class ymlConfigStrategy(IConfigStrategy): """ def __init__(self, expid, basic_config, parser_factory, extension=".yml"): logger.info("Creating AS4 Parser !!!!!") - self.conf_parser = Autosubmit4Config(expid) - self.conf_parser.reload(True) + self._conf_parser = Autosubmit4Config(expid) + self._conf_parser.reload(True) def jobs_parser(self): @@ -96,19 +96,20 @@ class ymlConfigStrategy(IConfigStrategy): pass def get_platform_queue(self, platform): - pass + return self._conf_parser.platforms_data[platform]["QUEUE"] def get_platform_serial_queue(self, platform): - pass + return self._conf_parser.platforms_data[platform]["SERIAL_QUEUE"] def get_platform_project(self, platform): - pass + return self._conf_parser.platforms_data[platform]["PROJECT"] def get_platform_wallclock(self, platform): - pass + return self._conf_parser.platforms_data[platform]["MAX_WALLCLOCK"] def get_wallclock(self, section): - return super().get_wallclock(section) + return str(self._conf_parser.platforms_data[section]["USER_TO"]).lower() + def get_synchronize(self, section): return super().get_synchronize(section) @@ -135,16 +136,16 @@ class ymlConfigStrategy(IConfigStrategy): return super().get_migrate_user_to(section) def get_current_user(self, section): - return super().get_current_user(section) + return str(self._conf_parser.platforms_data[section]["USER"]).lower() def get_current_project(self, section): - return super().get_current_project(section) + return str(self._conf_parser.platforms_data[section]["PROJECT"]).lower() def set_new_user(self, section, new_user): super().set_new_user(section, new_user) def get_migrate_project_to(self, section): - return super().get_migrate_project_to(section) + return str(self._conf_parser.platforms_data[section]["PROJECT_TO"]).lower() def set_new_project(self, section, new_project): super().set_new_project(section, new_project) -- GitLab From 3393834ab6a5099282a6a58e5588bbc31ec6bb1d Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Tue, 16 May 2023 12:23:23 +0200 Subject: [PATCH 16/25] small fix for getting custom directives for a job - #12 --- autosubmit_api/config/ymlConfigStrategy.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index d7236221..4e687196 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -151,7 +151,15 @@ class ymlConfigStrategy(IConfigStrategy): super().set_new_project(section, new_project) def get_custom_directives(self, section): - return super().get_custom_directives(section) + """ + Gets custom directives needed for the given job type + :param section: job type + :type section: str + :return: custom directives needed + :rtype: str + """ + return str(self._conf_parser.jobs_data.get(section, {}).get('CUSTOM_DIRECTIVES', "")) + def check_conf_files(self): return super().check_conf_files() -- GitLab From e184a7264384bcdecb03a87708a974455404f414 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Tue, 16 May 2023 16:24:28 +0200 Subject: [PATCH 17/25] More fixes for handling the data for the treeview - #12 --- autosubmit_api/config/ymlConfigStrategy.py | 32 ++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 4e687196..69eef71b 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -234,19 +234,41 @@ class ymlConfigStrategy(IConfigStrategy): return super().get_date_list() def get_num_chunks(self): - return super().get_num_chunks() + return self._conf_parser.get_num_chunks() def get_chunk_ini(self, default=1): - return super().get_chunk_ini(default) + return self._conf_parser.get_chunk_ini(default) def get_chunk_size_unit(self): - return super().get_chunk_size_unit() + """ + Unit for the chunk length + + :return: Unit for the chunk length Options: {hour, day, month, year} + :rtype: str + """ + return self._conf_parser.get_chunk_size_unit() + def get_chunk_size(self, default=1): - return super().get_chunk_size(default) + try: + chunk_size = self._conf_parser.get_chunk_size(default) + except Exception as exp: + print(exp) + chunk_size = '' + pass + if chunk_size == '': + return default + return int(chunk_size) def get_member_list(self, run_only=False): - return super().get_member_list(run_only) + #return super().get_member_list(run_only) + """ + Returns members list from experiment's config file + + :return: experiment's members + :rtype: list + """ + return self._conf_parser.get_member_list(run_only) def get_rerun(self): return super().get_rerun() -- GitLab From 12ccfd4ccf79f751eae6f82651d9f9f97e4cefb7 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Tue, 16 May 2023 17:33:41 +0200 Subject: [PATCH 18/25] Generalization of the calls using the functions of ConfigCommon from the lib - #12 --- autosubmit_api/config/ymlConfigStrategy.py | 150 +++++++++++---------- 1 file changed, 79 insertions(+), 71 deletions(-) diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 69eef71b..33787657 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -75,7 +75,7 @@ class ymlConfigStrategy(IConfigStrategy): @classmethod def check_proj_file(self): - super().check_proj_file() + self._conf_parser.check_proj_file() def jobs_file(self): pass @@ -105,35 +105,43 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.platforms_data[platform]["PROJECT"] def get_platform_wallclock(self, platform): - return self._conf_parser.platforms_data[platform]["MAX_WALLCLOCK"] + if not self._conf_parser.platforms_data[platform]: + logger.info("Platform " + platform + "doest exist/undefined") + elif not self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', ""): + return "" + else: + return self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', "") + + + def get_wallclock(self, section): - return str(self._conf_parser.platforms_data[section]["USER_TO"]).lower() + return self._conf_parser.jobs_data[section].get('WALLCLOCK', '') def get_synchronize(self, section): - return super().get_synchronize(section) + return self._conf_parser.get_synchronize(section) def get_processors(self, section): - return super().get_processors(section) + return self._conf_parser.get_processors(section) def get_threads(self, section): - return super().get_threads(section) + return self._conf_parser.get_threads(section) def get_tasks(self, section): - return super().get_tasks(section) + return self._conf_parser.get_tasks(section) def get_scratch_free_space(self, section): - return super().get_scratch_free_space(section) + return self._conf_parser.get_scratch_free_space(section) def get_memory(self, section): - return super().get_memory(section) + return self._conf_parser.get_memory(section) def get_memory_per_task(self, section): - return super().get_memory_per_task(section) + return self._conf_parser.get_memory_per_task(section) def get_migrate_user_to(self, section): - return super().get_migrate_user_to(section) + return self._conf_parser.get_migrate_user_to(section) def get_current_user(self, section): return str(self._conf_parser.platforms_data[section]["USER"]).lower() @@ -142,13 +150,13 @@ class ymlConfigStrategy(IConfigStrategy): return str(self._conf_parser.platforms_data[section]["PROJECT"]).lower() def set_new_user(self, section, new_user): - super().set_new_user(section, new_user) + self._conf_parser.set_new_user(section, new_user) def get_migrate_project_to(self, section): return str(self._conf_parser.platforms_data[section]["PROJECT_TO"]).lower() def set_new_project(self, section, new_project): - super().set_new_project(section, new_project) + self._conf_parser.set_new_project(section, new_project) def get_custom_directives(self, section): """ @@ -162,76 +170,76 @@ class ymlConfigStrategy(IConfigStrategy): def check_conf_files(self): - return super().check_conf_files() + return self._conf_parser.check_conf_files() def check_autosubmit_conf(self): - return super().check_autosubmit_conf() + return self._conf_parser.check_autosubmit_conf() def check_platforms_conf(self): - return super().check_platforms_conf() + return self._conf_parser.check_platforms_conf() def check_jobs_conf(self): - return super().check_jobs_conf() + return self._conf_parser.check_jobs_conf() def check_expdef_conf(self): - return super().check_expdef_conf() + return self._conf_parser.check_expdef_conf() def check_proj(self): - return super().check_proj() + return self._conf_parser.check_proj() def check_wrapper_conf(self): - super().check_wrapper_conf() + self._conf_parser.check_wrapper_conf() def reload(self): - super().reload() + self._conf_parser.reload() def load_parameters(self): - return super().load_parameters() + return self._conf_parser.load_parameters() def load_project_parameters(self): - return super().load_project_parameters() + return self._conf_parser.load_project_parameters() def set_expid(self, exp_id): - super().set_expid(exp_id) + self._conf_parser.set_expid(exp_id) def get_project_type(self): - return super().get_project_type() + return self._conf_parser.get_project_type() def get_file_project_conf(self): - return super().get_file_project_conf() + return self._conf_parser.get_file_project_conf() def get_file_jobs_conf(self): - return super().get_file_jobs_conf() + return self._conf_parser.get_file_jobs_conf() def get_git_project_origin(self): - return super().get_git_project_origin() + return self._conf_parser.get_git_project_origin() def get_git_project_branch(self): - return super().get_git_project_branch() + return self._conf_parser.get_git_project_branch() def get_git_project_commit(self): - return super().get_git_project_commit() + return self._conf_parser.get_git_project_commit() def get_submodules_list(self): - return super().get_submodules_list() + return self._conf_parser.get_submodules_list() def get_project_destination(self): - return super().get_project_destination() + return self._conf_parser.get_project_destination() def set_git_project_commit(self, as_conf): - super().set_git_project_commit(as_conf) + self._conf_parser.set_git_project_commit(as_conf) def get_svn_project_url(self): - return super().get_svn_project_url() + return self._conf_parser.get_svn_project_url() def get_svn_project_revision(self): - return super().get_svn_project_revision() + return self._conf_parser.get_svn_project_revision() def get_local_project_path(self): - return super().get_local_project_path() + return self._conf_parser.get_local_project_path() def get_date_list(self): - return super().get_date_list() + return self._conf_parser.get_date_list() def get_num_chunks(self): return self._conf_parser.get_num_chunks() @@ -261,7 +269,7 @@ class ymlConfigStrategy(IConfigStrategy): return int(chunk_size) def get_member_list(self, run_only=False): - #return super().get_member_list(run_only) + #return self._conf_parser.get_member_list(run_only) """ Returns members list from experiment's config file @@ -271,106 +279,106 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.get_member_list(run_only) def get_rerun(self): - return super().get_rerun() + return self._conf_parser.get_rerun() def get_chunk_list(self): - return super().get_chunk_list() + return self._conf_parser.get_chunk_list() def get_platform(self): - return super().get_platform() + return self._conf_parser.get_platform() def set_platform(self, hpc): - super().set_platform(hpc) + self._conf_parser.set_platform(hpc) def set_version(self, autosubmit_version): - super().set_version(autosubmit_version) + self._conf_parser.set_version(autosubmit_version) def get_version(self): - return super().get_version() + return self._conf_parser.get_version() def get_total_jobs(self): - return super().get_total_jobs() + return self._conf_parser.get_total_jobs() def get_max_wallclock(self): - return super().get_max_wallclock() + return self._conf_parser.get_max_wallclock() def get_max_processors(self): - return super().get_max_processors() + return self._conf_parser.get_max_processors() def get_max_waiting_jobs(self): - return super().get_max_waiting_jobs() + return self._conf_parser.get_max_waiting_jobs() def get_default_job_type(self): - return super().get_default_job_type() + return self._conf_parser.get_default_job_type() def get_safetysleeptime(self): - return super().get_safetysleeptime() + return self._conf_parser.get_safetysleeptime() def set_safetysleeptime(self, sleep_time): - super().set_safetysleeptime(sleep_time) + self._conf_parser.set_safetysleeptime(sleep_time) def get_retrials(self): - return super().get_retrials() + return self._conf_parser.get_retrials() def get_notifications(self): - return super().get_notifications() + return self._conf_parser.get_notifications() def get_remote_dependencies(self): - return super().get_remote_dependencies() + return self._conf_parser.get_remote_dependencies() def get_wrapper_type(self): - return super().get_wrapper_type() + return self._conf_parser.get_wrapper_type() def get_wrapper_jobs(self): - return super().get_wrapper_jobs() + return self._conf_parser.get_wrapper_jobs() def get_max_wrapped_jobs(self): - return super().get_max_wrapped_jobs() + return self._conf_parser.get_max_wrapped_jobs() def get_wrapper_check_time(self): - return super().get_wrapper_check_time() + return self._conf_parser.get_wrapper_check_time() def get_wrapper_machinefiles(self): - return super().get_wrapper_machinefiles() + return self._conf_parser.get_wrapper_machinefiles() def get_wrapper_queue(self): - return super().get_wrapper_queue() + return self._conf_parser.get_wrapper_queue() def get_jobs_sections(self): - return super().get_jobs_sections() + return self._conf_parser.get_jobs_sections() def get_copy_remote_logs(self): - return super().get_copy_remote_logs() + return self._conf_parser.get_copy_remote_logs() def get_mails_to(self): - return super().get_mails_to() + return self._conf_parser.get_mails_to() def get_communications_library(self): - return super().get_communications_library() + return self._conf_parser.get_communications_library() def get_storage_type(self): - return super().get_storage_type() + return self._conf_parser.get_storage_type() @staticmethod def is_valid_mail_address(mail_address): - return super().is_valid_mail_address(mail_address) + return self._conf_parser.is_valid_mail_address(mail_address) @classmethod def is_valid_communications_library(self): - return super().is_valid_communications_library() + return self._conf_parser.is_valid_communications_library() @classmethod def is_valid_storage_type(self): - return super().is_valid_storage_type() + return self._conf_parser.is_valid_storage_type() def is_valid_jobs_in_wrapper(self): - super().is_valid_jobs_in_wrapper() + self._conf_parser.is_valid_jobs_in_wrapper() def is_valid_git_repository(self): - super().is_valid_git_repository() + self._conf_parser.is_valid_git_repository() @staticmethod def get_parser(parser_factory, file_path): - return super().get_parser(parser_factory, file_path) + return Autosubmit4Config.get_parser(parser_factory, file_path) -- GitLab From a0b5886c3ab337dba615e9db564323cf39d07027 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Wed, 17 May 2023 17:23:01 +0200 Subject: [PATCH 19/25] fixed treview and graph visualizations for Autosubmit 4 experiments - #12 --- autosubmit_api/components/jobs/joblist_loader.py | 10 ++++++++-- autosubmit_api/config/ymlConfigStrategy.py | 10 +--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/autosubmit_api/components/jobs/joblist_loader.py b/autosubmit_api/components/jobs/joblist_loader.py index 50a94d89..88abb2a1 100644 --- a/autosubmit_api/components/jobs/joblist_loader.py +++ b/autosubmit_api/components/jobs/joblist_loader.py @@ -14,6 +14,10 @@ from typing import Dict, List, Set, Tuple from ...config.config_common import AutosubmitConfig from ...config.basicConfig import BasicConfig import json +import logging + + +logger = logging.getLogger('gunicorn.error') class JobListLoader(object): """ Class that manages loading the list of jobs from the pkl. Adds other resources. """ @@ -187,14 +191,16 @@ class JobListLoader(object): else: job_qos = self.configuration_facade.get_section_qos(job.section) if len(job_qos.strip()) == 0: - job_qos = self.configuration_facade.get_platform_qos(job.platform, job.ncpus) + if job.platform != "None": + job_qos = self.configuration_facade.get_platform_qos(job.platform, job.ncpus) return job_qos def _determine_wallclock(self, job): # type: (Job) -> None wallclock = self.configuration_facade.get_section_wallclock(job.section) if len(wallclock.strip()) == 0: - wallclock = self.configuration_facade.get_platform_max_wallclock(job.platform) + if job.platform != "None": + wallclock = self.configuration_facade.get_platform_max_wallclock(job.platform) return wallclock def assign_packages_to_jobs(self): diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 33787657..3e28e1f0 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -105,15 +105,7 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.platforms_data[platform]["PROJECT"] def get_platform_wallclock(self, platform): - if not self._conf_parser.platforms_data[platform]: - logger.info("Platform " + platform + "doest exist/undefined") - elif not self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', ""): - return "" - else: - return self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', "") - - - + return self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', "") def get_wallclock(self, section): return self._conf_parser.jobs_data[section].get('WALLCLOCK', '') -- GitLab From 05f68861e76e989bbd631d7b0fb2117bc3ccd6d6 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Thu, 18 May 2023 17:30:48 +0200 Subject: [PATCH 20/25] changes for configuration (still not working) - #12 --- autosubmit_api/config/ymlConfigStrategy.py | 29 ++++++++++++++++---- autosubmit_api/experiment/common_requests.py | 3 +- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 3e28e1f0..d1e43cfb 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -52,7 +52,6 @@ class ymlConfigStrategy(IConfigStrategy): self._conf_parser = Autosubmit4Config(expid) self._conf_parser.reload(True) - def jobs_parser(self): logger.info("Not yet implemented") pass @@ -81,16 +80,36 @@ class ymlConfigStrategy(IConfigStrategy): pass def get_full_config_as_dict(self): - pass + """ + Returns full configuration as json object + """ + _conf = _exp = _platforms = _jobs = _proj = None + result = {} + + def get_data(parser): + """ + dictionary comprehension to get data from parser + """ + res = {sec: {option: parser.get_section(sec, option) for option in parser.options(sec)} for sec in [ + section for section in parser.sections()]} + return res + + # print(self._conf_parser) + #result["conf"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._conf_parser else None + #result["exp"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._exp_parser else None + result["platforms"] = get_data( self._conf_parser.experiment_data["PLATFORMS"]) if self._conf_parser.experiment_data["PLATFORMS"] else None + result["jobs"] = get_data( self._conf_parser.experiment_data["JOBS"]) if self._conf_parser.experiment_data["JOBS"] else None + #result["proj"] = get_data( self._conf_parser.experiment_data["CONF"] ) if self._proj_parser else None + return result def get_full_config_as_json(self): - pass + return self._conf_parser.get_full_config_as_json() def get_project_dir(self): - pass + return self._conf_parser.get_project_dir() def get_queue(self, section): - pass + return self._conf_parser.get_queue() def get_job_platform(self, section): pass diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index eb0198f0..c0000a2b 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -1121,6 +1121,7 @@ def get_auto_conf_data(expid): max_wrapped = as_conf.get_max_wrapped_jobs() return (wrapper_type, max_wrapped) except Exception as ex: + logger.info(traceback.format_exc()) print(("Couldn't retrieve conf data (wrapper info) from {0}. Exception {1}.".format(expid, str(ex)))) return ("None", 0) @@ -1382,7 +1383,7 @@ def get_current_configuration_by_expid(expid, valid_user, log): warning = True warning_message = "The filesystem system configuration can't be retrieved because '{}'".format( exp) - log.info((traceback.format_exc())) + logger.info(traceback.format_exc()) currentFileSystemConfig["contains_nones"] = True log.info(warning_message) pass -- GitLab From 511fed8e2d2ef50d24aba350ec0dfe0c2e38eabe Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Fri, 19 May 2023 16:13:22 +0200 Subject: [PATCH 21/25] Configuration tab for platform part seems to work for AS4 experiments - #12 --- autosubmit_api/config/IConfigStrategy.py | 7 -- autosubmit_api/config/confConfigStrategy.py | 42 ++--------- autosubmit_api/config/ymlConfigStrategy.py | 81 ++++++++++++++++----- 3 files changed, 67 insertions(+), 63 deletions(-) diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py index 7d22fe2d..0fbc61fa 100644 --- a/autosubmit_api/config/IConfigStrategy.py +++ b/autosubmit_api/config/IConfigStrategy.py @@ -87,13 +87,6 @@ class IConfigStrategy(ABC): """ pass - @classmethod - def check_proj_file(self): - """ - Add a section header to the project's configuration file (if not exists) - """ - pass - @abstractmethod def jobs_file(self): """ diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py index bfa62067..b173f1fe 100644 --- a/autosubmit_api/config/confConfigStrategy.py +++ b/autosubmit_api/config/confConfigStrategy.py @@ -49,7 +49,7 @@ class confConfigStrategy(IConfigStrategy): :type expid: str """ - def __init__(self, expid, basic_config, parser_factory, extension=".yml"): + def __init__(self, expid, basic_config, parser_factory, extension=".conf"): # type: (str, BasicConfig, ConfigParserFactory, Extension) -> None self.expid = expid @@ -61,48 +61,32 @@ class confConfigStrategy(IConfigStrategy): self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "autosubmit_" + expid + extension) if os.path.exists(self._conf_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None + return None self._exp_parser = None # type: ConfigParser self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "expdef_" + expid + extension) if os.path.exists(self._exp_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None + return None self._platforms_parser = None # type: ConfigParser self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + extension) if os.path.exists(self._platforms_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None + return None self._jobs_parser = None # type: ConfigParser self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "jobs_" + expid + extension) if os.path.exists(self._jobs_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None + return None self._proj_parser = None # type: ConfigParser self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "proj_" + expid + extension) if os.path.exists(self._proj_parser_file) == False: - if extension == ".yml": - self.__init__(expid, basic_config, parser_factory, ".conf") - elif extension == ".conf": - return None + return None - self.check_proj_file() @property def jobs_parser(self): @@ -142,20 +126,6 @@ class confConfigStrategy(IConfigStrategy): """ return self._proj_parser_file - def check_proj_file(self): - """ - Add a section header to the project's configuration file (if not exists) - """ - # if os.path.exists(self._proj_parser_file): - # with open(self._proj_parser_file, 'w') as f: - # first_line = f.readline() - # if not re.match('[[a-zA-Z0-9]*]', first_line): - # content = f.read() - # f.seek(0, 0) - # f.write('[DEFAULT]'.rstrip('\r\n') + - # '\n' + first_line + content) - pass - @property def jobs_file(self): """ diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index d1e43cfb..0b302554 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -52,32 +52,57 @@ class ymlConfigStrategy(IConfigStrategy): self._conf_parser = Autosubmit4Config(expid) self._conf_parser.reload(True) + #set wrappers for the files + self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", + "autosubmit_" + expid + extension) + self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", + "expdef_" + expid + extension) + self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", + "platforms_" + expid + extension) + self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", + "jobs_" + expid + extension) + self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", + "proj_" + expid + extension) + def jobs_parser(self): logger.info("Not yet implemented") pass + #TODO: at the end of the implementation, check which methods can be moved to the top class for avoid code duplication + @property def experiment_file(self): - logger.info("Not yet implemented") - pass + """ + Returns experiment's config file name + """ + return self._exp_parser_file def platforms_parser(self): logger.info("OBSOLOTED - Not yet implemented") pass + @property def platforms_file(self): - logger.info("Not yet implemented") - pass + """ + Returns experiment's platforms config file name - def project_file(self): - logger.info("Not yet implemented") - pass + :return: platforms config file's name + :rtype: str + """ + return self._platforms_parser_file - @classmethod - def check_proj_file(self): - self._conf_parser.check_proj_file() + @property + def project_file(self): + """ + Returns project's config file name + """ + return self._proj_parser_file + @property def jobs_file(self): - pass + """ + Returns project's jobs file name + """ + return self._jobs_parser_file def get_full_config_as_dict(self): """ @@ -86,22 +111,29 @@ class ymlConfigStrategy(IConfigStrategy): _conf = _exp = _platforms = _jobs = _proj = None result = {} - def get_data(parser): + def get_data( parser): """ dictionary comprehension to get data from parser """ - res = {sec: {option: parser.get_section(sec, option) for option in parser.options(sec)} for sec in [ - section for section in parser.sections()]} - return res + logger.info(parser) + #res = {sec: {option: parser[sec][option] for option in parser[sec].keys()} for sec in [ + # section for section in parser.keys()]} + #return res + return parser + + # res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [ + # section for section in parser.sections()]} + # print(self._conf_parser) #result["conf"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._conf_parser else None #result["exp"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._exp_parser else None - result["platforms"] = get_data( self._conf_parser.experiment_data["PLATFORMS"]) if self._conf_parser.experiment_data["PLATFORMS"] else None - result["jobs"] = get_data( self._conf_parser.experiment_data["JOBS"]) if self._conf_parser.experiment_data["JOBS"] else None + result["platforms"] = self._conf_parser.platforms_data if self._conf_parser.platforms_data else None + #result["jobs"] = get_data( self._conf_parser.experiment_data["JOBS"]) if self._conf_parser.experiment_data["JOBS"] else None #result["proj"] = get_data( self._conf_parser.experiment_data["CONF"] ) if self._proj_parser else None return result + def get_full_config_as_json(self): return self._conf_parser.get_full_config_as_json() @@ -115,15 +147,19 @@ class ymlConfigStrategy(IConfigStrategy): pass def get_platform_queue(self, platform): + logger.info("get_platform_queue") return self._conf_parser.platforms_data[platform]["QUEUE"] def get_platform_serial_queue(self, platform): + logger.info("get_platform_serial_queue") return self._conf_parser.platforms_data[platform]["SERIAL_QUEUE"] def get_platform_project(self, platform): + logger.info("get_platform_project") return self._conf_parser.platforms_data[platform]["PROJECT"] def get_platform_wallclock(self, platform): + logger.info("get_platform_wallclock") return self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', "") def get_wallclock(self, section): @@ -152,19 +188,24 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.get_memory_per_task(section) def get_migrate_user_to(self, section): + """ + Returns the user to change to from platform config file. + :return: migrate user to + :rtype: str + """ return self._conf_parser.get_migrate_user_to(section) def get_current_user(self, section): - return str(self._conf_parser.platforms_data[section]["USER"]).lower() + return self._conf_parser.get_current_user(section) def get_current_project(self, section): - return str(self._conf_parser.platforms_data[section]["PROJECT"]).lower() + return self._conf_parser.get_current_project(section) def set_new_user(self, section, new_user): self._conf_parser.set_new_user(section, new_user) def get_migrate_project_to(self, section): - return str(self._conf_parser.platforms_data[section]["PROJECT_TO"]).lower() + return self._conf_parser.get_migrate_project_to(section) def set_new_project(self, section, new_project): self._conf_parser.set_new_project(section, new_project) -- GitLab From e61089d699226c9e5441af9d3e17fb07f9a42d30 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Thu, 1 Jun 2023 17:33:07 +0200 Subject: [PATCH 22/25] Fixings - #12 --- autosubmit_api/config/ymlConfigStrategy.py | 14 +------------- autosubmit_api/experiment/common_requests.py | 2 ++ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index 0b302554..a003876f 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -52,18 +52,6 @@ class ymlConfigStrategy(IConfigStrategy): self._conf_parser = Autosubmit4Config(expid) self._conf_parser.reload(True) - #set wrappers for the files - self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", - "autosubmit_" + expid + extension) - self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", - "expdef_" + expid + extension) - self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", - "platforms_" + expid + extension) - self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", - "jobs_" + expid + extension) - self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "yml", - "proj_" + expid + extension) - def jobs_parser(self): logger.info("Not yet implemented") pass @@ -141,7 +129,7 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.get_project_dir() def get_queue(self, section): - return self._conf_parser.get_queue() + return self._conf_parser.jobs_data.[section,'QUEUE'] def get_job_platform(self, section): pass diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py index c0000a2b..b75d58ae 100644 --- a/autosubmit_api/experiment/common_requests.py +++ b/autosubmit_api/experiment/common_requests.py @@ -28,6 +28,8 @@ import datetime import json import multiprocessing import subprocess +import logging + from collections import deque from ..autosubmit_legacy.autosubmit import Autosubmit from ..database import db_common as db_common -- GitLab From 0d68cd8e9110bccbff4818c4a34ae59daa6515da Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Thu, 1 Jun 2023 17:35:01 +0200 Subject: [PATCH 23/25] Fixings 2 - #12 --- autosubmit_api/config/ymlConfigStrategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index a003876f..e903ae66 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -129,7 +129,7 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.get_project_dir() def get_queue(self, section): - return self._conf_parser.jobs_data.[section,'QUEUE'] + return self._conf_parser.jobs_data[section,'QUEUE'] def get_job_platform(self, section): pass -- GitLab From acb83f3afbb3ed948b11f651854805eda0711539 Mon Sep 17 00:00:00 2001 From: jberlin Date: Fri, 2 Jun 2023 14:51:43 +0200 Subject: [PATCH 24/25] Several corrections and improvements - #12 --- autosubmit_api/common/utils.py | 3 ++- autosubmit_api/config/confConfigStrategy.py | 4 ++++ autosubmit_api/config/config_common.py | 12 ++++++------ autosubmit_api/config/ymlConfigStrategy.py | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/autosubmit_api/common/utils.py b/autosubmit_api/common/utils.py index 611a1fa7..2dd15707 100644 --- a/autosubmit_api/common/utils.py +++ b/autosubmit_api/common/utils.py @@ -201,13 +201,14 @@ class Status: HELD = 6 PREPARED = 7 SKIPPED = 8 + DELAYED = 9 FAILED = -1 UNKNOWN = -2 SUSPENDED = -3 ####### # Note: any change on constants must be applied on the dict below!!! VALUE_TO_KEY = {-3: 'SUSPENDED', -2: 'UNKNOWN', -1: 'FAILED', 0: 'WAITING', 1: 'READY', - 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED'} + 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED', 9: 'DELAYED'} STRING_TO_CODE = {v: k for k, v in list(VALUE_TO_KEY.items())} def retval(self, value): diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py index b173f1fe..fbeba438 100644 --- a/autosubmit_api/config/confConfigStrategy.py +++ b/autosubmit_api/config/confConfigStrategy.py @@ -88,6 +88,10 @@ class confConfigStrategy(IConfigStrategy): return None + @property + def jobs_parser(self): + return self._jobs_parser + @property def jobs_parser(self): return self._jobs_parser diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index 2fa05897..e0ddb79c 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -73,7 +73,7 @@ class AutosubmitConfig(object): @property def jobs_parser(self): - return self._configWrapper.jobs_parser() + return self._configWrapper.jobs_parser @property def experiment_file(self): @@ -81,7 +81,7 @@ class AutosubmitConfig(object): Returns experiment's config file name """ #return self._exp_parser_file - return self._configWrapper.experiment_file() + return self._configWrapper.experiment_file @property @@ -92,7 +92,7 @@ class AutosubmitConfig(object): :return: platforms config parser object :rtype: SafeConfigParser """ - return self._platforms_parser + return self._configWrapper.platforms_parser @property def platforms_file(self): @@ -102,14 +102,14 @@ class AutosubmitConfig(object): :return: platforms config file's name :rtype: str """ - return self._platforms_parser_file + return self._configWrapper.platforms_parser_file @property def project_file(self): """ Returns project's config file name """ - return self._configWrapper.project_file() + return self._configWrapper.project_file def check_proj_file(self): """ @@ -662,7 +662,7 @@ class AutosubmitConfig(object): :return: main platforms :rtype: int """ - return self._configWrapper.get_max_waiting_jobs(self) + return self._configWrapper.get_max_waiting_jobs() def get_default_job_type(self): """ diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py index e903ae66..ab42d380 100644 --- a/autosubmit_api/config/ymlConfigStrategy.py +++ b/autosubmit_api/config/ymlConfigStrategy.py @@ -129,7 +129,7 @@ class ymlConfigStrategy(IConfigStrategy): return self._conf_parser.get_project_dir() def get_queue(self, section): - return self._conf_parser.jobs_data[section,'QUEUE'] + return self._conf_parser.jobs_data[section].get('QUEUE', "") def get_job_platform(self, section): pass -- GitLab From 7344f677e3da66fe509b07ab1cc2d3c69e242817 Mon Sep 17 00:00:00 2001 From: Julian Rodrigo Berlin Date: Wed, 7 Jun 2023 15:01:44 +0200 Subject: [PATCH 25/25] Fixed version display on AS4 experiment card in the experiment list - #12 --- .../experiment/configuration_facade.py | 2 +- autosubmit_api/config/config_common.py | 1 - autosubmit_api/config/ymlConfigParser.py | 256 ------------------ autosubmit_api/database/db_common.py | 4 - 4 files changed, 1 insertion(+), 262 deletions(-) delete mode 100644 autosubmit_api/config/ymlConfigParser.py diff --git a/autosubmit_api/components/experiment/configuration_facade.py b/autosubmit_api/components/experiment/configuration_facade.py index e5651e88..e349861b 100644 --- a/autosubmit_api/components/experiment/configuration_facade.py +++ b/autosubmit_api/components/experiment/configuration_facade.py @@ -190,7 +190,7 @@ class AutosubmitConfigurationFacade(ConfigurationFacade): def get_wrapper_type(self): # type: () -> str | None - if self.autosubmit_conf.get_wrapper_type().upper() != "NONE": + if self.autosubmit_conf.get_wrapper_type() and self.autosubmit_conf.get_wrapper_type().upper() != "NONE": return self.autosubmit_conf.get_wrapper_type().upper() return None diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py index e0ddb79c..ca5cc4bc 100644 --- a/autosubmit_api/config/config_common.py +++ b/autosubmit_api/config/config_common.py @@ -37,7 +37,6 @@ from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser from bscearth.utils.date import parse_date from bscearth.utils.log import Log from ..config.basicConfig import BasicConfig -from ..config.ymlConfigParser import ymlConfigParser from ..config.IConfigStrategy import IConfigStrategy from ..config.ymlConfigStrategy import ymlConfigStrategy from ..config.confConfigStrategy import confConfigStrategy diff --git a/autosubmit_api/config/ymlConfigParser.py b/autosubmit_api/config/ymlConfigParser.py deleted file mode 100644 index 2df801d5..00000000 --- a/autosubmit_api/config/ymlConfigParser.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2023 Earth Sciences Department, BSC-CNS - -# This file is part of Autosubmit , Autosubmot API. - -# Autosubmit is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# Autosubmit is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with Autosubmit. If not, see . - -""" -Module containing functions to manage autosubmit 4 config files that are in yml format. -Sample files for a given experiment: - /esarchive/autosubmit/EXPID/conf/autosubmit_EXPID.yml - /esarchive/autosubmit/EXPID/conf/expdef_EXPID.yml - /esarchive/autosubmit/EXPID/conf/jobs_EXPID.yml - /esarchive/autosubmit/EXPID/conf/platforms_EXPID.yml - /esarchive/autosubmit/EXPID/conf/proj_EXPID.yml -""" -import os -import re -import subprocess -import json -import logging -import locale - -from pyparsing import nestedExpr -from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser -from bscearth.utils.date import parse_date -from bscearth.utils.log import Log -from ..config.basicConfig import BasicConfig -from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config - -logger = logging.getLogger('gunicorn.error') - - -class ymlConfigParser(object): - - def __init__(self, conf_type): - logger.info("Creating AS4 Parser !!!!!") - self.conf_parser = Autosubmit4Config(self.expid) - self.conf_parser.reload(True) - # set the type of config file (platforms, jobs, exp, etc) - self.conf_type = conf_type - # set the data object - self.conf_data = self.conf_parser[conf_type] - - - def has_option(self,section,option): - return self.conf_data[section].get('TYPE') is not None - - def get_option(self, section, option, default=None): - """ - Gets an option from given parser - - :param self: parser to use - :type self: SafeConfigParser - :param section: section that contains the option - :type section: str - :param option: option to get - :type option: str - :param default: value to be returned if option is not present - :type default: object - :return: option value - :rtype: str - """ - if self.has_option(section, option): - return self.conf_data[section].get(option) - else: - return default - - - def get_bool_option(self, section, option, default): - """ - Gets a boolean option from given parser - - :param self: parser to use - :type self: SafeConfigParser - :param section: section that contains the option - :type section: str - :param option: option to get - :type option: str - :param default: value to be returned if option is not present - :type default: bool - :return: option value - :rtype: bool - """ - if self.has_option(section, option): - return self.get(section, option).lower().strip() == 'true' - else: - return default - - def get_int_option(self, section, option, default=0): - """ - Gets an integer option - - :param section: section that contains the option - :type section: str - :param option: option to get - :type option: str - :param default: value to be returned if option is not present - :type default: int - :return: option value - :rtype: int - """ - return int(self.get_option(section, option, default)) - - def get_float_option(self, section, option, default=0.0): - """ - Gets a float option - - :param section: section that contains the option - :type section: str - :param option: option to get - :type option: str - :param default: value to be returned if option is not present - :type default: float - :return: option value - :rtype: float - """ - return float(self.get_option(section, option, default)) - - def get_choice_option(self, section, option, choices, default=None, ignore_case=False): - """ - Gets a boolean option - - :param ignore_case: if True, - :param choices: available choices - :type choices: [str] - :param section: section that contains the option - :type section: str - :param option: option to get - :type option: str - :param default: value to be returned if option is not present - :type default: str - :return: option value - :rtype: str - """ - - if self.has_option(section, option): - value = self.get_option(section, option, choices[0]) - if ignore_case: - value = value.lower() - for choice in choices: - if value == choice.lower(): - return choice - else: - if value in choices: - return value - raise ConfigError('Value {2} in option {0} in section {1} is not a valid choice'.format(option, section, - value)) - else: - if default: - return default - raise ConfigError('Option {0} in section {1} is not present and there is not a default value'.format(option, - section)) - - def check_exists(self, section, option): - """ - Checks if an option exists in given parser - - :param section: section that contains the option - :type section: str - :param option: option to check - :type option: str - :return: True if option exists, False otherwise - :rtype: bool - """ - if self.has_option(section, option): - return True - else: - Log.error('Option {0} in section {1} not found'.format(option, section)) - return False - - def check_is_boolean(self, section, option, must_exist): - """ - Checks if an option is a boolean value in given parser - - :param section: section that contains the option - :type section: str - :param option: option to check - :type option: str - :param must_exist: if True, option must exist - :type must_exist: bool - :return: True if option value is boolean, False otherwise - :rtype: bool - """ - if must_exist and not self.check_exists(section, option): - Log.error('Option {0} in section {1} must exist'.format(option, section)) - return False - if self.get_option(section, option, 'false').lower() not in ['false', 'true']: - Log.error('Option {0} in section {1} must be true or false'.format(option, section)) - return False - return True - - def check_is_choice(self, section, option, must_exist, choices): - """ - Checks if an option is a valid choice in given parser - - :param self: parser to use - :type self: SafeConfigParser - :param section: section that contains the option - :type section: str - :param option: option to check - :type option: str - :param must_exist: if True, option must exist - :type must_exist: bool - :param choices: valid choices - :type choices: list - :return: True if option value is a valid choice, False otherwise - :rtype: bool - """ - if must_exist and not self.check_exists(section, option): - return False - value = self.get_option(section, option, choices[0]) - if value not in choices: - Log.error('Value {2} in option {0} in section {1} is not a valid choice'.format(option, section, value)) - return False - return True - - def check_is_int(self, section, option, must_exist): - """ - Checks if an option is an integer value in given parser - - :param self: parser to use - :type self: SafeConfigParser - :param section: section that contains the option - :type section: str - :param option: option to check - :type option: str - :param must_exist: if True, option must exist - :type must_exist: bool - :return: True if option value is integer, False otherwise - :rtype: bool - """ - if must_exist and not self.check_exists(section, option): - return False - value = self.get_option(section, option, '1') - try: - int(value) - except ValueError: - Log.error('Option {0} in section {1} is not valid an integer'.format(option, section)) - return False - return True - - - diff --git a/autosubmit_api/database/db_common.py b/autosubmit_api/database/db_common.py index bde23d0e..e58ef8cb 100644 --- a/autosubmit_api/database/db_common.py +++ b/autosubmit_api/database/db_common.py @@ -365,8 +365,6 @@ def search_experiment_by_id(searchString, typeExp=None, onlyActive=None, owner=N wrapper = autosubmit_config_facade.get_wrapper_type() last_modified_pkl_datetime = autosubmit_config_facade.get_pkl_last_modified_time_as_datetime() except Exception as exp: - version = "Unknown" - wrapper = None last_modified_pkl_datetime = None pass status = experiment_status.get(expid, "NOT RUNNING") @@ -454,8 +452,6 @@ def get_current_running_exp(): wrapper = autosubmit_config_facade.get_wrapper_type() last_modified_pkl_datetime = autosubmit_config_facade.get_pkl_last_modified_time_as_datetime() except Exception as exp: - version = "Unknown" - wrapper = None last_modified_pkl_datetime = None pass if (expid in experiment_times): -- GitLab