diff --git a/autosubmit_api/autosubmit_legacy/job/job_dict.py b/autosubmit_api/autosubmit_legacy/job/job_dict.py
index 0f833cc31ed384196246fc53b9290ed9be3c5dbd..f67f7748b6c9d132ef4fc26842a50ba7fd617d1a 100644
--- a/autosubmit_api/autosubmit_legacy/job/job_dict.py
+++ b/autosubmit_api/autosubmit_legacy/job/job_dict.py
@@ -371,6 +371,9 @@ class DicJobs:
:param default: value to return if not defined in configuration file
:type default: object
"""
+ #todo: here should be implemented some of the changes for supporting yaml files
+
+
if self._parser.has_option(section, option):
return self._parser.get(section, option)
else:
diff --git a/autosubmit_api/common/utils.py b/autosubmit_api/common/utils.py
index 611a1fa726f252c1c5d219c58116f93ce94df58e..2dd157078984bd9fcb1cd5a99c704d428aeb6eed 100644
--- a/autosubmit_api/common/utils.py
+++ b/autosubmit_api/common/utils.py
@@ -201,13 +201,14 @@ class Status:
HELD = 6
PREPARED = 7
SKIPPED = 8
+ DELAYED = 9
FAILED = -1
UNKNOWN = -2
SUSPENDED = -3
#######
# Note: any change on constants must be applied on the dict below!!!
VALUE_TO_KEY = {-3: 'SUSPENDED', -2: 'UNKNOWN', -1: 'FAILED', 0: 'WAITING', 1: 'READY',
- 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED'}
+ 2: 'SUBMITTED', 3: 'QUEUING', 4: 'RUNNING', 5: 'COMPLETED', 6: 'HELD', 7: 'PREPARED', 8: 'SKIPPED', 9: 'DELAYED'}
STRING_TO_CODE = {v: k for k, v in list(VALUE_TO_KEY.items())}
def retval(self, value):
diff --git a/autosubmit_api/components/experiment/configuration_facade.py b/autosubmit_api/components/experiment/configuration_facade.py
index e5651e885f762823ea6f3eaa5d0d462bc66e2897..e349861b951145e6adaf007890af7e265ce6467c 100644
--- a/autosubmit_api/components/experiment/configuration_facade.py
+++ b/autosubmit_api/components/experiment/configuration_facade.py
@@ -190,7 +190,7 @@ class AutosubmitConfigurationFacade(ConfigurationFacade):
def get_wrapper_type(self):
# type: () -> str | None
- if self.autosubmit_conf.get_wrapper_type().upper() != "NONE":
+ if self.autosubmit_conf.get_wrapper_type() and self.autosubmit_conf.get_wrapper_type().upper() != "NONE":
return self.autosubmit_conf.get_wrapper_type().upper()
return None
diff --git a/autosubmit_api/components/jobs/joblist_loader.py b/autosubmit_api/components/jobs/joblist_loader.py
index 50a94d8922869860d561471991f23301740be44b..88abb2a161d05e1c941ae75902cbc5014d4e9648 100644
--- a/autosubmit_api/components/jobs/joblist_loader.py
+++ b/autosubmit_api/components/jobs/joblist_loader.py
@@ -14,6 +14,10 @@ from typing import Dict, List, Set, Tuple
from ...config.config_common import AutosubmitConfig
from ...config.basicConfig import BasicConfig
import json
+import logging
+
+
+logger = logging.getLogger('gunicorn.error')
class JobListLoader(object):
""" Class that manages loading the list of jobs from the pkl. Adds other resources. """
@@ -187,14 +191,16 @@ class JobListLoader(object):
else:
job_qos = self.configuration_facade.get_section_qos(job.section)
if len(job_qos.strip()) == 0:
- job_qos = self.configuration_facade.get_platform_qos(job.platform, job.ncpus)
+ if job.platform != "None":
+ job_qos = self.configuration_facade.get_platform_qos(job.platform, job.ncpus)
return job_qos
def _determine_wallclock(self, job):
# type: (Job) -> None
wallclock = self.configuration_facade.get_section_wallclock(job.section)
if len(wallclock.strip()) == 0:
- wallclock = self.configuration_facade.get_platform_max_wallclock(job.platform)
+ if job.platform != "None":
+ wallclock = self.configuration_facade.get_platform_max_wallclock(job.platform)
return wallclock
def assign_packages_to_jobs(self):
diff --git a/autosubmit_api/config/IConfigStrategy.py b/autosubmit_api/config/IConfigStrategy.py
new file mode 100644
index 0000000000000000000000000000000000000000..0fbc61fa254033cd814e14dcfe8d0e8d8d0045db
--- /dev/null
+++ b/autosubmit_api/config/IConfigStrategy.py
@@ -0,0 +1,817 @@
+#!/usr/bin/env python
+
+# Copyright 2015 Earth Sciences Department, BSC-CNS
+
+# This file is part of Autosubmit.
+
+# Autosubmit is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# Autosubmit is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Autosubmit. If not, see .
+
+try:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+ from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config
+except ImportError:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+
+
+import os
+import re
+import subprocess
+import json
+import logging
+import locale
+
+from pyparsing import nestedExpr
+from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser
+from bscearth.utils.date import parse_date
+from bscearth.utils.log import Log
+from ..config.basicConfig import BasicConfig
+from abc import ABC, abstractmethod
+
+
+class IConfigStrategy(ABC):
+ """
+ Public interface for Autosubmit config files
+
+ :param expid: experiment identifier
+ :type expid: str
+ """
+
+ @abstractmethod
+ def jobs_parser(self):
+ raise NotImplementedError
+
+ @abstractmethod
+ def experiment_file(self):
+ """
+ Returns experiment's config file name
+ """
+ return self._exp_parser_file
+
+ @abstractmethod
+ def platforms_parser(self):
+ """
+ Returns experiment's platforms parser object
+
+ :return: platforms config parser object
+ :rtype: SafeConfigParser
+ """
+ pass
+
+ @abstractmethod
+ def platforms_file(self):
+ """
+ Returns experiment's platforms config file name
+
+ :return: platforms config file's name
+ :rtype: str
+ """
+ pass
+
+ @abstractmethod
+ def project_file(self):
+ """
+ Returns project's config file name
+ """
+ pass
+
+ @abstractmethod
+ def jobs_file(self):
+ """
+ Returns project's jobs file name
+ """
+ pass
+
+ @abstractmethod
+ def get_full_config_as_dict(self):
+ """
+ Returns full configuration as json object
+ """
+ pass
+
+ @abstractmethod
+ def get_project_dir(self):
+ """
+ Returns experiment's project directory
+
+ :return: experiment's project directory
+ :rtype: str
+ """
+ pass
+
+ @abstractmethod
+ def get_queue(self, section):
+ """
+ Get queue for the given job type
+ :param section: job type
+ :type section: str
+ :return: queue
+ :rtype: str
+ """
+ pass
+
+ @abstractmethod
+ def get_job_platform(self, section):
+ pass
+
+ @abstractmethod
+ def get_platform_queue(self, platform):
+ pass
+
+ @abstractmethod
+ def get_platform_serial_queue(self, platform):
+ pass
+
+ @abstractmethod
+ def get_platform_project(self, platform):
+ pass
+
+ @abstractmethod
+ def get_platform_wallclock(self, platform):
+ pass
+
+ def get_wallclock(self, section):
+ """
+ Gets wallclock for the given job type
+ :param section: job type
+ :type section: str
+ :return: wallclock time
+ :rtype: str
+ """
+ pass
+
+ def get_synchronize(self, section):
+ """
+ Gets wallclock for the given job type
+ :param section: job type
+ :type section: str
+ :return: wallclock time
+ :rtype: str
+ """
+ pass
+
+ def get_processors(self, section):
+ """
+ Gets processors needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: wallclock time
+ :rtype: str
+ """
+ pass
+
+ def get_threads(self, section):
+ """
+ Gets threads needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: threads needed
+ :rtype: str
+ """
+ pass
+
+ def get_tasks(self, section):
+ """
+ Gets tasks needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: tasks (processes) per host
+ :rtype: str
+ """
+ pass
+
+ def get_scratch_free_space(self, section):
+ """
+ Gets scratch free space needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: percentage of scratch free space needed
+ :rtype: int
+ """
+ pass
+
+ def get_memory(self, section):
+ """
+ Gets memory needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: memory needed
+ :rtype: str
+ """
+ pass
+
+ def get_memory_per_task(self, section):
+ """
+ Gets memory per task needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: memory per task needed
+ :rtype: str
+ """
+ pass
+
+ def get_migrate_user_to(self, section):
+ """
+ Returns the user to change to from platform config file.
+
+ :return: migrate user to
+ :rtype: str
+ """
+ #return self._platforms_parser.get_option(section, 'USER_TO', '').lower()
+ pass
+
+ def get_current_user(self, section):
+ """
+ Returns the user to be changed from platform config file.
+
+ :return: migrate user to
+ :rtype: str
+ """
+ pass
+
+ def get_current_project(self, section):
+ """
+ Returns the project to be changed from platform config file.
+
+ :return: migrate user to
+ :rtype: str
+ """
+ return str(self._platforms_data[section]["USER"]).lower()
+
+ def set_new_user(self, section, new_user):
+ """
+ Sets new user for given platform
+ :param new_user:
+ :param section: platform name
+ :type: str
+ """
+ pass
+
+ def get_migrate_project_to(self, section):
+ """
+ Returns the project to change to from platform config file.
+
+ :return: migrate project to
+ :rtype: str
+ """
+ pass
+
+
+
+ def set_new_project(self, section, new_project):
+ """
+ Sets new project for given platform
+ :param new_project:
+ :param section: platform name
+ :type: str
+ """
+ pass
+
+
+ def get_custom_directives(self, section):
+ """
+ Gets custom directives needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: custom directives needed
+ :rtype: str
+ """
+ pass
+
+
+ def check_conf_files(self):
+ """
+ Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing
+ required options. Prints results in log
+
+ :return: True if everything is correct, False if it finds any error
+ :rtype: bool
+ """
+ pass
+
+ def check_autosubmit_conf(self):
+ """
+ Checks experiment's autosubmit configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ pass
+
+ def check_platforms_conf(self):
+ """
+ Checks experiment's queues configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+
+
+ def check_jobs_conf(self):
+ """
+ Checks experiment's jobs configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ pass
+
+
+ def check_expdef_conf(self):
+ """
+ Checks experiment's experiment configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ pass
+
+
+ def check_proj(self):
+ """
+ Checks project config file
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ pass
+
+
+ def check_wrapper_conf(self):
+ pass
+
+
+ def reload(self):
+ """
+ Creates parser objects for configuration files
+ """
+ pass
+
+ def load_parameters(self):
+ """
+ Load parameters from experiment and autosubmit config files. If experiment's type is not none,
+ also load parameters from model's config file
+
+ :return: a dictionary containing tuples [parameter_name, parameter_value]
+ :rtype: dict
+ """
+
+
+ def load_project_parameters(self):
+ """
+ Loads parameters from model config file
+
+ :return: dictionary containing tuples [parameter_name, parameter_value]
+ :rtype: dict
+ """
+ pass
+
+
+ def set_expid(self, exp_id):
+ """
+ Set experiment identifier in autosubmit and experiment config files
+
+ :param exp_id: experiment identifier to store
+ :type exp_id: str
+ """
+
+
+ def get_project_type(self):
+ """
+ Returns project type from experiment config file
+
+ :return: project type
+ :rtype: str
+ """
+ pass
+
+ def get_file_project_conf(self):
+ """
+ Returns path to project config file from experiment config file
+
+ :return: path to project config file
+ :rtype: str
+ """
+ pass
+
+ def get_file_jobs_conf(self):
+ """
+ Returns path to project config file from experiment config file
+
+ :return: path to project config file
+ :rtype: str
+ """
+
+
+ def get_git_project_origin(self):
+ """
+ Returns git origin from experiment config file
+
+ :return: git origin
+ :rtype: str
+ """
+
+ def get_git_project_branch(self):
+ """
+ Returns git branch from experiment's config file
+
+ :return: git branch
+ :rtype: str
+ """
+
+ def get_git_project_commit(self):
+ """
+ Returns git commit from experiment's config file
+
+ :return: git commit
+ :rtype: str
+ """
+
+ def get_submodules_list(self):
+ """
+ Returns submodules list from experiment's config file
+ Default is --recursive
+ :return: submodules to load
+ :rtype: list
+ """
+
+ def get_project_destination(self):
+ """
+ Returns git commit from experiment's config file
+
+ :return: git commit
+ :rtype: str
+ """
+
+
+ def set_git_project_commit(self, as_conf):
+ """
+ Function to register in the configuration the commit SHA of the git project version.
+ :param as_conf: Configuration class for exteriment
+ :type as_conf: AutosubmitConfig
+ """
+
+ def get_svn_project_url(self):
+ """
+ Gets subversion project url
+
+ :return: subversion project url
+ :rtype: str
+ """
+
+ def get_svn_project_revision(self):
+ """
+ Get revision for subversion project
+
+ :return: revision for subversion project
+ :rtype: str
+ """
+
+ def get_local_project_path(self):
+ """
+ Gets path to origin for local project
+
+ :return: path to local project
+ :rtype: str
+ """
+
+ def get_date_list(self):
+ """
+ Returns startdates list from experiment's config file
+
+ :return: experiment's startdates
+ :rtype: list
+ """
+
+
+ def get_num_chunks(self):
+ """
+ Returns number of chunks to run for each member
+
+ :return: number of chunks
+ :rtype: int
+ """
+
+ def get_chunk_ini(self, default=1):
+ """
+ Returns the first chunk from where the experiment will start
+
+ :param default:
+ :return: initial chunk
+ :rtype: int
+ """
+
+
+ def get_chunk_size_unit(self):
+ # type: () -> str
+ """
+ Unit for the chunk length
+
+ :return: Unit for the chunk length Options: {hour, day, month, year}
+ :rtype: str
+ """
+
+ pass
+
+ def get_chunk_size(self, default=1):
+ # type: (int) -> int
+ """
+ Chunk Size as defined in the expdef file.
+
+ :return: Chunksize, 1 as default.
+ :rtype: int
+ """
+ pass
+
+ def get_member_list(self, run_only=False):
+ """
+ Returns members list from experiment's config file
+
+ :return: experiment's members
+ :rtype: list
+ """
+ pass
+
+ def get_rerun(self):
+ """
+ Returns startdates list from experiment's config file
+
+ :return: rerurn value
+ :rtype: list
+ """
+
+ pass
+
+ def get_chunk_list(self):
+ """
+ Returns chunk list from experiment's config file
+
+ :return: experiment's chunks
+ :rtype: list
+ """
+ pass
+
+ def get_platform(self):
+ """
+ Returns main platforms from experiment's config file
+
+ :return: main platforms
+ :rtype: str
+ """
+ pass
+
+ def set_platform(self, hpc):
+ """
+ Sets main platforms in experiment's config file
+
+ :param hpc: main platforms
+ :type: str
+ """
+ pass
+
+ def set_version(self, autosubmit_version):
+ """
+ Sets autosubmit's version in autosubmit's config file
+
+ :param autosubmit_version: autosubmit's version
+ :type autosubmit_version: str
+ """
+ pass
+
+ def get_version(self):
+ """
+ Returns version number of the current experiment from autosubmit's config file
+
+ :return: version
+ :rtype: str
+ """
+ pass
+
+ def get_total_jobs(self):
+ """
+ Returns max number of running jobs from autosubmit's config file
+
+ :return: max number of running jobs
+ :rtype: int
+ """
+ pass
+
+ def get_max_wallclock(self):
+ """
+ Returns max wallclock
+
+ :rtype: str
+ """
+ pass
+
+ def get_max_processors(self):
+ """
+ Returns max processors from autosubmit's config file
+
+ :rtype: str
+ """
+ pass
+
+ def get_max_waiting_jobs(self):
+ """
+ Returns max number of waiting jobs from autosubmit's config file
+
+ :return: main platforms
+ :rtype: int
+ """
+ pass
+
+ def get_default_job_type(self):
+ """
+ Returns the default job type from experiment's config file
+
+ :return: default type such as bash, python, r..
+ :rtype: str
+ """
+ pass
+
+ def get_safetysleeptime(self):
+ """
+ Returns safety sleep time from autosubmit's config file
+
+ :return: safety sleep time
+ :rtype: int
+ """
+ pass
+
+ def set_safetysleeptime(self, sleep_time):
+ """
+ Sets autosubmit's version in autosubmit's config file
+
+ :param sleep_time: value to set
+ :type sleep_time: int
+ """
+ pass
+
+ def get_retrials(self):
+ """
+ Returns max number of retrials for job from autosubmit's config file
+
+ :return: safety sleep time
+ :rtype: int
+ """
+ pass
+
+ def get_notifications(self):
+ """
+ Returns if the user has enabled the notifications from autosubmit's config file
+
+ :return: if notifications
+ :rtype: string
+ """
+ pass
+
+ def get_remote_dependencies(self):
+ """
+ Returns if the user has enabled the remote dependencies from autosubmit's config file
+
+ :return: if remote dependencies
+ :rtype: bool
+ """
+ pass
+
+ def get_wrapper_type(self):
+ """
+ Returns what kind of wrapper (VERTICAL, MIXED-VERTICAL, HORIZONTAL, HYBRID, NONE) the user has configured in the autosubmit's config
+
+ :return: wrapper type (or none)
+ :rtype: string
+ """
+ pass
+
+ def get_wrapper_jobs(self):
+ """
+ Returns the jobs that should be wrapped, configured in the autosubmit's config
+
+ :return: expression (or none)
+ :rtype: string
+ """
+ pass
+
+ def get_max_wrapped_jobs(self):
+ """
+ Returns the maximum number of jobs that can be wrapped together as configured in autosubmit's config file
+
+ :return: maximum number of jobs (or total jobs)
+ :rtype: string
+ """
+ # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs()))
+
+ pass
+
+ def get_wrapper_check_time(self):
+ """
+ Returns time to check the status of jobs in the wrapper
+
+ :return: wrapper check time
+ :rtype: int
+ """
+ pass
+
+ def get_wrapper_machinefiles(self):
+ """
+ Returns the strategy for creating the machinefiles in wrapper jobs
+
+ :return: machinefiles function to use
+ :rtype: string
+ """
+ pass
+
+ def get_wrapper_queue(self):
+ """
+ Returns the wrapper queue if not defined, will be the one of the first job wrapped
+
+ :return: expression (or none)
+ :rtype: string
+ """
+ pass
+
+ def get_jobs_sections(self):
+ """
+ Returns the list of sections defined in the jobs config file
+
+ :return: sections
+ :rtype: list
+ """
+ return self._jobs_parser.sections()
+
+ def get_copy_remote_logs(self):
+ """
+ Returns if the user has enabled the logs local copy from autosubmit's config file
+
+ :return: if logs local copy
+ :rtype: bool
+ """
+ pass
+
+ def get_mails_to(self):
+ """
+ Returns the address where notifications will be sent from autosubmit's config file
+
+ :return: mail address
+ :rtype: [str]
+ """
+ pass
+
+ def get_communications_library(self):
+ """
+ Returns the communications library from autosubmit's config file. Paramiko by default.
+
+ :return: communications library
+ :rtype: str
+ """
+ pass
+
+ def get_storage_type(self):
+ """
+ Returns the communications library from autosubmit's config file. Paramiko by default.
+
+ :return: communications library
+ :rtype: str
+ """
+ pass
+
+ @staticmethod
+ def is_valid_mail_address(mail_address):
+ if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address):
+ return True
+ else:
+ return False
+ @classmethod
+ def is_valid_communications_library(self):
+ library = self.get_communications_library()
+ return library in ['paramiko', 'saga']
+
+ @classmethod
+ def is_valid_storage_type(self):
+ storage_type = self.get_storage_type()
+ return storage_type in ['pkl', 'db']
+
+
+ def is_valid_jobs_in_wrapper(self):
+ pass
+
+ def is_valid_git_repository(self):
+ pass
+
+ @staticmethod
+ def get_parser(parser_factory, file_path):
+ # type: (ConfigParserFactory, str) -> ConfigParser
+ pass
\ No newline at end of file
diff --git a/autosubmit_api/config/basicConfig.py b/autosubmit_api/config/basicConfig.py
index 3c43686608c026b58e35854050519e0928404e3a..38a25576784770a95009fa5e9ec6696ec339bba2 100644
--- a/autosubmit_api/config/basicConfig.py
+++ b/autosubmit_api/config/basicConfig.py
@@ -19,6 +19,7 @@
try:
# noinspection PyCompatibility
from configparser import SafeConfigParser
+ from autosubmitconfigparser.config.configcommon import AutosubmitConfig
except ImportError:
# noinspection PyCompatibility
from configparser import SafeConfigParser
diff --git a/autosubmit_api/config/confConfigStrategy.py b/autosubmit_api/config/confConfigStrategy.py
new file mode 100644
index 0000000000000000000000000000000000000000..fbeba43851bceb4fe026c099338f02d88d50f483
--- /dev/null
+++ b/autosubmit_api/config/confConfigStrategy.py
@@ -0,0 +1,1344 @@
+
+# !/usr/bin/env python
+
+# Copyright 2015 Earth Sciences Department, BSC-CNS
+
+# This file is part of Autosubmit.
+
+# Autosubmit is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# Autosubmit is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Autosubmit. If not, see .
+try:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+ from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config
+except ImportError:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+
+
+import os
+import re
+import subprocess
+import json
+import logging
+
+from pyparsing import nestedExpr
+from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser
+from bscearth.utils.date import parse_date
+from bscearth.utils.log import Log
+from ..config.basicConfig import BasicConfig
+from ..config.IConfigStrategy import IConfigStrategy
+
+logger = logging.getLogger('gunicorn.error')
+
+class confConfigStrategy(IConfigStrategy):
+ """
+ Class to handle experiment configuration coming from file or database
+
+ :param expid: experiment identifier
+ :type expid: str
+ """
+
+ def __init__(self, expid, basic_config, parser_factory, extension=".conf"):
+ # type: (str, BasicConfig, ConfigParserFactory, Extension) -> None
+
+ self.expid = expid
+ self.basic_config = basic_config
+ self.parser_factory = parser_factory
+
+ # By default check for .yml files first as it is the new standard for AS 4.0
+ self._conf_parser = None # type: ConfigParser
+ self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf",
+ "autosubmit_" + expid + extension)
+ if os.path.exists(self._conf_parser_file) == False:
+ return None
+
+ self._exp_parser = None # type: ConfigParser
+ self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf",
+ "expdef_" + expid + extension)
+ if os.path.exists(self._exp_parser_file) == False:
+ return None
+
+ self._platforms_parser = None # type: ConfigParser
+ self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf",
+ "platforms_" + expid + extension)
+ if os.path.exists(self._platforms_parser_file) == False:
+ return None
+
+ self._jobs_parser = None # type: ConfigParser
+ self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf",
+ "jobs_" + expid + extension)
+ if os.path.exists(self._jobs_parser_file) == False:
+ return None
+
+ self._proj_parser = None # type: ConfigParser
+ self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf",
+ "proj_" + expid + extension)
+ if os.path.exists(self._proj_parser_file) == False:
+ return None
+
+
+ @property
+ def jobs_parser(self):
+ return self._jobs_parser
+
+ @property
+ def jobs_parser(self):
+ return self._jobs_parser
+
+ @property
+ def experiment_file(self):
+ """
+ Returns experiment's config file name
+ """
+ return self._exp_parser_file
+
+ @property
+ def platforms_parser(self):
+ """
+ Returns experiment's platforms parser object
+
+ :return: platforms config parser object
+ :rtype: SafeConfigParser
+ """
+ return self._platforms_parser
+
+ @property
+ def platforms_file(self):
+ """
+ Returns experiment's platforms config file name
+
+ :return: platforms config file's name
+ :rtype: str
+ """
+ return self._platforms_parser_file
+
+ @property
+ def project_file(self):
+ """
+ Returns project's config file name
+ """
+ return self._proj_parser_file
+
+ @property
+ def jobs_file(self):
+ """
+ Returns project's jobs file name
+ """
+ return self._jobs_parser_file
+
+ def get_full_config_as_dict(self):
+ """
+ Returns full configuration as json object
+ """
+ _conf = _exp = _platforms = _jobs = _proj = None
+ result = {}
+
+ def get_data(parser):
+ """
+ dictionary comprehension to get data from parser
+ """
+ res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [
+ section for section in parser.sections()]}
+ return res
+
+ # print(self._conf_parser)
+ result["conf"] = get_data(
+ self._conf_parser) if self._conf_parser else None
+ result["exp"] = get_data(
+ self._exp_parser) if self._exp_parser else None
+ result["platforms"] = get_data(
+ self._platforms_parser) if self._platforms_parser else None
+ result["jobs"] = get_data(
+ self._jobs_parser) if self._jobs_parser else None
+ result["proj"] = get_data(
+ self._proj_parser) if self._proj_parser else None
+ return result
+
+ def get_full_config_as_json(self):
+ """
+ Return config as json object
+ """
+ try:
+ return json.dumps(self.get_full_config_as_dict())
+ except Exception as exp:
+ Log.warning(
+ "Autosubmit was not able to retrieve and save the configuration into the historical database.")
+ return ""
+
+ def get_project_dir(self):
+ """
+ Returns experiment's project directory
+
+ :return: experiment's project directory
+ :rtype: str
+ """
+ dir_templates = os.path.join(self.basic_config.LOCAL_ROOT_DIR, self.expid, BasicConfig.LOCAL_PROJ_DIR,
+ self.get_project_destination())
+ return dir_templates
+
+ def get_queue(self, section):
+ """
+ Get queue for the given job type
+ :param section: job type
+ :type section: str
+ :return: queue
+ :rtype: str
+ """
+ return self._jobs_parser.get_option(section, 'QUEUE', '')
+
+ def get_job_platform(self, section):
+ return self._jobs_parser.get_option(section, 'PLATFORM', '')
+
+ def get_platform_queue(self, platform):
+ return self._platforms_parser.get_option(platform, 'QUEUE', '')
+
+ def get_platform_serial_queue(self, platform):
+ return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '')
+
+ def get_platform_project(self, platform):
+ return self._platforms_parser.get_option(platform, "PROJECT", "")
+
+ def get_platform_wallclock(self, platform):
+ return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '')
+
+ def get_wallclock(self, section):
+ """
+ Gets wallclock for the given job type
+ :param section: job type
+ :type section: str
+ :return: wallclock time
+ :rtype: str
+ """
+ return self._jobs_parser.get_option(section, 'WALLCLOCK', '')
+
+ def get_synchronize(self, section):
+ """
+ Gets wallclock for the given job type
+ :param section: job type
+ :type section: str
+ :return: wallclock time
+ :rtype: str
+ """
+ return self._jobs_parser.get_option(section, 'SYNCHRONIZE', '')
+
+ def get_processors(self, section):
+ """
+ Gets processors needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: wallclock time
+ :rtype: str
+ """
+ return str(self._jobs_parser.get_option(section, 'PROCESSORS', 1))
+
+ def get_threads(self, section):
+ """
+ Gets threads needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: threads needed
+ :rtype: str
+ """
+ return str(self._jobs_parser.get_option(section, 'THREADS', 1))
+
+ def get_tasks(self, section):
+ """
+ Gets tasks needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: tasks (processes) per host
+ :rtype: str
+ """
+ return str(self._jobs_parser.get_option(section, 'TASKS', 0))
+
+ def get_scratch_free_space(self, section):
+ """
+ Gets scratch free space needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: percentage of scratch free space needed
+ :rtype: int
+ """
+ return int(self._jobs_parser.get_option(section, 'SCRATCH_FREE_SPACE', 0))
+
+ def get_memory(self, section):
+ """
+ Gets memory needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: memory needed
+ :rtype: str
+ """
+ return str(self._jobs_parser.get_option(section, 'MEMORY', ''))
+
+ def get_memory_per_task(self, section):
+ """
+ Gets memory per task needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: memory per task needed
+ :rtype: str
+ """
+ return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', ''))
+
+ def get_migrate_user_to(self, section):
+ """
+ Returns the user to change to from platform config file.
+
+ :return: migrate user to
+ :rtype: str
+ """
+ return self._platforms_parser.get_option(section, 'USER_TO', '').lower()
+
+ def get_current_user(self, section):
+ """
+ Returns the user to be changed from platform config file.
+
+ :return: migrate user to
+ :rtype: str
+ """
+ return self._platforms_parser.get_option(section, 'USER', '').lower()
+
+ def get_current_project(self, section):
+ """
+ Returns the project to be changed from platform config file.
+
+ :return: migrate user to
+ :rtype: str
+ """
+ return self._platforms_parser.get_option(section, 'PROJECT', '').lower()
+
+ def set_new_user(self, section, new_user):
+ """
+ Sets new user for given platform
+ :param new_user:
+ :param section: platform name
+ :type: str
+ """
+ with open(self._platforms_parser_file) as p_file:
+ contentLine = p_file.readline()
+ contentToMod = ""
+ content = ""
+ mod = False
+ while contentLine:
+ if re.search(section, contentLine):
+ mod = True
+ if mod:
+ contentToMod += contentLine
+ else:
+ content += contentLine
+ contentLine = p_file.readline()
+ if mod:
+ old_user = self.get_current_user(section)
+ contentToMod = contentToMod.replace(re.search(
+ r'[^#]\bUSER\b =.*', contentToMod).group(0)[1:], "USER = " + new_user)
+ contentToMod = contentToMod.replace(re.search(
+ r'[^#]\bUSER_TO\b =.*', contentToMod).group(0)[1:], "USER_TO = " + old_user)
+ open(self._platforms_parser_file, 'w').write(content)
+ open(self._platforms_parser_file, 'a').write(contentToMod)
+
+ def get_migrate_project_to(self, section):
+ """
+ Returns the project to change to from platform config file.
+
+ :return: migrate project to
+ :rtype: str
+ """
+ return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower()
+
+ def set_new_project(self, section, new_project):
+ """
+ Sets new project for given platform
+ :param new_project:
+ :param section: platform name
+ :type: str
+ """
+ with open(self._platforms_parser_file) as p_file:
+ contentLine = p_file.readline()
+ contentToMod = ""
+ content = ""
+ mod = False
+ while contentLine:
+ if re.search(section, contentLine):
+ mod = True
+ if mod:
+ contentToMod += contentLine
+ else:
+ content += contentLine
+ contentLine = p_file.readline()
+ if mod:
+ old_project = self.get_current_project(section)
+ contentToMod = contentToMod.replace(re.search(
+ r"[^#]\bPROJECT\b =.*", contentToMod).group(0)[1:], "PROJECT = " + new_project)
+ contentToMod = contentToMod.replace(re.search(
+ r"[^#]\bPROJECT_TO\b =.*", contentToMod).group(0)[1:], "PROJECT_TO = " + old_project)
+ open(self._platforms_parser_file, 'w').write(content)
+ open(self._platforms_parser_file, 'a').write(contentToMod)
+
+ def get_custom_directives(self, section):
+ """
+ Gets custom directives needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: custom directives needed
+ :rtype: str
+ """
+ return str(self._jobs_parser.get_option(section, 'CUSTOM_DIRECTIVES', ''))
+
+ def check_conf_files(self):
+ """
+ Checks configuration files (autosubmit, experiment jobs and platforms), looking for invalid values, missing
+ required options. Prints results in log
+
+ :return: True if everything is correct, False if it finds any error
+ :rtype: bool
+ """
+ Log.debug('\nChecking configuration files...')
+ self.reload()
+ # result = self.check_platforms_conf()
+ result = True
+ result = result and self.check_jobs_conf()
+ result = result and self.check_autosubmit_conf()
+ result = result and self.check_expdef_conf()
+ if result:
+ Log.debug("Configuration files OK\n")
+ else:
+ Log.error("Configuration files invalid\n")
+ return result
+
+ def check_autosubmit_conf(self):
+ """
+ Checks experiment's autosubmit configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ result = True
+
+ self._conf_parser.read(self._conf_parser_file)
+ # result = result and self._conf_parser.check_exists(
+ # 'config', 'AUTOSUBMIT_VERSION')
+ # result = result and self._conf_parser.check_is_int(
+ # 'config', 'MAXWAITINGJOBS', True)
+ # result = result and self._conf_parser.check_is_int(
+ # 'config', 'TOTALJOBS', True)
+ # result = result and self._conf_parser.check_is_int(
+ # 'config', 'SAFETYSLEEPTIME', True)
+ # result = result and self._conf_parser.check_is_int(
+ # 'config', 'RETRIALS', True)
+ # result = result and self._conf_parser.check_is_boolean(
+ # 'mail', 'NOTIFICATIONS', False)
+ result = result and self.is_valid_communications_library()
+ result = result and self.is_valid_storage_type()
+ if self.get_wrapper_type() != 'None':
+ result = result and self.check_wrapper_conf()
+
+ if self.get_notifications() == 'true':
+ for mail in self.get_mails_to():
+ if not self.is_valid_mail_address(mail):
+ # Log.warning(
+ # 'One or more of the email addresses configured for the mail notifications are wrong')
+ break
+
+ if not result:
+ # Log.critical("{0} is not a valid config file".format(
+ # os.path.basename(self._conf_parser_file)))
+ raise Exception("Permission denied for " +
+ str(os.path.basename(self._conf_parser_file)))
+ else:
+ Log.debug('{0} OK'.format(
+ os.path.basename(self._conf_parser_file)))
+ return result
+
+ def check_platforms_conf(self):
+ """
+ Checks experiment's queues configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ result = True
+ if len(self._platforms_parser.sections()) == 0:
+ Log.warning("No remote platforms configured")
+
+ if len(self._platforms_parser.sections()) != len(set(self._platforms_parser.sections())):
+ Log.error('There are repeated platforms names')
+
+ for section in self._platforms_parser.sections():
+ result = result and self._platforms_parser.check_exists(
+ section, 'TYPE')
+ platform_type = self._platforms_parser.get_option(
+ section, 'TYPE', '').lower()
+ if platform_type != 'ps':
+ result = result and self._platforms_parser.check_exists(
+ section, 'PROJECT')
+ result = result and self._platforms_parser.check_exists(
+ section, 'USER')
+
+ result = result and self._platforms_parser.check_exists(
+ section, 'HOST')
+ result = result and self._platforms_parser.check_exists(
+ section, 'SCRATCH_DIR')
+ result = result and self._platforms_parser.check_is_boolean(section,
+ 'ADD_PROJECT_TO_HOST', False)
+ result = result and self._platforms_parser.check_is_boolean(
+ section, 'TEST_SUITE', False)
+ result = result and self._platforms_parser.check_is_int(section, 'MAX_WAITING_JOBS',
+ False)
+ result = result and self._platforms_parser.check_is_int(
+ section, 'TOTAL_JOBS', False)
+
+ if not result:
+ Log.critical("{0} is not a valid config file".format(
+ os.path.basename(self._platforms_parser_file)))
+ raise Exception("Permission denied for " +
+ str(os.path.basename(self._platforms_parser_file)))
+ else:
+ Log.info('{0} OK'.format(
+ os.path.basename(self._platforms_parser_file)))
+ return result
+
+ def check_jobs_conf(self):
+ """
+ Checks experiment's jobs configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ result = True
+ parser = self._jobs_parser
+ sections = parser.sections()
+ platforms = self._platforms_parser.sections()
+ platforms.append('LOCAL')
+ possible_exception = ""
+ if len(sections) == 0:
+ possible_exception += "No remote platforms configured\n"
+
+ # if len(sections) != len(set(sections)):
+ # Log.error('There are repeated job names')
+
+ for section in sections:
+
+ result = result and parser.check_exists(section, 'FILE')
+ result = result and parser.check_is_boolean(
+ section, 'RERUN_ONLY', False)
+
+ if parser.has_option(section, 'PLATFORM'):
+ # result = result and parser.check_is_choice(
+ # section, 'PLATFORM', False, platforms)
+ pass
+ if parser.has_option(section, 'DEPENDENCIES'):
+ for dependency in str(parser.get_option(section, 'DEPENDENCIES', '')).split(' '):
+ if '-' in dependency:
+ dependency = dependency.split('-')[0]
+ elif '+' in dependency:
+ dependency = dependency.split('+')[0]
+ if '[' in dependency:
+ dependency = dependency[:dependency.find('[')]
+ # if dependency not in sections:
+ # Log.error(
+ # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section,
+ # dependency))
+
+ if parser.has_option(section, 'RERUN_DEPENDENCIES'):
+ for dependency in str(parser.get_option(section, 'RERUN_DEPENDENCIES',
+ '')).split(' '):
+ if '-' in dependency:
+ dependency = dependency.split('-')[0]
+ if '[' in dependency:
+ dependency = dependency[:dependency.find('[')]
+ # if dependency not in sections:
+ # Log.error(
+ # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section,
+ # dependency))
+ result = result and parser.check_is_choice(section, 'RUNNING', False,
+ ['once', 'date', 'member', 'chunk'])
+
+ if not result:
+ # Log.critical("{0} is not a valid config file".format(
+ # os.path.basename(self._jobs_parser_file)))
+ raise Exception("Exception while checking jobs_expid.conf " +
+ str(os.path.basename(self._jobs_parser_file)) + possible_exception)
+ else:
+ Log.debug('{0} OK'.format(
+ os.path.basename(self._jobs_parser_file)))
+
+ return result
+
+ def check_expdef_conf(self):
+ """
+ Checks experiment's experiment configuration file.
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ result = True
+ parser = self._exp_parser
+
+ # result = result and parser.check_exists('DEFAULT', 'EXPID')
+ # result = result and parser.check_exists('DEFAULT', 'HPCARCH')
+
+ result = result and parser.check_exists('experiment', 'DATELIST')
+ result = result and parser.check_exists('experiment', 'MEMBERS')
+ result = result and parser.check_is_choice('experiment', 'CHUNKSIZEUNIT', True,
+ ['year', 'month', 'day', 'hour'])
+ result = result and parser.check_is_int(
+ 'experiment', 'CHUNKSIZE', True)
+ result = result and parser.check_is_int(
+ 'experiment', 'NUMCHUNKS', True)
+ result = result and parser.check_is_choice('experiment', 'CALENDAR', True,
+ ['standard', 'noleap'])
+
+ result = result and parser.check_is_boolean('rerun', 'RERUN', True)
+
+ if parser.check_is_choice('project', 'PROJECT_TYPE', True,
+ ['none', 'git', 'svn', 'local']):
+ project_type = parser.get_option('project', 'PROJECT_TYPE', '')
+
+ if project_type == 'git':
+ result = result and parser.check_exists(
+ 'git', 'PROJECT_ORIGIN')
+ result = result and parser.check_exists(
+ 'git', 'PROJECT_BRANCH')
+
+ elif project_type == 'svn':
+ result = result and parser.check_exists('svn', 'PROJECT_URL')
+ result = result and parser.check_exists(
+ 'svn', 'PROJECT_REVISION')
+ elif project_type == 'local':
+ result = result and parser.check_exists(
+ 'local', 'PROJECT_PATH')
+
+ if project_type != 'none':
+ result = result and parser.check_exists(
+ 'project_files', 'FILE_PROJECT_CONF')
+ else:
+ result = True
+
+ if not result:
+ # Log.critical("{0} is not a valid config file".format(
+ # os.path.basename(self._exp_parser_file)))
+ raise Exception("Permission denied for " +
+ str(os.path.basename(self._exp_parser_file)))
+ else:
+ Log.debug('{0} OK'.format(
+ os.path.basename(self._exp_parser_file)))
+ return result
+
+ def check_proj(self):
+ """
+ Checks project config file
+
+ :return: True if everything is correct, False if it founds any error
+ :rtype: bool
+ """
+ try:
+ if self._proj_parser_file == '':
+ self._proj_parser = None
+ else:
+ self._proj_parser = AutosubmitConfig.get_parser(
+ self.parser_factory, self._proj_parser_file)
+ return True
+ except Exception as e:
+ Log.error('Project conf file error: {0}', e)
+ return False
+
+ def check_wrapper_conf(self):
+ result = True
+ result = result and self.is_valid_jobs_in_wrapper()
+ if not result:
+ Log.error(
+ "There are sections in JOBS_IN_WRAPPER that are not defined in your jobs.conf file")
+
+ if 'horizontal' in self.get_wrapper_type():
+ result = result and self._platforms_parser.check_exists(
+ self.get_platform(), 'PROCESSORS_PER_NODE')
+ result = result and self._platforms_parser.check_exists(
+ self.get_platform(), 'MAX_PROCESSORS')
+ if 'vertical' in self.get_wrapper_type():
+ result = result and self._platforms_parser.check_exists(
+ self.get_platform(), 'MAX_WALLCLOCK')
+ return result
+
+ def reload(self):
+ """
+ Creates parser objects for configuration files
+ """
+ if not os.path.exists(self._conf_parser_file): raise IOError(
+ "Required file not found {0}".format(self._conf_parser_file))
+ if not os.path.exists(self._platforms_parser_file): raise IOError(
+ "Required file not found {0}".format(self._platforms_parser_file))
+ if not os.path.exists(self._jobs_parser_file): raise IOError(
+ "Required file not found {0}".format(self._jobs_parser_file))
+ if not os.path.exists(self._exp_parser_file): raise IOError(
+ "Required file not found {0}".format(self._exp_parser_file))
+ self._conf_parser = confConfigStrategy.get_parser(self.parser_factory, self._conf_parser_file)
+ self._platforms_parser = confConfigStrategy.get_parser(self.parser_factory, self._platforms_parser_file)
+ self._jobs_parser = confConfigStrategy.get_parser(self.parser_factory, self._jobs_parser_file)
+ self._exp_parser = confConfigStrategy.get_parser(self.parser_factory, self._exp_parser_file)
+ if self._proj_parser_file == '':
+ self._proj_parser = None
+ else:
+ self._proj_parser = confConfigStrategy.get_parser(self.parser_factory, self._proj_parser_file)
+
+ def load_parameters(self):
+ """
+ Load parameters from experiment and autosubmit config files. If experiment's type is not none,
+ also load parameters from model's config file
+
+ :return: a dictionary containing tuples [parameter_name, parameter_value]
+ :rtype: dict
+ """
+ parameters = dict()
+ for section in self._exp_parser.sections():
+ for option in self._exp_parser.options(section):
+ parameters[option] = self._exp_parser.get(section, option)
+ for section in self._conf_parser.sections():
+ for option in self._conf_parser.options(section):
+ parameters[option] = self._conf_parser.get(section, option)
+
+ # project_type = self.get_project_type()
+ # if project_type != "none" and self._proj_parser is not None:
+ # # Load project parameters
+ # Log.debug("Loading project parameters...")
+ # parameters2 = parameters.copy()
+ # parameters2.update(self.load_project_parameters())
+ # parameters = parameters2
+
+ return parameters
+
+ def load_project_parameters(self):
+ """
+ Loads parameters from model config file
+
+ :return: dictionary containing tuples [parameter_name, parameter_value]
+ :rtype: dict
+ """
+ pass
+ # projdef = []
+ # for section in self._proj_parser.sections():
+ # print("SEction" + section)
+ # projdef += self._proj_parser.items(section)
+
+ # parameters = dict()
+ # for item in projdef:
+ # parameters[item[0]] = item[1]
+
+ # return parameters
+
+ def set_expid(self, exp_id):
+ """
+ Set experiment identifier in autosubmit and experiment config files
+
+ :param exp_id: experiment identifier to store
+ :type exp_id: str
+ """
+ # Experiment conf
+ content = open(self._exp_parser_file).read()
+ if re.search('EXPID =.*', content):
+ content = content.replace(
+ re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id)
+ open(self._exp_parser_file, 'w').write(content)
+
+ content = open(self._conf_parser_file).read()
+ if re.search('EXPID =.*', content):
+ content = content.replace(
+ re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id)
+ open(self._conf_parser_file, 'w').write(content)
+
+ def get_project_type(self):
+ """
+ Returns project type from experiment config file
+
+ :return: project type
+ :rtype: str
+ """
+ return self._exp_parser.get_option('project', 'PROJECT_TYPE', "NA").lower()
+
+ def get_file_project_conf(self):
+ """
+ Returns path to project config file from experiment config file
+
+ :return: path to project config file
+ :rtype: str
+ """
+ return self._exp_parser.get('project_files', 'FILE_PROJECT_CONF')
+
+ def get_file_jobs_conf(self):
+ """
+ Returns path to project config file from experiment config file
+
+ :return: path to project config file
+ :rtype: str
+ """
+ return self._exp_parser.get_option('project_files', 'FILE_JOBS_CONF', '')
+
+ def get_git_project_origin(self):
+ """
+ Returns git origin from experiment config file
+
+ :return: git origin
+ :rtype: str
+ """
+ return self._exp_parser.get_option('git', 'PROJECT_ORIGIN', '')
+
+ def get_git_project_branch(self):
+ """
+ Returns git branch from experiment's config file
+
+ :return: git branch
+ :rtype: str
+ """
+ return self._exp_parser.get_option('git', 'PROJECT_BRANCH', 'master')
+
+ def get_git_project_commit(self):
+ """
+ Returns git commit from experiment's config file
+
+ :return: git commit
+ :rtype: str
+ """
+ return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None)
+
+ def get_submodules_list(self):
+ """
+ Returns submodules list from experiment's config file
+ Default is --recursive
+ :return: submodules to load
+ :rtype: list
+ """
+ return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES', '').split()).split()
+
+ def get_project_destination(self):
+ """
+ Returns git commit from experiment's config file
+
+ :return: git commit
+ :rtype: str
+ """
+ value = self._exp_parser.get('project', 'PROJECT_DESTINATION')
+ if not value:
+ if self.get_project_type().lower() == "local":
+ value = os.path.split(self.get_local_project_path())[1]
+ elif self.get_project_type().lower() == "svn":
+ value = self.get_svn_project_url().split('/')[-1]
+ elif self.get_project_type().lower() == "git":
+ value = self.get_git_project_origin().split(
+ '/')[-1].split('.')[-2]
+ return value
+
+ def set_git_project_commit(self, as_conf):
+ """
+ Function to register in the configuration the commit SHA of the git project version.
+ :param as_conf: Configuration class for exteriment
+ :type as_conf: AutosubmitConfig
+ """
+ full_project_path = as_conf.get_project_dir()
+ try:
+ output = subprocess.check_output(
+ "cd {0}; git rev-parse --abbrev-ref HEAD".format(full_project_path),
+ shell=True)
+ except subprocess.CalledProcessError:
+ Log.critical("Failed to retrieve project branch...")
+ return False
+
+ project_branch = output
+ Log.debug("Project branch is: " + project_branch)
+ try:
+ output = subprocess.check_output(
+ "cd {0}; git rev-parse HEAD".format(full_project_path), shell=True)
+ except subprocess.CalledProcessError:
+ Log.critical("Failed to retrieve project commit SHA...")
+ return False
+ project_sha = output
+ Log.debug("Project commit SHA is: " + project_sha)
+
+ # register changes
+ content = open(self._exp_parser_file).read()
+ if re.search('PROJECT_BRANCH =.*', content):
+ content = content.replace(re.search('PROJECT_BRANCH =.*', content).group(0),
+ "PROJECT_BRANCH = " + project_branch)
+ if re.search('PROJECT_COMMIT =.*', content):
+ content = content.replace(re.search('PROJECT_COMMIT =.*', content).group(0),
+ "PROJECT_COMMIT = " + project_sha)
+ open(self._exp_parser_file, 'w').write(content)
+ Log.debug(
+ "Project commit SHA succesfully registered to the configuration file.")
+ return True
+
+ def get_svn_project_url(self):
+ """
+ Gets subversion project url
+
+ :return: subversion project url
+ :rtype: str
+ """
+ return self._exp_parser.get_option('svn', 'PROJECT_URL', 'NA')
+
+ def get_svn_project_revision(self):
+ """
+ Get revision for subversion project
+
+ :return: revision for subversion project
+ :rtype: str
+ """
+ return self._exp_parser.get('svn', 'PROJECT_REVISION')
+
+ def get_local_project_path(self):
+ """
+ Gets path to origin for local project
+
+ :return: path to local project
+ :rtype: str
+ """
+ return self._exp_parser.get('local', 'PROJECT_PATH')
+
+ def get_date_list(self):
+ """
+ Returns startdates list from experiment's config file
+
+ :return: experiment's startdates
+ :rtype: list
+ """
+ date_list = list()
+ string = self._exp_parser.get('experiment', 'DATELIST')
+ if not string.startswith("["):
+ string = '[{0}]'.format(string)
+ split_string = nestedExpr('[', ']').parseString(string).asList()
+ string_date = None
+ for split in split_string[0]:
+ if type(split) is list:
+ for split_in in split:
+ if split_in.find("-") != -1:
+ numbers = split_in.split("-")
+ for count in range(int(numbers[0]), int(numbers[1]) + 1):
+ date_list.append(parse_date(
+ string_date + str(count).zfill(len(numbers[0]))))
+ else:
+ date_list.append(parse_date(string_date + split_in))
+ string_date = None
+ else:
+ if string_date is not None:
+ date_list.append(parse_date(string_date))
+ string_date = split
+ if string_date is not None:
+ date_list.append(parse_date(string_date))
+ return date_list
+
+ def get_num_chunks(self):
+ """
+ Returns number of chunks to run for each member
+
+ :return: number of chunks
+ :rtype: int
+ """
+ return int(self._exp_parser.get('experiment', 'NUMCHUNKS'))
+
+ def get_chunk_ini(self, default=1):
+ """
+ Returns the first chunk from where the experiment will start
+
+ :param default:
+ :return: initial chunk
+ :rtype: int
+ """
+ chunk_ini = self._exp_parser.get_option(
+ 'experiment', 'CHUNKINI', default)
+ if chunk_ini == '':
+ return default
+ return int(chunk_ini)
+
+ def get_chunk_size_unit(self):
+ # type: () -> str
+ """
+ Unit for the chunk length
+
+ :return: Unit for the chunk length Options: {hour, day, month, year}
+ :rtype: str
+ """
+
+ # try:
+ # res = self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower()
+ # except Exception as e:
+ # try:
+ # res = self.autosubmit_conf.get('experiment')
+ return self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower()
+
+ def get_chunk_size(self, default=1):
+ # type: (int) -> int
+ """
+ Chunk Size as defined in the expdef file.
+
+ :return: Chunksize, 1 as default.
+ :rtype: int
+ """
+ try:
+ chunk_size = self._exp_parser.get_option(
+ 'experiment', 'CHUNKSIZE', default)
+ except Exception as exp:
+ print(exp)
+ chunk_size = ''
+ pass
+ if chunk_size == '':
+ return default
+ return int(chunk_size)
+
+ def get_member_list(self, run_only=False):
+ """
+ Returns members list from experiment's config file
+
+ :return: experiment's members
+ :rtype: list
+ """
+ member_list = list()
+ string = self._exp_parser.get('experiment',
+ 'MEMBERS') if run_only == False else self._exp_parser.get_option(
+ 'experiment', 'RUN_ONLY_MEMBERS', '')
+ if not string.startswith("["):
+ string = '[{0}]'.format(string)
+ split_string = nestedExpr('[', ']').parseString(string).asList()
+ string_member = None
+ for split in split_string[0]:
+ if type(split) is list:
+ for split_in in split:
+ if split_in.find("-") != -1:
+ numbers = split_in.split("-")
+ for count in range(int(numbers[0]), int(numbers[1]) + 1):
+ member_list.append(
+ string_member + str(count).zfill(len(numbers[0])))
+ else:
+ member_list.append(string_member + split_in)
+ string_member = None
+ else:
+ if string_member is not None:
+ member_list.append(string_member)
+ string_member = split
+ if string_member is not None:
+ member_list.append(string_member)
+ return member_list
+
+ def get_rerun(self):
+ """
+ Returns startdates list from experiment's config file
+
+ :return: rerurn value
+ :rtype: list
+ """
+
+ return self._exp_parser.get('rerun', 'RERUN').lower()
+
+ def get_chunk_list(self):
+ """
+ Returns chunk list from experiment's config file
+
+ :return: experiment's chunks
+ :rtype: list
+ """
+ return self._exp_parser.get('rerun', 'CHUNKLIST')
+
+ def get_platform(self):
+ """
+ Returns main platforms from experiment's config file
+
+ :return: main platforms
+ :rtype: str
+ """
+ return self._exp_parser.get('experiment', 'HPCARCH')
+
+ def set_platform(self, hpc):
+ """
+ Sets main platforms in experiment's config file
+
+ :param hpc: main platforms
+ :type: str
+ """
+ content = open(self._exp_parser_file).read()
+ if re.search('HPCARCH =.*', content):
+ content = content.replace(
+ re.search('HPCARCH =.*', content).group(0), "HPCARCH = " + hpc)
+ open(self._exp_parser_file, 'w').write(content)
+
+ def set_version(self, autosubmit_version):
+ """
+ Sets autosubmit's version in autosubmit's config file
+
+ :param autosubmit_version: autosubmit's version
+ :type autosubmit_version: str
+ """
+ content = open(self._conf_parser_file).read()
+ if re.search('AUTOSUBMIT_VERSION =.*', content):
+ content = content.replace(re.search('AUTOSUBMIT_VERSION =.*', content).group(0),
+ "AUTOSUBMIT_VERSION = " + autosubmit_version)
+ open(self._conf_parser_file, 'w').write(content)
+
+ def get_version(self):
+ """
+ Returns version number of the current experiment from autosubmit's config file
+
+ :return: version
+ :rtype: str
+ """
+ return self._conf_parser.get_option('config', 'AUTOSUBMIT_VERSION', 'None')
+
+ def get_total_jobs(self):
+ """
+ Returns max number of running jobs from autosubmit's config file
+
+ :return: max number of running jobs
+ :rtype: int
+ """
+ return int(self._conf_parser.get('config', 'TOTALJOBS'))
+
+ def get_max_wallclock(self):
+ """
+ Returns max wallclock
+
+ :rtype: str
+ """
+ return self._conf_parser.get_option('config', 'MAX_WALLCLOCK', '')
+
+ def get_max_processors(self):
+ """
+ Returns max processors from autosubmit's config file
+
+ :rtype: str
+ """
+ config_value = self._conf_parser.get_option(
+ 'config', 'MAX_PROCESSORS', None)
+ return int(config_value) if config_value is not None else config_value
+
+ def get_max_waiting_jobs(self):
+ """
+ Returns max number of waiting jobs from autosubmit's config file
+
+ :return: main platforms
+ :rtype: int
+ """
+ return int(self._conf_parser.get_option('config', 'MAXWAITINGJOBS', 10))
+
+ def get_default_job_type(self):
+ """
+ Returns the default job type from experiment's config file
+
+ :return: default type such as bash, python, r..
+ :rtype: str
+ """
+ return self._exp_parser.get_option('project_files', 'JOB_SCRIPTS_TYPE', 'bash')
+
+ def get_safetysleeptime(self):
+ """
+ Returns safety sleep time from autosubmit's config file
+
+ :return: safety sleep time
+ :rtype: int
+ """
+ return int(self._conf_parser.get_option('config', 'SAFETYSLEEPTIME', 10))
+
+ def set_safetysleeptime(self, sleep_time):
+ """
+ Sets autosubmit's version in autosubmit's config file
+
+ :param sleep_time: value to set
+ :type sleep_time: int
+ """
+ content = open(self._conf_parser_file).read()
+ content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0),
+ "SAFETYSLEEPTIME = %d" % sleep_time)
+ open(self._conf_parser_file, 'w').write(content)
+
+ def get_retrials(self):
+ """
+ Returns max number of retrials for job from autosubmit's config file
+
+ :return: safety sleep time
+ :rtype: int
+ """
+ return int(self._conf_parser.get('config', 'RETRIALS'))
+
+ def get_notifications(self):
+ """
+ Returns if the user has enabled the notifications from autosubmit's config file
+
+ :return: if notifications
+ :rtype: string
+ """
+ return self._conf_parser.get_option('mail', 'NOTIFICATIONS', 'false').lower()
+
+ def get_remote_dependencies(self):
+ """
+ Returns if the user has enabled the remote dependencies from autosubmit's config file
+
+ :return: if remote dependencies
+ :rtype: bool
+ """
+ return self._conf_parser.get_option('wrapper', 'DEPENDENCIES', 'false').lower() == 'true'
+
+ def get_wrapper_type(self):
+ """
+ Returns what kind of wrapper (VERTICAL, MIXED-VERTICAL, HORIZONTAL, HYBRID, NONE) the user has configured in the autosubmit's config
+
+ :return: wrapper type (or none)
+ :rtype: string
+ """
+ return self._conf_parser.get_option('wrapper', 'TYPE', 'None').lower()
+
+ def get_wrapper_jobs(self):
+ """
+ Returns the jobs that should be wrapped, configured in the autosubmit's config
+
+ :return: expression (or none)
+ :rtype: string
+ """
+ return self._conf_parser.get_option('wrapper', 'JOBS_IN_WRAPPER', 'None')
+
+ def get_max_wrapped_jobs(self):
+ """
+ Returns the maximum number of jobs that can be wrapped together as configured in autosubmit's config file
+
+ :return: maximum number of jobs (or total jobs)
+ :rtype: string
+ """
+ # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs()))
+
+ return int(self._conf_parser.get_option('wrapper', 'MAX_WRAPPED', self.get_total_jobs()))
+
+ def get_wrapper_check_time(self):
+ """
+ Returns time to check the status of jobs in the wrapper
+
+ :return: wrapper check time
+ :rtype: int
+ """
+ return int(self._conf_parser.get_option('wrapper', 'CHECK_TIME_WRAPPER', self.get_safetysleeptime()))
+
+ def get_wrapper_machinefiles(self):
+ """
+ Returns the strategy for creating the machinefiles in wrapper jobs
+
+ :return: machinefiles function to use
+ :rtype: string
+ """
+ return self._conf_parser.get_option('wrapper', 'MACHINEFILES', '')
+
+ def get_wrapper_queue(self):
+ """
+ Returns the wrapper queue if not defined, will be the one of the first job wrapped
+
+ :return: expression (or none)
+ :rtype: string
+ """
+ return self._conf_parser.get_option('wrapper', 'QUEUE', "")
+
+ def get_jobs_sections(self):
+ """
+ Returns the list of sections defined in the jobs config file
+
+ :return: sections
+ :rtype: list
+ """
+ return self._jobs_parser.sections()
+
+ def get_copy_remote_logs(self):
+ """
+ Returns if the user has enabled the logs local copy from autosubmit's config file
+
+ :return: if logs local copy
+ :rtype: bool
+ """
+ return self._conf_parser.get_option('storage', 'COPY_REMOTE_LOGS', 'true').lower()
+
+ def get_mails_to(self):
+ """
+ Returns the address where notifications will be sent from autosubmit's config file
+
+ :return: mail address
+ :rtype: [str]
+ """
+ return [str(x) for x in self._conf_parser.get_option('mail', 'TO', '').split(' ')]
+
+ def get_communications_library(self):
+ """
+ Returns the communications library from autosubmit's config file. Paramiko by default.
+
+ :return: communications library
+ :rtype: str
+ """
+ return self._conf_parser.get_option('communications', 'API', 'paramiko').lower()
+
+ def get_storage_type(self):
+ """
+ Returns the communications library from autosubmit's config file. Paramiko by default.
+
+ :return: communications library
+ :rtype: str
+ """
+ return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower()
+
+ @staticmethod
+ def is_valid_mail_address(mail_address):
+ if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address):
+ return True
+ else:
+ return False
+
+ def is_valid_communications_library(self):
+ library = self.get_communications_library()
+ return library in ['paramiko', 'saga']
+
+ def is_valid_storage_type(self):
+ storage_type = self.get_storage_type()
+ return storage_type in ['pkl', 'db']
+
+ def is_valid_jobs_in_wrapper(self):
+ expression = self.get_wrapper_jobs()
+ if expression != 'None':
+ parser = self._jobs_parser
+ sections = parser.sections()
+ for section in expression.split(" "):
+ if "&" in section:
+ for inner_section in section.split("&"):
+ if inner_section not in sections:
+ return False
+ elif section not in sections:
+ return False
+ return True
+
+ def is_valid_git_repository(self):
+ origin_exists = self._exp_parser.check_exists('git', 'PROJECT_ORIGIN')
+ branch = self.get_git_project_branch()
+ commit = self.get_git_project_commit()
+ return origin_exists and (branch is not None or commit is not None)
+
+ @staticmethod
+ def get_parser(parser_factory, file_path):
+ # type: (ConfigParserFactory, str) -> ConfigParser
+ """
+ Gets parser for given file
+
+ :param parser_factory:
+ :param file_path: path to file to be parsed
+ :type file_path: str
+ :return: parser
+ :rtype: SafeConfigParser
+ """
+ parser = parser_factory.create_parser()
+ parser.optionxform = str
+ # proj is not required
+ # print(file_path)
+ if file_path.find('proj_') > 0:
+ parser.read(file_path)
+ else:
+ with open(file_path) as f:
+ parser.read(file_path)
+ return parser
diff --git a/autosubmit_api/config/config_common.py b/autosubmit_api/config/config_common.py
index c73453711ed51bba574a90a0bd94b2e058320160..ca5cc4bca38fddfe2ce6eb4abe2ea241d77550dc 100644
--- a/autosubmit_api/config/config_common.py
+++ b/autosubmit_api/config/config_common.py
@@ -19,20 +19,29 @@
try:
# noinspection PyCompatibility
from configparser import SafeConfigParser
+ from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config
except ImportError:
# noinspection PyCompatibility
from configparser import SafeConfigParser
+
+
import os
import re
import subprocess
import json
+import logging
+import locale
from pyparsing import nestedExpr
from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser
from bscearth.utils.date import parse_date
from bscearth.utils.log import Log
from ..config.basicConfig import BasicConfig
+from ..config.IConfigStrategy import IConfigStrategy
+from ..config.ymlConfigStrategy import ymlConfigStrategy
+from ..config.confConfigStrategy import confConfigStrategy
+logger = logging.getLogger('gunicorn.error')
class AutosubmitConfig(object):
"""
@@ -40,70 +49,39 @@ class AutosubmitConfig(object):
:param expid: experiment identifier
:type expid: str
+ :configWrapper: IConfigStrategy -> handling strategy for the type of config files used
"""
- def __init__(self, expid, basic_config, parser_factory, extension=".yml"):
+ def __init__(self, expid, basic_config, parser_factory, extension="yml"):
# type: (str, BasicConfig, ConfigParserFactory, Extension) -> None
- self.expid = expid
+ self.expid = expid
+ self._configWrapper = None
self.basic_config = basic_config
-
self.parser_factory = parser_factory
- # By default check for .yml files first as it is the new standard for AS 4.0
-
- self._conf_parser = None # type: ConfigParser
- self._conf_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "autosubmit_" + expid + extension)
- if os.path.exists(self._conf_parser_file) == False:
- if extension == ".yml":
- self.__init__(expid, basic_config, parser_factory, ".conf")
- elif extension == ".conf":
- return None
-
- self._exp_parser = None # type: ConfigParser
- self._exp_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "expdef_" + expid + extension)
- if os.path.exists(self._exp_parser_file) == False:
- if extension == ".yml":
- self.__init__(expid, basic_config, parser_factory, ".conf")
- elif extension == ".conf":
- return None
-
- self._platforms_parser = None # type: ConfigParser
- self._platforms_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + extension)
- if os.path.exists(self._platforms_parser_file) == False:
- if extension == ".yml":
- self.__init__(expid, basic_config, parser_factory, ".conf")
- elif extension == ".conf":
- return None
-
- self._jobs_parser = None # type: ConfigParser
- self._jobs_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "jobs_" + expid + extension)
- if os.path.exists(self._jobs_parser_file) == False:
- if extension == ".yml":
- self.__init__(expid, basic_config, parser_factory, ".conf")
- elif extension == ".conf":
- return None
-
- self._proj_parser = None # type: ConfigParser
- self._proj_parser_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "proj_" + expid + extension)
- if os.path.exists(self._proj_parser_file) == False:
- if extension == ".yml":
- self.__init__(expid, basic_config, parser_factory, ".conf")
- elif extension == ".conf":
- return None
-
- self.check_proj_file()
+ # check which type of config files (AS3 or AS4)
+ platform_conf_file = os.path.join(self.basic_config.LOCAL_ROOT_DIR, expid, "conf", "platforms_" + expid + "." + extension)
+ if os.path.exists(platform_conf_file):
+ logger.info("Setting AS4 Config strategy - yml")
+ self._configWrapper = ymlConfigStrategy(expid, basic_config, parser_factory, ".yml")
+ else:
+ logger.info("Setting AS3 Config strategy - conf")
+ self._configWrapper = confConfigStrategy(expid, basic_config, parser_factory, ".conf")
+
@property
def jobs_parser(self):
- return self._jobs_parser
+ return self._configWrapper.jobs_parser
@property
def experiment_file(self):
"""
Returns experiment's config file name
"""
- return self._exp_parser_file
+ #return self._exp_parser_file
+ return self._configWrapper.experiment_file
+
@property
def platforms_parser(self):
@@ -113,7 +91,7 @@ class AutosubmitConfig(object):
:return: platforms config parser object
:rtype: SafeConfigParser
"""
- return self._platforms_parser
+ return self._configWrapper.platforms_parser
@property
def platforms_file(self):
@@ -123,27 +101,19 @@ class AutosubmitConfig(object):
:return: platforms config file's name
:rtype: str
"""
- return self._platforms_parser_file
+ return self._configWrapper.platforms_parser_file
@property
def project_file(self):
"""
Returns project's config file name
"""
- return self._proj_parser_file
+ return self._configWrapper.project_file
def check_proj_file(self):
"""
Add a section header to the project's configuration file (if not exists)
"""
- # if os.path.exists(self._proj_parser_file):
- # with open(self._proj_parser_file, 'w') as f:
- # first_line = f.readline()
- # if not re.match('[[a-zA-Z0-9]*]', first_line):
- # content = f.read()
- # f.seek(0, 0)
- # f.write('[DEFAULT]'.rstrip('\r\n') +
- # '\n' + first_line + content)
pass
@property
@@ -151,45 +121,19 @@ class AutosubmitConfig(object):
"""
Returns project's jobs file name
"""
- return self._jobs_parser_file
+ return self._configWrapper.jobs_file()
def get_full_config_as_dict(self):
"""
Returns full configuration as json object
"""
- _conf = _exp = _platforms = _jobs = _proj = None
- result = {}
-
- def get_data(parser):
- """
- dictionary comprehension to get data from parser
- """
- res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [
- section for section in parser.sections()]}
- return res
- # print(self._conf_parser)
- result["conf"] = get_data(
- self._conf_parser) if self._conf_parser else None
- result["exp"] = get_data(
- self._exp_parser) if self._exp_parser else None
- result["platforms"] = get_data(
- self._platforms_parser) if self._platforms_parser else None
- result["jobs"] = get_data(
- self._jobs_parser) if self._jobs_parser else None
- result["proj"] = get_data(
- self._proj_parser) if self._proj_parser else None
- return result
+ return self._configWrapper.get_full_config_as_dict()
def get_full_config_as_json(self):
"""
Return config as json object
"""
- try:
- return json.dumps(self.get_full_config_as_dict())
- except Exception as exp:
- Log.warning(
- "Autosubmit was not able to retrieve and save the configuration into the historical database.")
- return ""
+ return self._configWrapper.get_full_config_as_json()
def get_project_dir(self):
"""
@@ -198,9 +142,8 @@ class AutosubmitConfig(object):
:return: experiment's project directory
:rtype: str
"""
- dir_templates = os.path.join(self.basic_config.LOCAL_ROOT_DIR, self.expid, BasicConfig.LOCAL_PROJ_DIR,
- self.get_project_destination())
- return dir_templates
+ return self._configWrapper.get_project_dir()
+
def get_queue(self, section):
"""
@@ -210,22 +153,22 @@ class AutosubmitConfig(object):
:return: queue
:rtype: str
"""
- return self._jobs_parser.get_option(section, 'QUEUE', '')
+ return self._configWrapper.get_queue(section)
def get_job_platform(self, section):
- return self._jobs_parser.get_option(section, 'PLATFORM', '')
+ return self._configWrapper.get_job_platform(section)
def get_platform_queue(self, platform):
- return self._platforms_parser.get_option(platform, 'QUEUE', '')
+ return self._configWrapper.get_platform_queue(platform)
def get_platform_serial_queue(self, platform):
- return self._platforms_parser.get_option(platform, 'SERIAL_QUEUE', '')
+ return self._configWrapper.get_platform_serial_queue(platform)
def get_platform_project(self, platform):
- return self._platforms_parser.get_option(platform, "PROJECT", "")
+ return self._configWrapper.get_platform_project(platform)
def get_platform_wallclock(self, platform):
- return self._platforms_parser.get_option(platform, 'MAX_WALLCLOCK', '')
+ return self._configWrapper.get_platform_wallclock(platform)
def get_wallclock(self, section):
"""
@@ -235,7 +178,7 @@ class AutosubmitConfig(object):
:return: wallclock time
:rtype: str
"""
- return self._jobs_parser.get_option(section, 'WALLCLOCK', '')
+ return self._configWrapper.get_wallclock(section)
def get_synchronize(self, section):
"""
@@ -245,7 +188,7 @@ class AutosubmitConfig(object):
:return: wallclock time
:rtype: str
"""
- return self._jobs_parser.get_option(section, 'SYNCHRONIZE', '')
+ return self._configWrapper.get_synchronize(section)
def get_processors(self, section):
"""
@@ -255,7 +198,7 @@ class AutosubmitConfig(object):
:return: wallclock time
:rtype: str
"""
- return str(self._jobs_parser.get_option(section, 'PROCESSORS', 1))
+ return self._configWrapper.get_processors(section)
def get_threads(self, section):
"""
@@ -265,7 +208,7 @@ class AutosubmitConfig(object):
:return: threads needed
:rtype: str
"""
- return str(self._jobs_parser.get_option(section, 'THREADS', 1))
+ return self._configWrapper.get_threads(section)
def get_tasks(self, section):
"""
@@ -275,7 +218,7 @@ class AutosubmitConfig(object):
:return: tasks (processes) per host
:rtype: str
"""
- return str(self._jobs_parser.get_option(section, 'TASKS', 0))
+ return self._configWrapper.get_tasks(section)
def get_scratch_free_space(self, section):
"""
@@ -285,7 +228,7 @@ class AutosubmitConfig(object):
:return: percentage of scratch free space needed
:rtype: int
"""
- return int(self._jobs_parser.get_option(section, 'SCRATCH_FREE_SPACE', 0))
+ return self._configWrapper.get_scratch_free_space(section)
def get_memory(self, section):
"""
@@ -295,7 +238,7 @@ class AutosubmitConfig(object):
:return: memory needed
:rtype: str
"""
- return str(self._jobs_parser.get_option(section, 'MEMORY', ''))
+ return self._configWrapper.get_memory(section)
def get_memory_per_task(self, section):
"""
@@ -305,7 +248,7 @@ class AutosubmitConfig(object):
:return: memory per task needed
:rtype: str
"""
- return str(self._jobs_parser.get_option(section, 'MEMORY_PER_TASK', ''))
+ return self._configWrapper.get_memory_per_task(section)
def get_migrate_user_to(self, section):
"""
@@ -314,7 +257,7 @@ class AutosubmitConfig(object):
:return: migrate user to
:rtype: str
"""
- return self._platforms_parser.get_option(section, 'USER_TO', '').lower()
+ return self.get_migrate_user_to(section)
def get_current_user(self, section):
"""
@@ -323,7 +266,7 @@ class AutosubmitConfig(object):
:return: migrate user to
:rtype: str
"""
- return self._platforms_parser.get_option(section, 'USER', '').lower()
+ return self._configWrapper.get_current_user(section)
def get_current_project(self, section):
"""
@@ -332,7 +275,7 @@ class AutosubmitConfig(object):
:return: migrate user to
:rtype: str
"""
- return self._platforms_parser.get_option(section, 'PROJECT', '').lower()
+ return self._configWrapper.get_current_project(section)
def set_new_user(self, section, new_user):
"""
@@ -341,27 +284,7 @@ class AutosubmitConfig(object):
:param section: platform name
:type: str
"""
- with open(self._platforms_parser_file) as p_file:
- contentLine = p_file.readline()
- contentToMod = ""
- content = ""
- mod = False
- while contentLine:
- if re.search(section, contentLine):
- mod = True
- if mod:
- contentToMod += contentLine
- else:
- content += contentLine
- contentLine = p_file.readline()
- if mod:
- old_user = self.get_current_user(section)
- contentToMod = contentToMod.replace(re.search(
- r'[^#]\bUSER\b =.*', contentToMod).group(0)[1:], "USER = " + new_user)
- contentToMod = contentToMod.replace(re.search(
- r'[^#]\bUSER_TO\b =.*', contentToMod).group(0)[1:], "USER_TO = " + old_user)
- open(self._platforms_parser_file, 'w').write(content)
- open(self._platforms_parser_file, 'a').write(contentToMod)
+ self._configWrapper.set_new_user(section,new_user)
def get_migrate_project_to(self, section):
"""
@@ -370,7 +293,8 @@ class AutosubmitConfig(object):
:return: migrate project to
:rtype: str
"""
- return self._platforms_parser.get_option(section, 'PROJECT_TO', '').lower()
+ return self._configWrapper.get_migrate_project_to(section)
+
def set_new_project(self, section, new_project):
"""
@@ -379,27 +303,7 @@ class AutosubmitConfig(object):
:param section: platform name
:type: str
"""
- with open(self._platforms_parser_file) as p_file:
- contentLine = p_file.readline()
- contentToMod = ""
- content = ""
- mod = False
- while contentLine:
- if re.search(section, contentLine):
- mod = True
- if mod:
- contentToMod += contentLine
- else:
- content += contentLine
- contentLine = p_file.readline()
- if mod:
- old_project = self.get_current_project(section)
- contentToMod = contentToMod.replace(re.search(
- r"[^#]\bPROJECT\b =.*", contentToMod).group(0)[1:], "PROJECT = " + new_project)
- contentToMod = contentToMod.replace(re.search(
- r"[^#]\bPROJECT_TO\b =.*", contentToMod).group(0)[1:], "PROJECT_TO = " + old_project)
- open(self._platforms_parser_file, 'w').write(content)
- open(self._platforms_parser_file, 'a').write(contentToMod)
+ self._configWrapper.set_new_project(section,new_project)
def get_custom_directives(self, section):
"""
@@ -409,7 +313,7 @@ class AutosubmitConfig(object):
:return: custom directives needed
:rtype: str
"""
- return str(self._jobs_parser.get_option(section, 'CUSTOM_DIRECTIVES', ''))
+ return self._configWrapper.get_custom_directives(section)
def check_conf_files(self):
"""
@@ -419,18 +323,7 @@ class AutosubmitConfig(object):
:return: True if everything is correct, False if it finds any error
:rtype: bool
"""
- Log.debug('\nChecking configuration files...')
- self.reload()
- # result = self.check_platforms_conf()
- result = True
- result = result and self.check_jobs_conf()
- result = result and self.check_autosubmit_conf()
- result = result and self.check_expdef_conf()
- if result:
- Log.debug("Configuration files OK\n")
- else:
- Log.error("Configuration files invalid\n")
- return result
+ return self._configWrapper.check_conf_files()
def check_autosubmit_conf(self):
"""
@@ -439,42 +332,7 @@ class AutosubmitConfig(object):
:return: True if everything is correct, False if it founds any error
:rtype: bool
"""
- result = True
-
- self._conf_parser.read(self._conf_parser_file)
- # result = result and self._conf_parser.check_exists(
- # 'config', 'AUTOSUBMIT_VERSION')
- # result = result and self._conf_parser.check_is_int(
- # 'config', 'MAXWAITINGJOBS', True)
- # result = result and self._conf_parser.check_is_int(
- # 'config', 'TOTALJOBS', True)
- # result = result and self._conf_parser.check_is_int(
- # 'config', 'SAFETYSLEEPTIME', True)
- # result = result and self._conf_parser.check_is_int(
- # 'config', 'RETRIALS', True)
- # result = result and self._conf_parser.check_is_boolean(
- # 'mail', 'NOTIFICATIONS', False)
- result = result and self.is_valid_communications_library()
- result = result and self.is_valid_storage_type()
- if self.get_wrapper_type() != 'None':
- result = result and self.check_wrapper_conf()
-
- if self.get_notifications() == 'true':
- for mail in self.get_mails_to():
- if not self.is_valid_mail_address(mail):
- # Log.warning(
- # 'One or more of the email addresses configured for the mail notifications are wrong')
- break
-
- if not result:
- # Log.critical("{0} is not a valid config file".format(
- # os.path.basename(self._conf_parser_file)))
- raise Exception("Permission denied for " +
- str(os.path.basename(self._conf_parser_file)))
- else:
- Log.debug('{0} OK'.format(
- os.path.basename(self._conf_parser_file)))
- return result
+ return self._configWrapper.check_autosubmit_conf()
def check_platforms_conf(self):
"""
@@ -483,46 +341,7 @@ class AutosubmitConfig(object):
:return: True if everything is correct, False if it founds any error
:rtype: bool
"""
- result = True
- if len(self._platforms_parser.sections()) == 0:
- Log.warning("No remote platforms configured")
-
- if len(self._platforms_parser.sections()) != len(set(self._platforms_parser.sections())):
- Log.error('There are repeated platforms names')
-
- for section in self._platforms_parser.sections():
- result = result and self._platforms_parser.check_exists(
- section, 'TYPE')
- platform_type = self._platforms_parser.get_option(
- section, 'TYPE', '').lower()
- if platform_type != 'ps':
- result = result and self._platforms_parser.check_exists(
- section, 'PROJECT')
- result = result and self._platforms_parser.check_exists(
- section, 'USER')
-
- result = result and self._platforms_parser.check_exists(
- section, 'HOST')
- result = result and self._platforms_parser.check_exists(
- section, 'SCRATCH_DIR')
- result = result and self._platforms_parser.check_is_boolean(section,
- 'ADD_PROJECT_TO_HOST', False)
- result = result and self._platforms_parser.check_is_boolean(
- section, 'TEST_SUITE', False)
- result = result and self._platforms_parser.check_is_int(section, 'MAX_WAITING_JOBS',
- False)
- result = result and self._platforms_parser.check_is_int(
- section, 'TOTAL_JOBS', False)
-
- if not result:
- Log.critical("{0} is not a valid config file".format(
- os.path.basename(self._platforms_parser_file)))
- raise Exception("Permission denied for " +
- str(os.path.basename(self._platforms_parser_file)))
- else:
- Log.info('{0} OK'.format(
- os.path.basename(self._platforms_parser_file)))
- return result
+ return self._configWrapper.check_platforms_conf()
def check_jobs_conf(self):
"""
@@ -531,65 +350,7 @@ class AutosubmitConfig(object):
:return: True if everything is correct, False if it founds any error
:rtype: bool
"""
- result = True
- parser = self._jobs_parser
- sections = parser.sections()
- platforms = self._platforms_parser.sections()
- platforms.append('LOCAL')
- possible_exception = ""
- if len(sections) == 0:
- possible_exception += "No remote platforms configured\n"
-
- # if len(sections) != len(set(sections)):
- # Log.error('There are repeated job names')
-
- for section in sections:
-
- result = result and parser.check_exists(section, 'FILE')
- result = result and parser.check_is_boolean(
- section, 'RERUN_ONLY', False)
-
- if parser.has_option(section, 'PLATFORM'):
- # result = result and parser.check_is_choice(
- # section, 'PLATFORM', False, platforms)
- pass
- if parser.has_option(section, 'DEPENDENCIES'):
- for dependency in str(parser.get_option(section, 'DEPENDENCIES', '')).split(' '):
- if '-' in dependency:
- dependency = dependency.split('-')[0]
- elif '+' in dependency:
- dependency = dependency.split('+')[0]
- if '[' in dependency:
- dependency = dependency[:dependency.find('[')]
- # if dependency not in sections:
- # Log.error(
- # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section,
- # dependency))
-
- if parser.has_option(section, 'RERUN_DEPENDENCIES'):
- for dependency in str(parser.get_option(section, 'RERUN_DEPENDENCIES',
- '')).split(' '):
- if '-' in dependency:
- dependency = dependency.split('-')[0]
- if '[' in dependency:
- dependency = dependency[:dependency.find('[')]
- # if dependency not in sections:
- # Log.error(
- # 'Job {0} depends on job {1} that is not defined. It will be ignored.'.format(section,
- # dependency))
- result = result and parser.check_is_choice(section, 'RUNNING', False,
- ['once', 'date', 'member', 'chunk'])
-
- if not result:
- # Log.critical("{0} is not a valid config file".format(
- # os.path.basename(self._jobs_parser_file)))
- raise Exception("Exception while checking jobs_expid.conf " +
- str(os.path.basename(self._jobs_parser_file)) + possible_exception)
- else:
- Log.debug('{0} OK'.format(
- os.path.basename(self._jobs_parser_file)))
-
- return result
+ return self._configWrapper.check_jobs_conf()
def check_expdef_conf(self):
"""
@@ -598,58 +359,7 @@ class AutosubmitConfig(object):
:return: True if everything is correct, False if it founds any error
:rtype: bool
"""
- result = True
- parser = self._exp_parser
-
- # result = result and parser.check_exists('DEFAULT', 'EXPID')
- # result = result and parser.check_exists('DEFAULT', 'HPCARCH')
-
- result = result and parser.check_exists('experiment', 'DATELIST')
- result = result and parser.check_exists('experiment', 'MEMBERS')
- result = result and parser.check_is_choice('experiment', 'CHUNKSIZEUNIT', True,
- ['year', 'month', 'day', 'hour'])
- result = result and parser.check_is_int(
- 'experiment', 'CHUNKSIZE', True)
- result = result and parser.check_is_int(
- 'experiment', 'NUMCHUNKS', True)
- result = result and parser.check_is_choice('experiment', 'CALENDAR', True,
- ['standard', 'noleap'])
-
- result = result and parser.check_is_boolean('rerun', 'RERUN', True)
-
- if parser.check_is_choice('project', 'PROJECT_TYPE', True,
- ['none', 'git', 'svn', 'local']):
- project_type = parser.get_option('project', 'PROJECT_TYPE', '')
-
- if project_type == 'git':
- result = result and parser.check_exists(
- 'git', 'PROJECT_ORIGIN')
- result = result and parser.check_exists(
- 'git', 'PROJECT_BRANCH')
-
- elif project_type == 'svn':
- result = result and parser.check_exists('svn', 'PROJECT_URL')
- result = result and parser.check_exists(
- 'svn', 'PROJECT_REVISION')
- elif project_type == 'local':
- result = result and parser.check_exists(
- 'local', 'PROJECT_PATH')
-
- if project_type != 'none':
- result = result and parser.check_exists(
- 'project_files', 'FILE_PROJECT_CONF')
- else:
- result = True
-
- if not result:
- # Log.critical("{0} is not a valid config file".format(
- # os.path.basename(self._exp_parser_file)))
- raise Exception("Permission denied for " +
- str(os.path.basename(self._exp_parser_file)))
- else:
- Log.debug('{0} OK'.format(
- os.path.basename(self._exp_parser_file)))
- return result
+ return self._configWrapper.check_expdef_conf()
def check_proj(self):
"""
@@ -658,50 +368,17 @@ class AutosubmitConfig(object):
:return: True if everything is correct, False if it founds any error
:rtype: bool
"""
- try:
- if self._proj_parser_file == '':
- self._proj_parser = None
- else:
- self._proj_parser = AutosubmitConfig.get_parser(
- self.parser_factory, self._proj_parser_file)
- return True
- except Exception as e:
- Log.error('Project conf file error: {0}', e)
- return False
+ return self._configWrapper.check_proj()
def check_wrapper_conf(self):
- result = True
- result = result and self.is_valid_jobs_in_wrapper()
- if not result:
- Log.error(
- "There are sections in JOBS_IN_WRAPPER that are not defined in your jobs.conf file")
-
- if 'horizontal' in self.get_wrapper_type():
- result = result and self._platforms_parser.check_exists(
- self.get_platform(), 'PROCESSORS_PER_NODE')
- result = result and self._platforms_parser.check_exists(
- self.get_platform(), 'MAX_PROCESSORS')
- if 'vertical' in self.get_wrapper_type():
- result = result and self._platforms_parser.check_exists(
- self.get_platform(), 'MAX_WALLCLOCK')
- return result
+ return
def reload(self):
"""
Creates parser objects for configuration files
"""
- if not os.path.exists(self._conf_parser_file): raise IOError("Required file not found {0}".format(self._conf_parser_file))
- if not os.path.exists(self._platforms_parser_file): raise IOError("Required file not found {0}".format(self._platforms_parser_file))
- if not os.path.exists(self._jobs_parser_file): raise IOError("Required file not found {0}".format(self._jobs_parser_file))
- if not os.path.exists(self._exp_parser_file): raise IOError("Required file not found {0}".format(self._exp_parser_file))
- self._conf_parser = AutosubmitConfig.get_parser(self.parser_factory, self._conf_parser_file)
- self._platforms_parser = AutosubmitConfig.get_parser(self.parser_factory, self._platforms_parser_file)
- self._jobs_parser = AutosubmitConfig.get_parser(self.parser_factory, self._jobs_parser_file)
- self._exp_parser = AutosubmitConfig.get_parser(self.parser_factory, self._exp_parser_file)
- if self._proj_parser_file == '':
- self._proj_parser = None
- else:
- self._proj_parser = AutosubmitConfig.get_parser(self.parser_factory, self._proj_parser_file)
+ self._configWrapper.reload()
+
def load_parameters(self):
"""
@@ -711,23 +388,7 @@ class AutosubmitConfig(object):
:return: a dictionary containing tuples [parameter_name, parameter_value]
:rtype: dict
"""
- parameters = dict()
- for section in self._exp_parser.sections():
- for option in self._exp_parser.options(section):
- parameters[option] = self._exp_parser.get(section, option)
- for section in self._conf_parser.sections():
- for option in self._conf_parser.options(section):
- parameters[option] = self._conf_parser.get(section, option)
-
- # project_type = self.get_project_type()
- # if project_type != "none" and self._proj_parser is not None:
- # # Load project parameters
- # Log.debug("Loading project parameters...")
- # parameters2 = parameters.copy()
- # parameters2.update(self.load_project_parameters())
- # parameters = parameters2
-
- return parameters
+ return self._configWrapper.load_parameters()
def load_project_parameters(self):
"""
@@ -736,17 +397,7 @@ class AutosubmitConfig(object):
:return: dictionary containing tuples [parameter_name, parameter_value]
:rtype: dict
"""
- pass
- # projdef = []
- # for section in self._proj_parser.sections():
- # print("SEction" + section)
- # projdef += self._proj_parser.items(section)
-
- # parameters = dict()
- # for item in projdef:
- # parameters[item[0]] = item[1]
-
- # return parameters
+ return self._configWrapper.load_project_parameters()
def set_expid(self, exp_id):
"""
@@ -755,18 +406,7 @@ class AutosubmitConfig(object):
:param exp_id: experiment identifier to store
:type exp_id: str
"""
- # Experiment conf
- content = open(self._exp_parser_file).read()
- if re.search('EXPID =.*', content):
- content = content.replace(
- re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id)
- open(self._exp_parser_file, 'w').write(content)
-
- content = open(self._conf_parser_file).read()
- if re.search('EXPID =.*', content):
- content = content.replace(
- re.search('EXPID =.*', content).group(0), "EXPID = " + exp_id)
- open(self._conf_parser_file, 'w').write(content)
+ self._configWrapper.set_expid(exp_id)
def get_project_type(self):
"""
@@ -775,7 +415,7 @@ class AutosubmitConfig(object):
:return: project type
:rtype: str
"""
- return self._exp_parser.get_option('project', 'PROJECT_TYPE', "NA").lower()
+ return self._configWrapper.get_project_type()
def get_file_project_conf(self):
"""
@@ -784,7 +424,7 @@ class AutosubmitConfig(object):
:return: path to project config file
:rtype: str
"""
- return self._exp_parser.get('project_files', 'FILE_PROJECT_CONF')
+ return self._configWrapper.get_file_project_conf()
def get_file_jobs_conf(self):
"""
@@ -793,7 +433,8 @@ class AutosubmitConfig(object):
:return: path to project config file
:rtype: str
"""
- return self._exp_parser.get_option('project_files', 'FILE_JOBS_CONF', '')
+ return self._configWrapper.get_file_jobs_conf()
+
def get_git_project_origin(self):
"""
@@ -802,7 +443,7 @@ class AutosubmitConfig(object):
:return: git origin
:rtype: str
"""
- return self._exp_parser.get_option('git', 'PROJECT_ORIGIN', '')
+ return self._configWrapper.get_git_project_origin()
def get_git_project_branch(self):
"""
@@ -811,7 +452,7 @@ class AutosubmitConfig(object):
:return: git branch
:rtype: str
"""
- return self._exp_parser.get_option('git', 'PROJECT_BRANCH', 'master')
+ return self._configWrapper.get_git_project_branch()
def get_git_project_commit(self):
"""
@@ -820,7 +461,7 @@ class AutosubmitConfig(object):
:return: git commit
:rtype: str
"""
- return self._exp_parser.get_option('git', 'PROJECT_COMMIT', None)
+ return self._configWrapper.get_git_project_commit()
def get_submodules_list(self):
"""
@@ -829,7 +470,7 @@ class AutosubmitConfig(object):
:return: submodules to load
:rtype: list
"""
- return ' '.join(self._exp_parser.get_option('git', 'PROJECT_SUBMODULES', '').split()).split()
+ return self._configWrapper.get_submodules_list()
def get_project_destination(self):
"""
@@ -838,16 +479,7 @@ class AutosubmitConfig(object):
:return: git commit
:rtype: str
"""
- value = self._exp_parser.get('project', 'PROJECT_DESTINATION')
- if not value:
- if self.get_project_type().lower() == "local":
- value = os.path.split(self.get_local_project_path())[1]
- elif self.get_project_type().lower() == "svn":
- value = self.get_svn_project_url().split('/')[-1]
- elif self.get_project_type().lower() == "git":
- value = self.get_git_project_origin().split(
- '/')[-1].split('.')[-2]
- return value
+ return self._configWrapper.get_project_destination()
def set_git_project_commit(self, as_conf):
"""
@@ -855,37 +487,7 @@ class AutosubmitConfig(object):
:param as_conf: Configuration class for exteriment
:type as_conf: AutosubmitConfig
"""
- full_project_path = as_conf.get_project_dir()
- try:
- output = subprocess.check_output("cd {0}; git rev-parse --abbrev-ref HEAD".format(full_project_path),
- shell=True)
- except subprocess.CalledProcessError:
- Log.critical("Failed to retrieve project branch...")
- return False
-
- project_branch = output
- Log.debug("Project branch is: " + project_branch)
- try:
- output = subprocess.check_output(
- "cd {0}; git rev-parse HEAD".format(full_project_path), shell=True)
- except subprocess.CalledProcessError:
- Log.critical("Failed to retrieve project commit SHA...")
- return False
- project_sha = output
- Log.debug("Project commit SHA is: " + project_sha)
-
- # register changes
- content = open(self._exp_parser_file).read()
- if re.search('PROJECT_BRANCH =.*', content):
- content = content.replace(re.search('PROJECT_BRANCH =.*', content).group(0),
- "PROJECT_BRANCH = " + project_branch)
- if re.search('PROJECT_COMMIT =.*', content):
- content = content.replace(re.search('PROJECT_COMMIT =.*', content).group(0),
- "PROJECT_COMMIT = " + project_sha)
- open(self._exp_parser_file, 'w').write(content)
- Log.debug(
- "Project commit SHA succesfully registered to the configuration file.")
- return True
+ return self._configWrapper.get_git_project_commit(as_conf)
def get_svn_project_url(self):
"""
@@ -894,7 +496,7 @@ class AutosubmitConfig(object):
:return: subversion project url
:rtype: str
"""
- return self._exp_parser.get_option('svn', 'PROJECT_URL', 'NA')
+ return self._configWrapper.get_svn_project_url()
def get_svn_project_revision(self):
"""
@@ -903,7 +505,7 @@ class AutosubmitConfig(object):
:return: revision for subversion project
:rtype: str
"""
- return self._exp_parser.get('svn', 'PROJECT_REVISION')
+ return self._configWrapper.get_svn_project_revision()
def get_local_project_path(self):
"""
@@ -912,7 +514,7 @@ class AutosubmitConfig(object):
:return: path to local project
:rtype: str
"""
- return self._exp_parser.get('local', 'PROJECT_PATH')
+ return self._configWrapper.get_local_project_path()
def get_date_list(self):
"""
@@ -921,30 +523,7 @@ class AutosubmitConfig(object):
:return: experiment's startdates
:rtype: list
"""
- date_list = list()
- string = self._exp_parser.get('experiment', 'DATELIST')
- if not string.startswith("["):
- string = '[{0}]'.format(string)
- split_string = nestedExpr('[', ']').parseString(string).asList()
- string_date = None
- for split in split_string[0]:
- if type(split) is list:
- for split_in in split:
- if split_in.find("-") != -1:
- numbers = split_in.split("-")
- for count in range(int(numbers[0]), int(numbers[1]) + 1):
- date_list.append(parse_date(
- string_date + str(count).zfill(len(numbers[0]))))
- else:
- date_list.append(parse_date(string_date + split_in))
- string_date = None
- else:
- if string_date is not None:
- date_list.append(parse_date(string_date))
- string_date = split
- if string_date is not None:
- date_list.append(parse_date(string_date))
- return date_list
+ return self._configWrapper.get_date_list()
def get_num_chunks(self):
"""
@@ -953,7 +532,7 @@ class AutosubmitConfig(object):
:return: number of chunks
:rtype: int
"""
- return int(self._exp_parser.get('experiment', 'NUMCHUNKS'))
+ return self._configWrapper.get_num_chunks()
def get_chunk_ini(self, default=1):
"""
@@ -963,11 +542,7 @@ class AutosubmitConfig(object):
:return: initial chunk
:rtype: int
"""
- chunk_ini = self._exp_parser.get_option(
- 'experiment', 'CHUNKINI', default)
- if chunk_ini == '':
- return default
- return int(chunk_ini)
+ return self._configWrapper.get_chunk_ini(default)
def get_chunk_size_unit(self):
# type: () -> str
@@ -978,12 +553,7 @@ class AutosubmitConfig(object):
:rtype: str
"""
- # try:
- # res = self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower()
- # except Exception as e:
- # try:
- # res = self.autosubmit_conf.get('experiment')
- return self._exp_parser.get('experiment', 'CHUNKSIZEUNIT').lower()
+ return self._configWrapper.get_chunk_size_unit()
def get_chunk_size(self, default=1):
# type: (int) -> int
@@ -993,16 +563,7 @@ class AutosubmitConfig(object):
:return: Chunksize, 1 as default.
:rtype: int
"""
- try:
- chunk_size = self._exp_parser.get_option(
- 'experiment', 'CHUNKSIZE', default)
- except Exception as exp:
- print(exp)
- chunk_size = ''
- pass
- if chunk_size == '':
- return default
- return int(chunk_size)
+ return self._configWrapper.get_chunk_size(default)
def get_member_list(self, run_only=False):
"""
@@ -1011,31 +572,7 @@ class AutosubmitConfig(object):
:return: experiment's members
:rtype: list
"""
- member_list = list()
- string = self._exp_parser.get('experiment', 'MEMBERS') if run_only == False else self._exp_parser.get_option(
- 'experiment', 'RUN_ONLY_MEMBERS', '')
- if not string.startswith("["):
- string = '[{0}]'.format(string)
- split_string = nestedExpr('[', ']').parseString(string).asList()
- string_member = None
- for split in split_string[0]:
- if type(split) is list:
- for split_in in split:
- if split_in.find("-") != -1:
- numbers = split_in.split("-")
- for count in range(int(numbers[0]), int(numbers[1]) + 1):
- member_list.append(
- string_member + str(count).zfill(len(numbers[0])))
- else:
- member_list.append(string_member + split_in)
- string_member = None
- else:
- if string_member is not None:
- member_list.append(string_member)
- string_member = split
- if string_member is not None:
- member_list.append(string_member)
- return member_list
+ return self._configWrapper.get_member_list(run_only)
def get_rerun(self):
"""
@@ -1045,7 +582,7 @@ class AutosubmitConfig(object):
:rtype: list
"""
- return self._exp_parser.get('rerun', 'RERUN').lower()
+ return self._configWrapper.get_rerun()
def get_chunk_list(self):
"""
@@ -1054,7 +591,7 @@ class AutosubmitConfig(object):
:return: experiment's chunks
:rtype: list
"""
- return self._exp_parser.get('rerun', 'CHUNKLIST')
+ return self._configWrapper.get_chunk_list()
def get_platform(self):
"""
@@ -1063,7 +600,7 @@ class AutosubmitConfig(object):
:return: main platforms
:rtype: str
"""
- return self._exp_parser.get('experiment', 'HPCARCH')
+ return self._configWrapper.get_platform()
def set_platform(self, hpc):
"""
@@ -1072,11 +609,7 @@ class AutosubmitConfig(object):
:param hpc: main platforms
:type: str
"""
- content = open(self._exp_parser_file).read()
- if re.search('HPCARCH =.*', content):
- content = content.replace(
- re.search('HPCARCH =.*', content).group(0), "HPCARCH = " + hpc)
- open(self._exp_parser_file, 'w').write(content)
+ self._configWrapper.set_platform(hpc)
def set_version(self, autosubmit_version):
"""
@@ -1085,11 +618,7 @@ class AutosubmitConfig(object):
:param autosubmit_version: autosubmit's version
:type autosubmit_version: str
"""
- content = open(self._conf_parser_file).read()
- if re.search('AUTOSUBMIT_VERSION =.*', content):
- content = content.replace(re.search('AUTOSUBMIT_VERSION =.*', content).group(0),
- "AUTOSUBMIT_VERSION = " + autosubmit_version)
- open(self._conf_parser_file, 'w').write(content)
+ return self._configWrapper.set_version(autosubmit_version)
def get_version(self):
"""
@@ -1098,7 +627,7 @@ class AutosubmitConfig(object):
:return: version
:rtype: str
"""
- return self._conf_parser.get_option('config', 'AUTOSUBMIT_VERSION', 'None')
+ return self._configWrapper.get_version()
def get_total_jobs(self):
"""
@@ -1107,7 +636,7 @@ class AutosubmitConfig(object):
:return: max number of running jobs
:rtype: int
"""
- return int(self._conf_parser.get('config', 'TOTALJOBS'))
+ return self._configWrapper.get_total_jobs()
def get_max_wallclock(self):
"""
@@ -1115,7 +644,7 @@ class AutosubmitConfig(object):
:rtype: str
"""
- return self._conf_parser.get_option('config', 'MAX_WALLCLOCK', '')
+ return self._configWrapper.get_max_wallclock()
def get_max_processors(self):
"""
@@ -1123,9 +652,7 @@ class AutosubmitConfig(object):
:rtype: str
"""
- config_value = self._conf_parser.get_option(
- 'config', 'MAX_PROCESSORS', None)
- return int(config_value) if config_value is not None else config_value
+ return self._configWrapper.get_max_processors()
def get_max_waiting_jobs(self):
"""
@@ -1134,7 +661,7 @@ class AutosubmitConfig(object):
:return: main platforms
:rtype: int
"""
- return int(self._conf_parser.get_option('config', 'MAXWAITINGJOBS', 10))
+ return self._configWrapper.get_max_waiting_jobs()
def get_default_job_type(self):
"""
@@ -1143,7 +670,7 @@ class AutosubmitConfig(object):
:return: default type such as bash, python, r..
:rtype: str
"""
- return self._exp_parser.get_option('project_files', 'JOB_SCRIPTS_TYPE', 'bash')
+ return self._configWrapper.get_default_job_type()
def get_safetysleeptime(self):
"""
@@ -1152,7 +679,7 @@ class AutosubmitConfig(object):
:return: safety sleep time
:rtype: int
"""
- return int(self._conf_parser.get_option('config', 'SAFETYSLEEPTIME', 10))
+ return self._configWrapper.get_safetysleeptime()
def set_safetysleeptime(self, sleep_time):
"""
@@ -1161,10 +688,7 @@ class AutosubmitConfig(object):
:param sleep_time: value to set
:type sleep_time: int
"""
- content = open(self._conf_parser_file).read()
- content = content.replace(re.search('SAFETYSLEEPTIME =.*', content).group(0),
- "SAFETYSLEEPTIME = %d" % sleep_time)
- open(self._conf_parser_file, 'w').write(content)
+ self._configWrapper.set_safetysleeptime(sleep_time)
def get_retrials(self):
"""
@@ -1173,7 +697,7 @@ class AutosubmitConfig(object):
:return: safety sleep time
:rtype: int
"""
- return int(self._conf_parser.get('config', 'RETRIALS'))
+ return self._configWrapper.get_retrials()
def get_notifications(self):
"""
@@ -1182,7 +706,7 @@ class AutosubmitConfig(object):
:return: if notifications
:rtype: string
"""
- return self._conf_parser.get_option('mail', 'NOTIFICATIONS', 'false').lower()
+ return self._configWrapper.get_notifications()
def get_remote_dependencies(self):
"""
@@ -1191,7 +715,7 @@ class AutosubmitConfig(object):
:return: if remote dependencies
:rtype: bool
"""
- return self._conf_parser.get_option('wrapper', 'DEPENDENCIES', 'false').lower() == 'true'
+ return self._configWrapper.get_remote_dependencies()
def get_wrapper_type(self):
"""
@@ -1200,7 +724,7 @@ class AutosubmitConfig(object):
:return: wrapper type (or none)
:rtype: string
"""
- return self._conf_parser.get_option('wrapper', 'TYPE', 'None').lower()
+ return self._configWrapper.get_wrapper_type()
def get_wrapper_jobs(self):
"""
@@ -1209,7 +733,7 @@ class AutosubmitConfig(object):
:return: expression (or none)
:rtype: string
"""
- return self._conf_parser.get_option('wrapper', 'JOBS_IN_WRAPPER', 'None')
+ return self._configWrapper.get_wrapper_jobs()
def get_max_wrapped_jobs(self):
"""
@@ -1218,9 +742,7 @@ class AutosubmitConfig(object):
:return: maximum number of jobs (or total jobs)
:rtype: string
"""
- # return int(self._conf_parser.get_option('wrapper', 'MAXWRAPPEDJOBS', self.get_total_jobs()))
-
- return int(self._conf_parser.get_option('wrapper', 'MAX_WRAPPED', self.get_total_jobs()))
+ return self._configWrapper.get_max_wrapped_jobs()
def get_wrapper_check_time(self):
"""
@@ -1229,7 +751,7 @@ class AutosubmitConfig(object):
:return: wrapper check time
:rtype: int
"""
- return int(self._conf_parser.get_option('wrapper', 'CHECK_TIME_WRAPPER', self.get_safetysleeptime()))
+ return self._configWrapper.get_wrapper_check_time()
def get_wrapper_machinefiles(self):
"""
@@ -1238,7 +760,7 @@ class AutosubmitConfig(object):
:return: machinefiles function to use
:rtype: string
"""
- return self._conf_parser.get_option('wrapper', 'MACHINEFILES', '')
+ return self._configWrapper.get_wrapper_machinefiles()
def get_wrapper_queue(self):
"""
@@ -1247,7 +769,7 @@ class AutosubmitConfig(object):
:return: expression (or none)
:rtype: string
"""
- return self._conf_parser.get_option('wrapper', 'QUEUE', "")
+ return self._configWrapper.get_wrapper_queue()
def get_jobs_sections(self):
"""
@@ -1256,7 +778,7 @@ class AutosubmitConfig(object):
:return: sections
:rtype: list
"""
- return self._jobs_parser.sections()
+ return self._configWrapper.get_jobs_sections()
def get_copy_remote_logs(self):
"""
@@ -1265,7 +787,7 @@ class AutosubmitConfig(object):
:return: if logs local copy
:rtype: bool
"""
- return self._conf_parser.get_option('storage', 'COPY_REMOTE_LOGS', 'true').lower()
+ return self._configWrapper.get_copy_remote_logs()
def get_mails_to(self):
"""
@@ -1274,7 +796,7 @@ class AutosubmitConfig(object):
:return: mail address
:rtype: [str]
"""
- return [str(x) for x in self._conf_parser.get_option('mail', 'TO', '').split(' ')]
+ return self._configWrapper.get_mails_to()
def get_communications_library(self):
"""
@@ -1283,7 +805,7 @@ class AutosubmitConfig(object):
:return: communications library
:rtype: str
"""
- return self._conf_parser.get_option('communications', 'API', 'paramiko').lower()
+ return self._configWrapper.get_communications_library()
def get_storage_type(self):
"""
@@ -1292,45 +814,26 @@ class AutosubmitConfig(object):
:return: communications library
:rtype: str
"""
- return self._conf_parser.get_option('storage', 'TYPE', 'pkl').lower()
+ return self._configWrapper.get_storage_type()
- @staticmethod
- def is_valid_mail_address(mail_address):
- if re.match('^[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+(\.[a-z0-9-]+)*(\.[a-z]{2,4})$', mail_address):
- return True
- else:
- return False
+ def is_valid_mail_address(self, mail_address):
+ #TODO: push to parent class as static method
+ return self._configWrapper.is_valid_mail_address(mail_address)
def is_valid_communications_library(self):
- library = self.get_communications_library()
- return library in ['paramiko', 'saga']
+ return self._configWrapper.is_valid_communications_library()
def is_valid_storage_type(self):
- storage_type = self.get_storage_type()
- return storage_type in ['pkl', 'db']
+ return self._configWrapper.is_valid_storage_type()
def is_valid_jobs_in_wrapper(self):
- expression = self.get_wrapper_jobs()
- if expression != 'None':
- parser = self._jobs_parser
- sections = parser.sections()
- for section in expression.split(" "):
- if "&" in section:
- for inner_section in section.split("&"):
- if inner_section not in sections:
- return False
- elif section not in sections:
- return False
- return True
+ return self._configWrapper.is_valid_jobs_in_wrapper()
def is_valid_git_repository(self):
- origin_exists = self._exp_parser.check_exists('git', 'PROJECT_ORIGIN')
- branch = self.get_git_project_branch()
- commit = self.get_git_project_commit()
- return origin_exists and (branch is not None or commit is not None)
+ return self._configWrapper.is_valid_git_repository()
+
- @staticmethod
- def get_parser(parser_factory, file_path):
+ def get_parser(self, parser_factory, file_path):
# type: (ConfigParserFactory, str) -> ConfigParser
"""
Gets parser for given file
@@ -1341,13 +844,6 @@ class AutosubmitConfig(object):
:return: parser
:rtype: SafeConfigParser
"""
- parser = parser_factory.create_parser()
- parser.optionxform = str
- # proj is not required
- # print(file_path)
- if file_path.find('proj_') > 0:
- parser.read(file_path)
- else:
- with open(file_path) as f:
- parser.read(file_path)
- return parser
+ # TODO: this was static method, check usages
+
+ return self._configWrapper.get_parser(parser_factory, file_path)
\ No newline at end of file
diff --git a/autosubmit_api/config/ymlConfigStrategy.py b/autosubmit_api/config/ymlConfigStrategy.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab42d38035ee6fb47cb2a1c3fcc74c500a19cdee
--- /dev/null
+++ b/autosubmit_api/config/ymlConfigStrategy.py
@@ -0,0 +1,424 @@
+#!/usr/bin/env python
+
+# Copyright 2015 Earth Sciences Department, BSC-CNS
+
+# This file is part of Autosubmit.
+
+# Autosubmit is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# Autosubmit is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with Autosubmit. If not, see .
+
+try:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+ from autosubmitconfigparser.config.configcommon import AutosubmitConfig as Autosubmit4Config
+except ImportError:
+ # noinspection PyCompatibility
+ from configparser import SafeConfigParser
+
+import os
+import re
+import subprocess
+import json
+import logging
+
+from pyparsing import nestedExpr
+from bscearth.utils.config_parser import ConfigParserFactory, ConfigParser
+from bscearth.utils.date import parse_date
+from bscearth.utils.log import Log
+from ..config.basicConfig import BasicConfig
+from ..config.IConfigStrategy import IConfigStrategy
+
+logger = logging.getLogger('gunicorn.error')
+
+class ymlConfigStrategy(IConfigStrategy):
+ """
+ Class to handle experiment configuration coming from file or database
+
+ :param expid: experiment identifier
+ :type expid: str
+ """
+ def __init__(self, expid, basic_config, parser_factory, extension=".yml"):
+ logger.info("Creating AS4 Parser !!!!!")
+ self._conf_parser = Autosubmit4Config(expid)
+ self._conf_parser.reload(True)
+
+ def jobs_parser(self):
+ logger.info("Not yet implemented")
+ pass
+
+ #TODO: at the end of the implementation, check which methods can be moved to the top class for avoid code duplication
+ @property
+ def experiment_file(self):
+ """
+ Returns experiment's config file name
+ """
+ return self._exp_parser_file
+
+ def platforms_parser(self):
+ logger.info("OBSOLOTED - Not yet implemented")
+ pass
+
+ @property
+ def platforms_file(self):
+ """
+ Returns experiment's platforms config file name
+
+ :return: platforms config file's name
+ :rtype: str
+ """
+ return self._platforms_parser_file
+
+ @property
+ def project_file(self):
+ """
+ Returns project's config file name
+ """
+ return self._proj_parser_file
+
+ @property
+ def jobs_file(self):
+ """
+ Returns project's jobs file name
+ """
+ return self._jobs_parser_file
+
+ def get_full_config_as_dict(self):
+ """
+ Returns full configuration as json object
+ """
+ _conf = _exp = _platforms = _jobs = _proj = None
+ result = {}
+
+ def get_data( parser):
+ """
+ dictionary comprehension to get data from parser
+ """
+ logger.info(parser)
+ #res = {sec: {option: parser[sec][option] for option in parser[sec].keys()} for sec in [
+ # section for section in parser.keys()]}
+ #return res
+ return parser
+
+ # res = {sec: {option: parser.get(sec, option) for option in parser.options(sec)} for sec in [
+ # section for section in parser.sections()]}
+
+
+ # print(self._conf_parser)
+ #result["conf"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._conf_parser else None
+ #result["exp"] = get_data( self._conf_parser.experiment_data["CONF"]) if self._exp_parser else None
+ result["platforms"] = self._conf_parser.platforms_data if self._conf_parser.platforms_data else None
+ #result["jobs"] = get_data( self._conf_parser.experiment_data["JOBS"]) if self._conf_parser.experiment_data["JOBS"] else None
+ #result["proj"] = get_data( self._conf_parser.experiment_data["CONF"] ) if self._proj_parser else None
+ return result
+
+
+ def get_full_config_as_json(self):
+ return self._conf_parser.get_full_config_as_json()
+
+ def get_project_dir(self):
+ return self._conf_parser.get_project_dir()
+
+ def get_queue(self, section):
+ return self._conf_parser.jobs_data[section].get('QUEUE', "")
+
+ def get_job_platform(self, section):
+ pass
+
+ def get_platform_queue(self, platform):
+ logger.info("get_platform_queue")
+ return self._conf_parser.platforms_data[platform]["QUEUE"]
+
+ def get_platform_serial_queue(self, platform):
+ logger.info("get_platform_serial_queue")
+ return self._conf_parser.platforms_data[platform]["SERIAL_QUEUE"]
+
+ def get_platform_project(self, platform):
+ logger.info("get_platform_project")
+ return self._conf_parser.platforms_data[platform]["PROJECT"]
+
+ def get_platform_wallclock(self, platform):
+ logger.info("get_platform_wallclock")
+ return self._conf_parser.platforms_data[platform].get('MAX_WALLCLOCK', "")
+
+ def get_wallclock(self, section):
+ return self._conf_parser.jobs_data[section].get('WALLCLOCK', '')
+
+
+ def get_synchronize(self, section):
+ return self._conf_parser.get_synchronize(section)
+
+ def get_processors(self, section):
+ return self._conf_parser.get_processors(section)
+
+ def get_threads(self, section):
+ return self._conf_parser.get_threads(section)
+
+ def get_tasks(self, section):
+ return self._conf_parser.get_tasks(section)
+
+ def get_scratch_free_space(self, section):
+ return self._conf_parser.get_scratch_free_space(section)
+
+ def get_memory(self, section):
+ return self._conf_parser.get_memory(section)
+
+ def get_memory_per_task(self, section):
+ return self._conf_parser.get_memory_per_task(section)
+
+ def get_migrate_user_to(self, section):
+ """
+ Returns the user to change to from platform config file.
+ :return: migrate user to
+ :rtype: str
+ """
+ return self._conf_parser.get_migrate_user_to(section)
+
+ def get_current_user(self, section):
+ return self._conf_parser.get_current_user(section)
+
+ def get_current_project(self, section):
+ return self._conf_parser.get_current_project(section)
+
+ def set_new_user(self, section, new_user):
+ self._conf_parser.set_new_user(section, new_user)
+
+ def get_migrate_project_to(self, section):
+ return self._conf_parser.get_migrate_project_to(section)
+
+ def set_new_project(self, section, new_project):
+ self._conf_parser.set_new_project(section, new_project)
+
+ def get_custom_directives(self, section):
+ """
+ Gets custom directives needed for the given job type
+ :param section: job type
+ :type section: str
+ :return: custom directives needed
+ :rtype: str
+ """
+ return str(self._conf_parser.jobs_data.get(section, {}).get('CUSTOM_DIRECTIVES', ""))
+
+
+ def check_conf_files(self):
+ return self._conf_parser.check_conf_files()
+
+ def check_autosubmit_conf(self):
+ return self._conf_parser.check_autosubmit_conf()
+
+ def check_platforms_conf(self):
+ return self._conf_parser.check_platforms_conf()
+
+ def check_jobs_conf(self):
+ return self._conf_parser.check_jobs_conf()
+
+ def check_expdef_conf(self):
+ return self._conf_parser.check_expdef_conf()
+
+ def check_proj(self):
+ return self._conf_parser.check_proj()
+
+ def check_wrapper_conf(self):
+ self._conf_parser.check_wrapper_conf()
+
+ def reload(self):
+ self._conf_parser.reload()
+
+ def load_parameters(self):
+ return self._conf_parser.load_parameters()
+
+ def load_project_parameters(self):
+ return self._conf_parser.load_project_parameters()
+
+ def set_expid(self, exp_id):
+ self._conf_parser.set_expid(exp_id)
+
+ def get_project_type(self):
+ return self._conf_parser.get_project_type()
+
+ def get_file_project_conf(self):
+ return self._conf_parser.get_file_project_conf()
+
+ def get_file_jobs_conf(self):
+ return self._conf_parser.get_file_jobs_conf()
+
+ def get_git_project_origin(self):
+ return self._conf_parser.get_git_project_origin()
+
+ def get_git_project_branch(self):
+ return self._conf_parser.get_git_project_branch()
+
+ def get_git_project_commit(self):
+ return self._conf_parser.get_git_project_commit()
+
+ def get_submodules_list(self):
+ return self._conf_parser.get_submodules_list()
+
+ def get_project_destination(self):
+ return self._conf_parser.get_project_destination()
+
+ def set_git_project_commit(self, as_conf):
+ self._conf_parser.set_git_project_commit(as_conf)
+
+ def get_svn_project_url(self):
+ return self._conf_parser.get_svn_project_url()
+
+ def get_svn_project_revision(self):
+ return self._conf_parser.get_svn_project_revision()
+
+ def get_local_project_path(self):
+ return self._conf_parser.get_local_project_path()
+
+ def get_date_list(self):
+ return self._conf_parser.get_date_list()
+
+ def get_num_chunks(self):
+ return self._conf_parser.get_num_chunks()
+
+ def get_chunk_ini(self, default=1):
+ return self._conf_parser.get_chunk_ini(default)
+
+ def get_chunk_size_unit(self):
+ """
+ Unit for the chunk length
+
+ :return: Unit for the chunk length Options: {hour, day, month, year}
+ :rtype: str
+ """
+ return self._conf_parser.get_chunk_size_unit()
+
+
+ def get_chunk_size(self, default=1):
+ try:
+ chunk_size = self._conf_parser.get_chunk_size(default)
+ except Exception as exp:
+ print(exp)
+ chunk_size = ''
+ pass
+ if chunk_size == '':
+ return default
+ return int(chunk_size)
+
+ def get_member_list(self, run_only=False):
+ #return self._conf_parser.get_member_list(run_only)
+ """
+ Returns members list from experiment's config file
+
+ :return: experiment's members
+ :rtype: list
+ """
+ return self._conf_parser.get_member_list(run_only)
+
+ def get_rerun(self):
+ return self._conf_parser.get_rerun()
+
+ def get_chunk_list(self):
+ return self._conf_parser.get_chunk_list()
+
+ def get_platform(self):
+ return self._conf_parser.get_platform()
+
+ def set_platform(self, hpc):
+ self._conf_parser.set_platform(hpc)
+
+ def set_version(self, autosubmit_version):
+ self._conf_parser.set_version(autosubmit_version)
+
+ def get_version(self):
+ return self._conf_parser.get_version()
+
+ def get_total_jobs(self):
+ return self._conf_parser.get_total_jobs()
+
+ def get_max_wallclock(self):
+ return self._conf_parser.get_max_wallclock()
+
+ def get_max_processors(self):
+ return self._conf_parser.get_max_processors()
+
+ def get_max_waiting_jobs(self):
+ return self._conf_parser.get_max_waiting_jobs()
+
+ def get_default_job_type(self):
+ return self._conf_parser.get_default_job_type()
+
+ def get_safetysleeptime(self):
+ return self._conf_parser.get_safetysleeptime()
+
+ def set_safetysleeptime(self, sleep_time):
+ self._conf_parser.set_safetysleeptime(sleep_time)
+
+ def get_retrials(self):
+ return self._conf_parser.get_retrials()
+
+ def get_notifications(self):
+ return self._conf_parser.get_notifications()
+
+ def get_remote_dependencies(self):
+ return self._conf_parser.get_remote_dependencies()
+
+ def get_wrapper_type(self):
+ return self._conf_parser.get_wrapper_type()
+
+ def get_wrapper_jobs(self):
+ return self._conf_parser.get_wrapper_jobs()
+
+ def get_max_wrapped_jobs(self):
+ return self._conf_parser.get_max_wrapped_jobs()
+
+ def get_wrapper_check_time(self):
+ return self._conf_parser.get_wrapper_check_time()
+
+ def get_wrapper_machinefiles(self):
+ return self._conf_parser.get_wrapper_machinefiles()
+
+ def get_wrapper_queue(self):
+ return self._conf_parser.get_wrapper_queue()
+
+ def get_jobs_sections(self):
+ return self._conf_parser.get_jobs_sections()
+
+ def get_copy_remote_logs(self):
+ return self._conf_parser.get_copy_remote_logs()
+
+ def get_mails_to(self):
+ return self._conf_parser.get_mails_to()
+
+ def get_communications_library(self):
+ return self._conf_parser.get_communications_library()
+
+ def get_storage_type(self):
+ return self._conf_parser.get_storage_type()
+
+ @staticmethod
+ def is_valid_mail_address(mail_address):
+ return self._conf_parser.is_valid_mail_address(mail_address)
+
+ @classmethod
+ def is_valid_communications_library(self):
+ return self._conf_parser.is_valid_communications_library()
+
+ @classmethod
+ def is_valid_storage_type(self):
+ return self._conf_parser.is_valid_storage_type()
+
+ def is_valid_jobs_in_wrapper(self):
+ self._conf_parser.is_valid_jobs_in_wrapper()
+
+ def is_valid_git_repository(self):
+ self._conf_parser.is_valid_git_repository()
+
+ @staticmethod
+ def get_parser(parser_factory, file_path):
+ return Autosubmit4Config.get_parser(parser_factory, file_path)
+
+
diff --git a/autosubmit_api/database/db_common.py b/autosubmit_api/database/db_common.py
index bde23d0eca9c3f62c0745b9dbdd202cb7209eea3..e58ef8cbd41f21e2629190b25a889e8bee91f288 100644
--- a/autosubmit_api/database/db_common.py
+++ b/autosubmit_api/database/db_common.py
@@ -365,8 +365,6 @@ def search_experiment_by_id(searchString, typeExp=None, onlyActive=None, owner=N
wrapper = autosubmit_config_facade.get_wrapper_type()
last_modified_pkl_datetime = autosubmit_config_facade.get_pkl_last_modified_time_as_datetime()
except Exception as exp:
- version = "Unknown"
- wrapper = None
last_modified_pkl_datetime = None
pass
status = experiment_status.get(expid, "NOT RUNNING")
@@ -454,8 +452,6 @@ def get_current_running_exp():
wrapper = autosubmit_config_facade.get_wrapper_type()
last_modified_pkl_datetime = autosubmit_config_facade.get_pkl_last_modified_time_as_datetime()
except Exception as exp:
- version = "Unknown"
- wrapper = None
last_modified_pkl_datetime = None
pass
if (expid in experiment_times):
diff --git a/autosubmit_api/experiment/common_requests.py b/autosubmit_api/experiment/common_requests.py
index dc5584ed310ad30740a8a119ffa3d64ad84d7b6d..b75d58aeda54e7e11270cbebc480008dca68b9b2 100644
--- a/autosubmit_api/experiment/common_requests.py
+++ b/autosubmit_api/experiment/common_requests.py
@@ -28,6 +28,8 @@ import datetime
import json
import multiprocessing
import subprocess
+import logging
+
from collections import deque
from ..autosubmit_legacy.autosubmit import Autosubmit
from ..database import db_common as db_common
@@ -1121,6 +1123,7 @@ def get_auto_conf_data(expid):
max_wrapped = as_conf.get_max_wrapped_jobs()
return (wrapper_type, max_wrapped)
except Exception as ex:
+ logger.info(traceback.format_exc())
print(("Couldn't retrieve conf data (wrapper info) from {0}. Exception {1}.".format(expid, str(ex))))
return ("None", 0)
@@ -1382,6 +1385,7 @@ def get_current_configuration_by_expid(expid, valid_user, log):
warning = True
warning_message = "The filesystem system configuration can't be retrieved because '{}'".format(
exp)
+ logger.info(traceback.format_exc())
currentFileSystemConfig["contains_nones"] = True
log.info(warning_message)
pass
diff --git a/deployment/README.md b/deployment/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..16206472223cff374eb87ed7c82e85bae5be354e
--- /dev/null
+++ b/deployment/README.md
@@ -0,0 +1,2 @@
+conda env create -f conda_environment.yml
+conda activate autosubmit_api
\ No newline at end of file
diff --git a/deployment/conda_environment.yaml b/deployment/conda_environment.yaml
index 61c5ed8a954088ed6904be924789cd353f0f8d07..cc9fedcb9593e92f5f4b5a61de0e3f061a61a334 100644
--- a/deployment/conda_environment.yaml
+++ b/deployment/conda_environment.yaml
@@ -75,3 +75,4 @@ dependencies:
- urllib3==1.26.14
- werkzeug==2.2.2
- zipp==3.12.0
+ - autosubmitconfigparser==1.0.27
diff --git a/deployment/update_launch_autosubmit_API_conda.sh b/deployment/update_launch_autosubmit_API_conda.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f36f8f6b0f655f5fdfe44e268ab04cd97570cc93
--- /dev/null
+++ b/deployment/update_launch_autosubmit_API_conda.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+. ~/.bashrc
+
+LOG_PATH=${PLOG}
+UPDATE=false
+
+while getopts ":e:u" opt; do
+ case "$opt" in
+ u) UPDATE=true;
+ ;;
+ esac
+done
+
+
+# Stop current instance of unicorn
+pstree -ap | grep gunicorn | awk -F',' '{print $2}' | awk -F' ' '{print $1}' | head -n 1 | xargs -r kill
+pstree -ap | grep gunicorn
+
+# activate conda environment
+conda activate autosubmit_api
+
+# if update to a new version we install it from pip
+if [ "${UPDATE}" = true ]; then
+ pip install autosubmit_api --upgrade
+fi
+
+# prepare to launch
+echo "Set SECRET KEY"
+export SECRET_KEY='c&X=