diff --git a/VERSION b/VERSION index e89c23ab4875f1ebc08d52c97c8714108e0338c9..b2cb6d09d5090c9421d1946b7a291eb9ab853fce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.0.87 +4.0.88 diff --git a/autosubmit/database/db_common.py b/autosubmit/database/db_common.py index 626cfa1e930ac0925862f4fff6829d7b7f15d576..bdc381b8907f6dce89c418c4a497ec4bed355e24 100644 --- a/autosubmit/database/db_common.py +++ b/autosubmit/database/db_common.py @@ -20,11 +20,12 @@ """ Module containing functions to manage autosubmit's database. """ +import multiprocessing import os import sqlite3 -import multiprocessing -import autosubmit -from log.log import Log, AutosubmitCritical, AutosubmitError + +from log.log import Log, AutosubmitCritical + Log.get_logger("Autosubmit") from autosubmitconfigparser.config.basicconfig import BasicConfig diff --git a/autosubmit/database/db_manager.py b/autosubmit/database/db_manager.py index 6e7376b9b1fbdd9c9b3081f42493c23f52ece210..51639f2b70353d8b63f2b613b78324c7f0333dd7 100644 --- a/autosubmit/database/db_manager.py +++ b/autosubmit/database/db_manager.py @@ -17,8 +17,8 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import sqlite3 import os +import sqlite3 class DbManager(object): diff --git a/autosubmit/database/db_structure.py b/autosubmit/database/db_structure.py index b42854359edc2dea0c71c0b682057fcee0d28e89..f6681b5e67149f9d9b25b154c327d9501af9b083 100644 --- a/autosubmit/database/db_structure.py +++ b/autosubmit/database/db_structure.py @@ -18,13 +18,14 @@ # along with Autosubmit. If not, see . import os - +import sqlite3 import textwrap import traceback -import sqlite3 - from typing import Dict, List + from log.log import Log + + # from networkx import DiGraph # DB_FILE_AS_TIMES = "/esarchive/autosubmit/as_times.db" diff --git a/autosubmit/experiment/experiment_common.py b/autosubmit/experiment/experiment_common.py index 17eba23fcee5061461b30406b53367d5a1032e84..2ee9c3929504ac22fc4ff14d98803bf1728094f8 100644 --- a/autosubmit/experiment/experiment_common.py +++ b/autosubmit/experiment/experiment_common.py @@ -21,8 +21,10 @@ Module containing functions to manage autosubmit's experiments. """ import string + import autosubmit.database.db_common as db_common -from log.log import Log,AutosubmitCritical +from log.log import Log, AutosubmitCritical + Log.get_logger("Autosubmit") def new_experiment(description, version, test=False, operational=False): diff --git a/autosubmit/experiment/statistics.py b/autosubmit/experiment/statistics.py index 793210923d0e28c3325cc1ad098e2dc73621d51c..1ca50fdbd09887e8bfa382cf99bb02ed11405fec 100644 --- a/autosubmit/experiment/statistics.py +++ b/autosubmit/experiment/statistics.py @@ -18,9 +18,10 @@ # along with Autosubmit. If not, see . import datetime + from autosubmit.job.job import Job from autosubmit.monitor.utils import FixedSizeList -from log.log import Log, AutosubmitError, AutosubmitCritical +from log.log import Log def timedelta2hours(deltatime): diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 976b02f9fb157bc5728e667b58efbbc388dc93aa..42e795e9b9d22b633ca05b83f5ecc2bf8b0f793b 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -14,18 +14,21 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +from os import path + # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import locale -from os import path import os -from shutil import rmtree -import subprocess import shutil +import subprocess +from shutil import rmtree +from time import time + # from autosubmit import Autosubmit from autosubmitconfigparser.config.basicconfig import BasicConfig -from time import time -from log.log import Log, AutosubmitCritical, AutosubmitError +from log.log import Log, AutosubmitCritical + Log.get_logger("Autosubmit") diff --git a/autosubmit/helpers/autosubmit_helper.py b/autosubmit/helpers/autosubmit_helper.py index 2aef35c49d8ee8a94312add09e5e931df8ebf34d..cf04f79bc825d3a300d0070833cd4ad85d1f0732 100644 --- a/autosubmit/helpers/autosubmit_helper.py +++ b/autosubmit/helpers/autosubmit_helper.py @@ -17,16 +17,18 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -from log.log import AutosubmitCritical, Log -from time import sleep -from autosubmitconfigparser.config.basicconfig import BasicConfig -from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from autosubmit.history.experiment_history import ExperimentHistory -from autosubmit.database.db_common import check_experiment_exists import datetime import sys +from time import sleep from typing import List +from autosubmit.database.db_common import check_experiment_exists +from autosubmit.history.experiment_history import ExperimentHistory +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from log.log import AutosubmitCritical, Log + + def handle_start_time(start_time): # type: (str) -> None """ Wait until the supplied time. """ diff --git a/autosubmit/helpers/parameters.py b/autosubmit/helpers/parameters.py index 97ddb6235b476d554e304e057e7316dfe5a7840b..e52d3d664a761a50ecc840fdfacdf6735c654948 100644 --- a/autosubmit/helpers/parameters.py +++ b/autosubmit/helpers/parameters.py @@ -1,6 +1,7 @@ +from collections import defaultdict + import functools import inspect -from collections import defaultdict from typing import Dict PARAMETERS = defaultdict(defaultdict) diff --git a/autosubmit/helpers/utils.py b/autosubmit/helpers/utils.py index fca94a126a7310ab6184ca25a0580ab12d1b520a..5436643c7dda222ccaac2ff829c633fdcf3b2eee 100644 --- a/autosubmit/helpers/utils.py +++ b/autosubmit/helpers/utils.py @@ -1,9 +1,8 @@ import os import pwd -from log.log import Log, AutosubmitCritical -from autosubmitconfigparser.config.basicconfig import BasicConfig -from typing import Tuple +from log.log import AutosubmitCritical + def check_experiment_ownership(expid, basic_config, raise_error=False, logger=None): # [A-Za-z09]+ variable is not needed, LOG is global thus it will be read if available diff --git a/autosubmit/history/data_classes/job_data.py b/autosubmit/history/data_classes/job_data.py index bf1d394e54f94ca2d2845eb42ed378ecb806bc80..33b3b8857db3e494e709d043e8da80ebfde5b404 100644 --- a/autosubmit/history/data_classes/job_data.py +++ b/autosubmit/history/data_classes/job_data.py @@ -16,11 +16,14 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . +from json import dumps, loads + import collections -import autosubmit.history.utils as HUtils -import autosubmit.history.database_managers.database_models as Models from datetime import datetime, timedelta -from json import dumps , loads + +import autosubmit.history.database_managers.database_models as Models +import autosubmit.history.utils as HUtils + class JobData(object): """ diff --git a/autosubmit/history/database_managers/database_manager.py b/autosubmit/history/database_managers/database_manager.py index ccc702f5c63f752c69083d90ed35bd1a9e54fa19..6e882660c387e7d54b4b19405b9f17c0fb33ddb4 100644 --- a/autosubmit/history/database_managers/database_manager.py +++ b/autosubmit/history/database_managers/database_manager.py @@ -16,12 +16,13 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import sqlite3 import os -import autosubmit.history.utils as HUtils -import autosubmit.history.database_managers.database_models as Models +import sqlite3 from abc import ABCMeta +import autosubmit.history.database_managers.database_models as Models +import autosubmit.history.utils as HUtils + DEFAULT_JOBDATA_DIR = os.path.join('/esarchive', 'autosubmit', 'as_metadata', 'data') DEFAULT_HISTORICAL_LOGS_DIR = os.path.join('/esarchive', 'autosubmit', 'as_metadata', 'logs') DEFAULT_LOCAL_ROOT_DIR = os.path.join('/esarchive', 'autosubmit') diff --git a/autosubmit/history/database_managers/experiment_history_db_manager.py b/autosubmit/history/database_managers/experiment_history_db_manager.py index 9e5662af6943de368c61428c1125e25bdfb642c1..78e9e23b289eecb779247c1ae8c28567652a4717 100644 --- a/autosubmit/history/database_managers/experiment_history_db_manager.py +++ b/autosubmit/history/database_managers/experiment_history_db_manager.py @@ -17,10 +17,11 @@ # along with Autosubmit. If not, see . import os import textwrap + import autosubmit.history.utils as HUtils -from . import database_models as Models -from autosubmit.history.data_classes.job_data import JobData from autosubmit.history.data_classes.experiment_run import ExperimentRun +from autosubmit.history.data_classes.job_data import JobData +from . import database_models as Models from .database_manager import DatabaseManager, DEFAULT_JOBDATA_DIR CURRENT_DB_VERSION = 18 diff --git a/autosubmit/history/database_managers/experiment_status_db_manager.py b/autosubmit/history/database_managers/experiment_status_db_manager.py index ed2f9f7b8b1c7def2fac6be5aa0cb60d81752090..4ca93f885680c52cc25a13396f67002205b04040 100644 --- a/autosubmit/history/database_managers/experiment_status_db_manager.py +++ b/autosubmit/history/database_managers/experiment_status_db_manager.py @@ -20,10 +20,11 @@ import os import textwrap import time -from .database_manager import DatabaseManager, DEFAULT_LOCAL_ROOT_DIR -from autosubmitconfigparser.config.basicconfig import BasicConfig + import autosubmit.history.utils as HUtils +from autosubmitconfigparser.config.basicconfig import BasicConfig from . import database_models as Models +from .database_manager import DatabaseManager, DEFAULT_LOCAL_ROOT_DIR BasicConfig.read() diff --git a/autosubmit/history/experiment_history.py b/autosubmit/history/experiment_history.py index 7f6a496487edbd0d1b891f375fa3594326851fb9..fba54b9e6802ac583c8bf368d3d77d2c7e5eb7a4 100644 --- a/autosubmit/history/experiment_history.py +++ b/autosubmit/history/experiment_history.py @@ -16,17 +16,19 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import traceback +from time import time, sleep + import autosubmit.history.database_managers.database_models as Models import autosubmit.history.utils as HUtils -from time import time, sleep -from .database_managers.experiment_history_db_manager import ExperimentHistoryDbManager -from .database_managers.database_manager import DEFAULT_JOBDATA_DIR, DEFAULT_HISTORICAL_LOGS_DIR -from .strategies import PlatformInformationHandler, SingleAssociationStrategy, StraightWrapperAssociationStrategy, TwoDimWrapperDistributionStrategy, GeneralizedWrapperDistributionStrategy -from .data_classes.job_data import JobData +from autosubmitconfigparser.config.basicconfig import BasicConfig from .data_classes.experiment_run import ExperimentRun -from .platform_monitor.slurm_monitor import SlurmMonitor +from .data_classes.job_data import JobData +from .database_managers.database_manager import DEFAULT_JOBDATA_DIR, DEFAULT_HISTORICAL_LOGS_DIR +from .database_managers.experiment_history_db_manager import ExperimentHistoryDbManager from .internal_logging import Logging -from autosubmitconfigparser.config.basicconfig import BasicConfig +from .platform_monitor.slurm_monitor import SlurmMonitor +from .strategies import PlatformInformationHandler, SingleAssociationStrategy, StraightWrapperAssociationStrategy, \ + TwoDimWrapperDistributionStrategy, GeneralizedWrapperDistributionStrategy SECONDS_WAIT_PLATFORM = 60 diff --git a/autosubmit/history/experiment_status.py b/autosubmit/history/experiment_status.py index 9d4c7deb056c88931e32fc6da469c0cf20369e96..4d17262eb9889d8657db9c35bad31d8ad2dd3b18 100644 --- a/autosubmit/history/experiment_status.py +++ b/autosubmit/history/experiment_status.py @@ -17,10 +17,12 @@ # along with Autosubmit. If not, see . import traceback -from .database_managers.experiment_status_db_manager import ExperimentStatusDbManager + +from autosubmitconfigparser.config.basicconfig import BasicConfig from .database_managers.database_manager import DEFAULT_LOCAL_ROOT_DIR, DEFAULT_HISTORICAL_LOGS_DIR +from .database_managers.experiment_status_db_manager import ExperimentStatusDbManager from .internal_logging import Logging -from autosubmitconfigparser.config.basicconfig import BasicConfig + class ExperimentStatus: """ Represents the Experiment Status Mechanism that keeps track of currently active experiments """ diff --git a/autosubmit/history/internal_logging.py b/autosubmit/history/internal_logging.py index f9b667814b62716e0216ac763ea3c470164de4cd..39c6740978eeed4d32ab8b86ec1fc2af8ca48cb3 100644 --- a/autosubmit/history/internal_logging.py +++ b/autosubmit/history/internal_logging.py @@ -16,9 +16,11 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import os + from . import utils as HUtils from .database_managers.database_manager import DEFAULT_HISTORICAL_LOGS_DIR + class Logging: def __init__(self, expid, historiclog_dir_path=DEFAULT_HISTORICAL_LOGS_DIR): self.expid = expid diff --git a/autosubmit/history/platform_monitor/platform_utils.py b/autosubmit/history/platform_monitor/platform_utils.py index 433e654ec225a2d7190ae4ecac067bdb07a8aab7..fc00541dc20698b66e30f81128be691eeb877319 100644 --- a/autosubmit/history/platform_monitor/platform_utils.py +++ b/autosubmit/history/platform_monitor/platform_utils.py @@ -16,9 +16,8 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import os -from time import mktime from datetime import datetime +from time import mktime SLURM_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S" diff --git a/autosubmit/history/strategies.py b/autosubmit/history/strategies.py index c18ae96f5142c3f2851f021d52b55fbece3f8952..d86cd2aa0003dda2e18b1fa4b79805e68fe4af6d 100644 --- a/autosubmit/history/strategies.py +++ b/autosubmit/history/strategies.py @@ -16,11 +16,13 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . +import traceback from abc import ABCMeta, abstractmethod + import autosubmit.history.database_managers.database_models as Models -import traceback +from .database_managers.database_manager import DEFAULT_HISTORICAL_LOGS_DIR from .internal_logging import Logging -from .database_managers.database_manager import DEFAULT_LOCAL_ROOT_DIR, DEFAULT_HISTORICAL_LOGS_DIR + class PlatformInformationHandler: def __init__(self, strategy): diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index e5b921de281206d4d80cc4a06cfff80ae1e7f9d2..90e25fbdb514bd5dd7c86410cde3e7583d37fdf2 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -150,7 +150,8 @@ class Job(object): self._platform = None self._queue = None self._partition = None - + self.x11 = None + self.x11_options = None self.retry_delay = "0" self.platform_name = None # type: str #: (str): Type of the job, as given on job configuration file. (job: TASKTYPE) @@ -188,7 +189,6 @@ class Job(object): self.file = None self.additional_files = [] self.executable = None - self.x11 = False self._local_logs = ('', '') self._remote_logs = ('', '') self.script_name = self.name + ".cmd" @@ -686,6 +686,23 @@ class Job(object): """ self.fail_count += 1 + @property + @autosubmit_parameter(name='x11') + def x11(self): + """Whether to use X11 forwarding""" + return self._x11 + @x11.setter + def x11(self, value): + self._x11 = value + + @property + @autosubmit_parameter(name='x11_options') + def x11_options(self): + """Allows to set salloc parameters for x11""" + return self._x11_options + @x11_options.setter + def x11_options(self, value): + self._x11_options = value # Maybe should be renamed to the plural? def add_parent(self, *parents): """ @@ -1279,6 +1296,7 @@ class Job(object): return parameters def update_platform_associated_parameters(self,as_conf, parameters, job_platform, chunk): + self.x11_options = str(as_conf.jobs_data[self.section].get("X11_OPTIONS", as_conf.platforms_data.get(job_platform.name,{}).get("X11_OPTIONS",""))) self.executable = str(as_conf.jobs_data[self.section].get("EXECUTABLE", as_conf.platforms_data.get(job_platform.name,{}).get("EXECUTABLE",""))) self.total_jobs = int(as_conf.jobs_data[self.section].get("TOTALJOBS", job_platform.total_jobs)) self.max_waiting_jobs = int(as_conf.jobs_data[self.section].get("MAXWAITINGJOBS", job_platform.max_waiting_jobs)) diff --git a/autosubmit/job/job_common.py b/autosubmit/job/job_common.py index d705b1d1dd4205aac431f5e1e657cb86b3107b95..f167d59a8a0e3af64ca3510556a701bc27dece57 100644 --- a/autosubmit/job/job_common.py +++ b/autosubmit/job/job_common.py @@ -13,6 +13,8 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. + +import datetime # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import datetime diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index e2f673563efe1b6859cdfdf37350b05b8c5a08f7..e9c3ef491f14ecfa4efc3db51b07b0de9277ce88 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -17,11 +17,13 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -from autosubmit.job.job import Job from bscearth.utils.date import date2str + +from autosubmit.job.job import Job from autosubmit.job.job_common import Status, Type -from log.log import Log, AutosubmitError, AutosubmitCritical -from collections.abc import Iterable +from log.log import AutosubmitCritical + + class DicJobs: """ Class to create jobs from conf file and to find jobs by start date, member and chunk diff --git a/autosubmit/job/job_grouping.py b/autosubmit/job/job_grouping.py index bcddaf038371b1708fee4d8eb198c77e0855a136..ad9a09bf0402ab2380bf603a978df52d66a6929a 100644 --- a/autosubmit/job/job_grouping.py +++ b/autosubmit/job/job_grouping.py @@ -17,9 +17,11 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -from autosubmit.job.job_common import Status -from bscearth.utils.date import date2str import copy +from bscearth.utils.date import date2str + +from autosubmit.job.job_common import Status + class JobGrouping(object): diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 59580672b51c670dad12b07f05984808bc3bbefb..91a328225f27baa135c737f747dad7cac7274f40 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -13,6 +13,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. + # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import copy @@ -896,7 +897,7 @@ class JobList(object): members_to = str(filter_.get("MEMBERS_TO", "natural")).lower() chunks_to = str(filter_.get("CHUNKS_TO", "natural")).lower() splits_to = str(filter_.get("SPLITS_TO", "natural")).lower() - + if not is_a_natural_relation: if dates_to == "natural": dates_to = "none" @@ -3111,4 +3112,4 @@ class JobList(object): # timestamp).strftime('%Y-%m-%d %H:%M:%S')) return datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') else: - return None + return None \ No newline at end of file diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 7554ddad746eeee5bcdbbb6920b78080a8024e68..abffe50804139649a57084fe96d5f4b228b42391 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -14,15 +14,14 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +import os # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import pickle from sys import setrecursionlimit -import os - -from log.log import Log from autosubmit.database.db_manager import DbManager +from log.log import Log class JobListPersistence(object): diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index 67e833e2719049822cd43220dc0ea68a96bc7ad5..2fce137a7c9e1d0985000b8889d482b1f63f0c60 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -17,17 +17,17 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import collections -from log.log import Log, AutosubmitCritical, AutosubmitError -from autosubmit.job.job_common import Status, Type +import copy +import operator from bscearth.utils.date import sum_str_hours -from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical, JobPackageHorizontal, \ - JobPackageSimpleWrapped, JobPackageHorizontalVertical, JobPackageVerticalHorizontal, JobPackageBase -from operator import attrgetter from math import ceil -import operator +from operator import attrgetter from typing import List -import copy +from autosubmit.job.job_common import Status, Type +from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical, JobPackageHorizontal, \ + JobPackageSimpleWrapped, JobPackageHorizontalVertical, JobPackageVerticalHorizontal, JobPackageBase +from log.log import Log, AutosubmitCritical class JobPackager(object): diff --git a/autosubmit/job/job_utils.py b/autosubmit/job/job_utils.py index 9782122738093e02d00be3c1df3aedd8f3840247..d7071bb7fe1934990220fff7148a13eeb5dba869 100644 --- a/autosubmit/job/job_utils.py +++ b/autosubmit/job/job_utils.py @@ -19,14 +19,14 @@ import networkx import os - -from networkx.algorithms.dag import is_directed_acyclic_graph from networkx import DiGraph -from networkx import dfs_edges from networkx import NetworkXError +from networkx import dfs_edges +from networkx.algorithms.dag import is_directed_acyclic_graph +from typing import Dict + from autosubmit.job.job_package_persistence import JobPackagePersistence from autosubmitconfigparser.config.basicconfig import BasicConfig -from typing import Dict def transitive_reduction(graph): diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index f1de4888578fce3b3c09d5ba6eddc6c8af4ebb3f..ed55f64fbd8a20674dd340f50d7776b429ab445f 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -14,34 +14,31 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# You should have received a copy of the GNU General Public License -# along with Autosubmit. If not, see . -import datetime -import os -import time -import sys -from os import path from os import chdir from os import listdir +from os import path from os import remove -import py3dotplus as pydotplus import copy - +# You should have received a copy of the GNU General Public License +# along with Autosubmit. If not, see . +import datetime +import os +import py3dotplus as pydotplus import subprocess -import autosubmit.history.utils as HUtils -import autosubmit.helpers.utils as HelperUtils +import sys +import time +from typing import Dict, List -from autosubmit.job.job_common import Status +import autosubmit.helpers.utils as HelperUtils +import autosubmit.history.utils as HUtils from autosubmit.job.job import Job +from autosubmit.job.job_common import Status from autosubmitconfigparser.config.basicconfig import BasicConfig from autosubmitconfigparser.config.configcommon import AutosubmitConfig - -from log.log import Log, AutosubmitCritical from autosubmitconfigparser.config.yamlparser import YAMLParserFactory - +from log.log import Log, AutosubmitCritical from .diagram import create_bar_diagram -from typing import Dict, List GENERAL_STATS_OPTION_MAX_LENGTH = 1000 diff --git a/autosubmit/notifications/mail_notifier.py b/autosubmit/notifications/mail_notifier.py index 45c01c1c96071b350c0b7a46ca930c27bb719777..be8e0dcfd4e12d2fd3797ee978cbe21d25c69afe 100644 --- a/autosubmit/notifications/mail_notifier.py +++ b/autosubmit/notifications/mail_notifier.py @@ -17,11 +17,13 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import smtplib import email.utils +import smtplib from email.mime.text import MIMEText + from log.log import Log + class MailNotifier: def __init__(self, basic_config): self.config = basic_config diff --git a/autosubmit/platforms/ecplatform.py b/autosubmit/platforms/ecplatform.py index 99c44aa7af7b2074e1fd40261054d4e03170c4c1..517eed2a1baddb123fa48e049d7fba7b80f099c0 100644 --- a/autosubmit/platforms/ecplatform.py +++ b/autosubmit/platforms/ecplatform.py @@ -14,19 +14,21 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. +import locale # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import locale import os import subprocess -from autosubmit.platforms.paramiko_platform import ParamikoPlatform, ParamikoPlatformException -from log.log import Log,AutosubmitError -from autosubmit.platforms.headers.ec_header import EcHeader +from time import sleep + from autosubmit.platforms.headers.ec_cca_header import EcCcaHeader +from autosubmit.platforms.headers.ec_header import EcHeader from autosubmit.platforms.headers.slurm_header import SlurmHeader +from autosubmit.platforms.paramiko_platform import ParamikoPlatform, ParamikoPlatformException from autosubmit.platforms.wrappers.wrapper_factory import EcWrapperFactory -from time import sleep -import locale +from log.log import Log, AutosubmitError + + class EcPlatform(ParamikoPlatform): """ Class to manage queues with ecaccess diff --git a/autosubmit/platforms/locplatform.py b/autosubmit/platforms/locplatform.py index 2950a71765acafab4a665a174d1ffc527b8e6fe4..543e8b631ec9aadc32d1114962833139f3539cc6 100644 --- a/autosubmit/platforms/locplatform.py +++ b/autosubmit/platforms/locplatform.py @@ -18,16 +18,15 @@ # along with Autosubmit. If not, see . import locale import os -from xml.dom.minidom import parseString import subprocess +from time import sleep +from xml.dom.minidom import parseString - -from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.headers.local_header import LocalHeader - +from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmitconfigparser.config.basicconfig import BasicConfig -from time import sleep -from log.log import Log, AutosubmitError, AutosubmitCritical +from log.log import Log, AutosubmitError + class LocalPlatform(ParamikoPlatform): """ diff --git a/autosubmit/platforms/lsfplatform.py b/autosubmit/platforms/lsfplatform.py index a03ec5dee262ed14507dd98361453148c61c3306..c401bdb3265ac0da89bdedb91aa8e8374df9ddce 100644 --- a/autosubmit/platforms/lsfplatform.py +++ b/autosubmit/platforms/lsfplatform.py @@ -19,8 +19,8 @@ import os -from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.headers.lsf_header import LsfHeader +from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.wrappers.wrapper_factory import LSFWrapperFactory diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index 9788a482a095989d8fc5ebab40dd463dbcb753c1..30f6c588901393ecf8ba293e3b8d9bf898bd6f3d 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -1,26 +1,24 @@ +import Xlib.support.connect as xlib_connect +import datetime import locale -from binascii import hexlify -from contextlib import suppress -from time import sleep -import sys -import socket import os import paramiko -import datetime -import time -import select +import random import re +import select +import socket +import sys +from bscearth.utils.date import date2str from datetime import timedelta -import random +from paramiko.ssh_exception import (SSHException) +from threading import Thread +from time import sleep + from autosubmit.job.job_common import Status from autosubmit.job.job_common import Type from autosubmit.platforms.platform import Platform -from bscearth.utils.date import date2str from log.log import AutosubmitError, AutosubmitCritical, Log -from paramiko.ssh_exception import (SSHException) -import Xlib.support.connect as xlib_connect -from threading import Thread - +import re def threaded(fn): def wrapper(*args, **kwargs): @@ -364,7 +362,7 @@ class ParamikoPlatform(Platform): except Exception as e: try: os.remove(file_path) - except Exception as e: + except Exception as ep: pass if str(e) in "Garbage": if not ignore_log: @@ -827,25 +825,30 @@ class ParamikoPlatform(Platform): self.transport._queue_incoming_channel(channel) def flush_out(self,session): + #flush_out is not working fine + while session.recv_ready(): - sys.stdout.write(session.recv(4096)) + sys.stdout.write(session.recv(4096).decode(locale.getlocale()[1])) while session.recv_stderr_ready(): - sys.stderr.write(session.recv_stderr(4096)) + sys.stderr.write(session.recv_stderr(4096).decode(locale.getlocale()[1])) + @threaded def x11_status_checker(self, session, session_fileno): + poller = None self.transport.accept() while not session.exit_status_ready(): try: - if sys.platform != "linux": - self.poller = self.poller.kqueue() - else: - self.poller = self.poller.poll() + if type(self.poller) is not list: + if sys.platform != "linux": + poller = self.poller.kqueue() + else: + poller = self.poller.poll() # accept subsequent x11 connections if any if len(self.transport.server_accepts) > 0: self.transport.accept() - if not self.poller: # this should not happen, as we don't have a timeout. + if not poller: # this should not happen, as we don't have a timeout. break - for fd, event in self.poller: + for fd, event in poller: if fd == session_fileno: self.flush_out(session) # data either on local/remote x11 socket @@ -862,7 +865,6 @@ class ParamikoPlatform(Platform): except Exception as e: pass - def exec_command(self, command, bufsize=-1, timeout=None, get_pty=False,retries=3, x11=False): """ Execute a command on the SSH server. A new `.Channel` is opened and @@ -887,18 +889,28 @@ class ParamikoPlatform(Platform): """ while retries > 0: try: - chan = self.transport.open_session() if x11 == "true": display = os.getenv('DISPLAY') if display is None or not display: display = "localhost:0" self.local_x11_display = xlib_connect.get_display(display) - chan.request_x11(handler=self.x11_handler) + chan = self.transport.open_session() + chan.request_x11(single_connection=False,handler=self.x11_handler) else: + chan = self.transport.open_session() chan.settimeout(timeout) if x11 == "true": - command = command + " ; sleep infinity" - chan.exec_command(command) + if "timeout" in command: + timeout_command = command.split("timeout ")[1].split(" ")[0] + if timeout_command == 0: + timeout_command = "infinity" + command = f'{command} ; sleep {timeout_command} 2>/dev/null' + #command = f'export display {command}' + Log.info(command) + try: + chan.exec_command(command) + except BaseException as e: + raise AutosubmitCritical("Failed to execute command: %s" % e) chan_fileno = chan.fileno() self.poller.register(chan_fileno, select.POLLIN) self.x11_status_checker(chan, chan_fileno) @@ -916,14 +928,7 @@ class ParamikoPlatform(Platform): retries = retries - 1 if retries <= 0: return False , False, False - def exec_command_x11(self, command, bufsize=-1, timeout=None, get_pty=False,retries=3, x11=False): - session = self.transport.open_session() - session.request_x11(handler=self.x11_handler) - session.exec_command(command + " ; sleep infinity") - session_fileno = session.fileno() - self.poller.register(session_fileno, select.POLLIN) - self.x11_status_checker(session, session_fileno) - pass + def send_command(self, command, ignore_log=False, x11 = False): """ Sends given command to HPC @@ -956,8 +961,10 @@ class ParamikoPlatform(Platform): channel.settimeout(timeout) stdin.close() channel.shutdown_write() - stdout_chunks.append(stdout.channel.recv(len(stdout.channel.in_buffer))) - while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready(): + stdout_chunks.append(stdout.channel.recv(len(stdout.channel.in_buffer))) + i = 0 + x11_exit = False + while (not channel.closed or channel.recv_ready() or channel.recv_stderr_ready() ) and not x11_exit: # stop if channel was closed prematurely, and there is no data in the buffers. got_chunk = False readq, _, _ = select.select([stdout.channel], [], [], 2) @@ -965,19 +972,22 @@ class ParamikoPlatform(Platform): if c.recv_ready(): stdout_chunks.append( stdout.channel.recv(len(c.in_buffer))) - #stdout_chunks.append(" ") got_chunk = True if c.recv_stderr_ready(): # make sure to read stderr to prevent stall stderr_readlines.append( stderr.channel.recv_stderr(len(c.in_stderr_buffer))) - #stdout_chunks.append(" ") got_chunk = True - if x11 == "true": - got_chunk = True - break + if x11 == "true": + if len(stderr_readlines) > 0: + job_id = re.findall(r'\d+', str(stderr_readlines[0]))[0] + stdout_chunks.append(job_id.encode(lang)) + print(job_id) + stderr_readlines = [] + x11_exit = True + break if not got_chunk and stdout.channel.exit_status_ready() and not stderr.channel.recv_stderr_ready() and not stdout.channel.recv_ready(): - # indicate that we're not going to read from this channel anymore + # indicate that we're not going to read from this channel any more stdout.channel.shutdown_read() # close the channel stdout.channel.close() @@ -986,8 +996,6 @@ class ParamikoPlatform(Platform): if not x11: stdout.close() stderr.close() - - self._ssh_output = "" self._ssh_output_err = "" for s in stdout_chunks: @@ -995,7 +1003,6 @@ class ParamikoPlatform(Platform): self._ssh_output += s.decode(lang) for errorLineCase in stderr_readlines: self._ssh_output_err += errorLineCase.decode(lang) - errorLine = errorLineCase.lower().decode(lang) if "not active" in errorLine: raise AutosubmitError( diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index 534a64a425a4db213e0093b70369ca98223a8192..8589f6813280576db2a94bd805069d6d8827b893 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -18,22 +18,23 @@ # along with Autosubmit. If not, see . -import os from collections import defaultdict -from log.log import Log,AutosubmitCritical,AutosubmitError -from autosubmitconfigparser.config.basicconfig import BasicConfig -from autosubmitconfigparser.config.configcommon import AutosubmitConfig -from .submitter import Submitter -from autosubmit.platforms.psplatform import PsPlatform +import os + +from autosubmit.platforms.ecplatform import EcPlatform +from autosubmit.platforms.locplatform import LocalPlatform from autosubmit.platforms.lsfplatform import LsfPlatform +from autosubmit.platforms.paramiko_platform import ParamikoPlatformException from autosubmit.platforms.pbsplatform import PBSPlatform +from autosubmit.platforms.pjmplatform import PJMPlatform +from autosubmit.platforms.psplatform import PsPlatform from autosubmit.platforms.sgeplatform import SgePlatform -from autosubmit.platforms.ecplatform import EcPlatform from autosubmit.platforms.slurmplatform import SlurmPlatform -from autosubmit.platforms.pjmplatform import PJMPlatform -from autosubmit.platforms.locplatform import LocalPlatform -from autosubmit.platforms.paramiko_platform import ParamikoPlatformException +from autosubmitconfigparser.config.basicconfig import BasicConfig +from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from log.log import Log, AutosubmitError +from .submitter import Submitter class ParamikoSubmitter(Submitter): diff --git a/autosubmit/platforms/pbsplatform.py b/autosubmit/platforms/pbsplatform.py index 132b8715c03cdddd367669af807384a8134a933e..9960e424680adc84bec77f7c477a4b198d10d5c5 100644 --- a/autosubmit/platforms/pbsplatform.py +++ b/autosubmit/platforms/pbsplatform.py @@ -19,12 +19,11 @@ import os -from autosubmit.platforms.paramiko_platform import ParamikoPlatform -from log.log import Log, AutosubmitCritical - from autosubmit.platforms.headers.pbs10_header import Pbs10Header from autosubmit.platforms.headers.pbs11_header import Pbs11Header from autosubmit.platforms.headers.pbs12_header import Pbs12Header +from autosubmit.platforms.paramiko_platform import ParamikoPlatform +from log.log import Log, AutosubmitCritical class PBSPlatform(ParamikoPlatform): diff --git a/autosubmit/platforms/pjmplatform.py b/autosubmit/platforms/pjmplatform.py index 3b88cbe318329fae0f0c8e2e00f4e2afdaf5cdc4..7acca0fe7e5232f80a0a48fb7045d57cfa48d453 100644 --- a/autosubmit/platforms/pjmplatform.py +++ b/autosubmit/platforms/pjmplatform.py @@ -23,11 +23,12 @@ from time import sleep from typing import List, Union from autosubmit.job.job_common import Status -from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.headers.pjm_header import PJMHeader +from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.wrappers.wrapper_factory import PJMWrapperFactory from log.log import AutosubmitCritical, AutosubmitError, Log + class PJMPlatform(ParamikoPlatform): """ Class to manage jobs to host using PJM scheduler diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 79ea7919aa9b2ecdf427f732ac5961075397c44f..20cb28d4ecb1be72d633e51a74cd9a7974f41718 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -62,6 +62,7 @@ class Platform(object): self._submit_hold_cmd = None self._submit_command_name = None self._submit_cmd = None + self._submit_cmd_x11 = None self._checkhost_cmd = None self.cancel_cmd = None @@ -265,7 +266,8 @@ class Platform(object): save = True if not inspect: job_list.save() - valid_packages_to_submit.append(package) + if package.x11 != "true": + valid_packages_to_submit.append(package) # Log.debug("FD end-submit: {0}".format(log.fd_show.fd_table_status_str(open())) except (IOError, OSError): if package.jobs[0].id != 0: diff --git a/autosubmit/platforms/psplatform.py b/autosubmit/platforms/psplatform.py index 2a5fe05d51eafdd74abb8e49afc5b3df446ab25b..433eb4063752714c209fefbeac1819fd4029cfd3 100644 --- a/autosubmit/platforms/psplatform.py +++ b/autosubmit/platforms/psplatform.py @@ -20,8 +20,8 @@ import os from xml.dom.minidom import parseString -from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.headers.ps_header import PsHeader +from autosubmit.platforms.paramiko_platform import ParamikoPlatform class PsPlatform(ParamikoPlatform): @@ -97,7 +97,7 @@ class PsPlatform(ParamikoPlatform): def get_submit_cmd(self, job_script, job, hold=False, export=""): wallclock = self.parse_time(job.wallclock) - seconds = int(wallclock.days * 86400 + wallclock.seconds * 60) + seconds = int(wallclock.days * 86400 + wallclock.seconds) * 1.30 if export == "none" or export == "None" or export is None or export == "": export = "" else: diff --git a/autosubmit/platforms/sgeplatform.py b/autosubmit/platforms/sgeplatform.py index 58671cd98896fcd2c32f8c086dd73b51878f8f3b..662d58cb2cbba4c0310c2059a3a0811779f7e4ce 100644 --- a/autosubmit/platforms/sgeplatform.py +++ b/autosubmit/platforms/sgeplatform.py @@ -19,11 +19,10 @@ import os import subprocess - from xml.dom.minidom import parseString -from autosubmit.platforms.paramiko_platform import ParamikoPlatform from autosubmit.platforms.headers.sge_header import SgeHeader +from autosubmit.platforms.paramiko_platform import ParamikoPlatform class SgePlatform(ParamikoPlatform): diff --git a/autosubmit/platforms/slurmplatform.py b/autosubmit/platforms/slurmplatform.py index a64d386e8375f9e19a4f01f1b2a5b76751503b0a..d83aed18bbdaa4bbaee629afec3867314c2032a0 100644 --- a/autosubmit/platforms/slurmplatform.py +++ b/autosubmit/platforms/slurmplatform.py @@ -49,6 +49,7 @@ class SlurmPlatform(ParamikoPlatform): self._submit_hold_cmd = None self._submit_command_name = None self._submit_cmd = None + self.x11_options = None self._checkhost_cmd = None self.cancel_cmd = None self._header = SlurmHeader() @@ -280,6 +281,7 @@ class SlurmPlatform(ParamikoPlatform): self._checkhost_cmd = "echo 1" self._submit_cmd = 'sbatch -D {1} {1}/'.format( self.host, self.remote_log_dir) + self._submit_cmd_x11 = f'-D {self.remote_log_dir} {self.remote_log_dir}' self._submit_command_name = "sbatch" self._submit_hold_cmd = 'sbatch -H -D {1} {1}/'.format( self.host, self.remote_log_dir) @@ -288,6 +290,12 @@ class SlurmPlatform(ParamikoPlatform): self.get_cmd = "scp" self.mkdir_cmd = "mkdir -p " + self.remote_log_dir + def get_submit_cmd_x11(self, args, script_name): + """ + Returns the submit command for the platform + """ + return f'salloc {args} {self._submit_cmd_x11}/{script_name}' + def hold_job(self, job): try: cmd = "scontrol release {0} ; sleep 2 ; scontrol hold {0} ".format(job.id) @@ -496,12 +504,12 @@ class SlurmPlatform(ParamikoPlatform): if outputlines.find("failed") != -1: raise AutosubmitCritical( "Submission failed. Command Failed", 7014) - jobs_id = [] - for output in outputlines.splitlines(): - jobs_id.append(int(output.split(' ')[3])) if x11 == "true": - return jobs_id[0] + return int(outputlines.splitlines()[0]) else: + jobs_id = [] + for output in outputlines.splitlines(): + jobs_id.append(int(output.split(' ')[3])) return jobs_id except IndexError: raise AutosubmitCritical( @@ -523,10 +531,7 @@ class SlurmPlatform(ParamikoPlatform): x11 = job.x11 if x11 == "true": - if not hold: - return export + self._submit_cmd + job_script - else: - return export + self._submit_hold_cmd + job_script + return export + self.get_submit_cmd_x11(job.x11_options.strip(""), job_script.strip("")) else: try: lang = locale.getlocale()[1] diff --git a/autosubmit/statistics/jobs_stat.py b/autosubmit/statistics/jobs_stat.py index b2d1de97b02f51c41555046304d32743d4c106be..c0338fe18b9f55b31a160a8756df27468c10c318 100644 --- a/autosubmit/statistics/jobs_stat.py +++ b/autosubmit/statistics/jobs_stat.py @@ -1,7 +1,9 @@ #!/bin/env/python from datetime import datetime, timedelta + from .utils import timedelta2hours + class JobStat(object): def __init__(self, name, processors, wallclock, section, date, member, chunk): # type: (str, int, float, str, str, str, str) -> None diff --git a/autosubmit/statistics/utils.py b/autosubmit/statistics/utils.py index 4dc9acf2f16d869fb67faab8a72085e9083f6967..a0260b8752106a2603c86978df65d2cc61f1d50a 100644 --- a/autosubmit/statistics/utils.py +++ b/autosubmit/statistics/utils.py @@ -1,12 +1,13 @@ #!/bin/env/python import math -from autosubmit.job.job import Job from datetime import datetime, timedelta -from autosubmit.job.job_common import Status from typing import List, Tuple +from autosubmit.job.job import Job +from autosubmit.job.job_common import Status from log.log import AutosubmitCritical + def filter_by_section(jobs, section): # type: (List[Job], str) -> List[Job] """ Filter jobs by provided section """ diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index b0a000bc09def9d33be0b8a7e61a24e606023b65..0b12f29efcab1dd5d3f3b9c05d85ce6cffe0e9d6 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -111,7 +111,6 @@ To do this use: * NODES: nodes number to be submitted to the HPC. If not specified, the directive is not added. - * HYPERTHREADING: Enables Hyper-threading, this will double the max amount of threads. defaults to false. ( Not available on slurm platforms ) * QUEUE: queue to add the job to. If not specified, uses PLATFORM default. @@ -147,6 +146,10 @@ There are also other, less used features that you can use: * QUEUE: queue to add the job to. If not specified, uses PLATFORM default. +* X11: Allows to run a job with X11 forwarding using salloc. + +* X11_OPTIONS: Allows to set salloc ( X11 ) options. EX: "-n 4 --qos=debug --partition=debug --x11=all --time=00:30:00" + How to configure email notifications ------------------------------------ diff --git a/setup.py b/setup.py index f96a9a0c9f7159cd9285994bb3c6acbe0baec28b..676a96fc164a95009c04f32e40efe9a6cf4cbb0a 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( url='http://www.bsc.es/projects/earthscience/autosubmit/', download_url='https://earth.bsc.es/wiki/doku.php?id=tools:autosubmit', keywords=['climate', 'weather', 'workflow', 'HPC'], - install_requires=['ruamel.yaml==0.17.21','cython','autosubmitconfigparser','bcrypt>=3.2','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','pythondialog','pytest','nose','coverage','PyNaCl>=1.4.0','Pygments','psutil'], + install_requires=['PyNaCl>=1.5.0','ruamel.yaml==0.17.21','cython','autosubmitconfigparser','bcrypt>=3.2','packaging>19','six>=1.10.0','configobj>=5.0.6','argparse>=1.4.0','python-dateutil>=2.8.2','matplotlib<3.6','numpy<1.22','py3dotplus>=1.1.0','pyparsing>=3.0.7','paramiko>=2.9.2','mock>=4.0.3','portalocker>=2.3.2','networkx==2.6.3','requests>=2.27.1','bscearth.utils>=0.5.2','cryptography>=36.0.1','setuptools>=60.8.2','xlib>=0.21','pip>=22.0.3','pythondialog','pytest','nose','coverage','Pygments','psutil'], classifiers=[ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.9",