From 7b7aec77f02d7efad5b2808b07e42219ecb25b40 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Tue, 7 Nov 2023 10:30:37 +0100 Subject: [PATCH] Apply more fixes for IDE warnings and errors --- autosubmit/autosubmit.py | 4 ++-- autosubmit/database/db_common.py | 3 +-- autosubmit/experiment/statistics.py | 2 +- autosubmit/git/autosubmit_git.py | 2 +- autosubmit/helpers/utils.py | 4 +--- autosubmit/history/platform_monitor/platform_utils.py | 1 - autosubmit/history/strategies.py | 2 +- autosubmit/job/job_dict.py | 4 ++-- autosubmit/job/job_list.py | 4 ++-- autosubmit/job/job_packager.py | 3 +-- autosubmit/job/job_packages.py | 1 - autosubmit/monitor/monitor.py | 4 ++-- autosubmit/platforms/ecplatform.py | 1 - autosubmit/platforms/locplatform.py | 4 +--- autosubmit/platforms/paramiko_platform.py | 3 --- autosubmit/platforms/paramiko_submitter.py | 2 +- autosubmit/platforms/platform.py | 3 +-- autosubmit/platforms/slurmplatform.py | 1 - 18 files changed, 17 insertions(+), 31 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1796337f0..c4d6c34df 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -73,7 +73,7 @@ import re import random import signal import datetime -import log.fd_show as fd_show +# import log.fd_show as fd_show import portalocker from pkg_resources import require, resource_listdir, resource_string, resource_filename from collections import defaultdict @@ -5449,7 +5449,7 @@ class Autosubmit: referenced_jobs_to_remove = set() job_list_wrappers = copy.deepcopy(job_list) jobs_wr = copy.deepcopy(job_list.get_job_list()) - [job for job in jobs_wr if ( + jobs_wr = [job for job in jobs_wr if ( job.status != Status.COMPLETED)] for job in jobs_wr: for child in job.children: diff --git a/autosubmit/database/db_common.py b/autosubmit/database/db_common.py index 9f93e04c5..81692d954 100644 --- a/autosubmit/database/db_common.py +++ b/autosubmit/database/db_common.py @@ -23,8 +23,7 @@ Module containing functions to manage autosubmit's database. import os import sqlite3 import multiprocessing -import autosubmit -from log.log import Log, AutosubmitCritical, AutosubmitError +from log.log import Log, AutosubmitCritical Log.get_logger("Autosubmit") from autosubmitconfigparser.config.basicconfig import BasicConfig diff --git a/autosubmit/experiment/statistics.py b/autosubmit/experiment/statistics.py index 793210923..3f2ec09ed 100644 --- a/autosubmit/experiment/statistics.py +++ b/autosubmit/experiment/statistics.py @@ -20,7 +20,7 @@ import datetime from autosubmit.job.job import Job from autosubmit.monitor.utils import FixedSizeList -from log.log import Log, AutosubmitError, AutosubmitCritical +from log.log import Log def timedelta2hours(deltatime): diff --git a/autosubmit/git/autosubmit_git.py b/autosubmit/git/autosubmit_git.py index 976b02f9f..6ef5ae462 100644 --- a/autosubmit/git/autosubmit_git.py +++ b/autosubmit/git/autosubmit_git.py @@ -25,7 +25,7 @@ import shutil # from autosubmit import Autosubmit from autosubmitconfigparser.config.basicconfig import BasicConfig from time import time -from log.log import Log, AutosubmitCritical, AutosubmitError +from log.log import Log, AutosubmitCritical Log.get_logger("Autosubmit") diff --git a/autosubmit/helpers/utils.py b/autosubmit/helpers/utils.py index fca94a126..146a727af 100644 --- a/autosubmit/helpers/utils.py +++ b/autosubmit/helpers/utils.py @@ -1,9 +1,7 @@ import os import pwd -from log.log import Log, AutosubmitCritical -from autosubmitconfigparser.config.basicconfig import BasicConfig -from typing import Tuple +from log.log import AutosubmitCritical def check_experiment_ownership(expid, basic_config, raise_error=False, logger=None): # [A-Za-z09]+ variable is not needed, LOG is global thus it will be read if available diff --git a/autosubmit/history/platform_monitor/platform_utils.py b/autosubmit/history/platform_monitor/platform_utils.py index 433e654ec..7dbad5e20 100644 --- a/autosubmit/history/platform_monitor/platform_utils.py +++ b/autosubmit/history/platform_monitor/platform_utils.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import os from time import mktime from datetime import datetime diff --git a/autosubmit/history/strategies.py b/autosubmit/history/strategies.py index c18ae96f5..5a544fd46 100644 --- a/autosubmit/history/strategies.py +++ b/autosubmit/history/strategies.py @@ -20,7 +20,7 @@ from abc import ABCMeta, abstractmethod import autosubmit.history.database_managers.database_models as Models import traceback from .internal_logging import Logging -from .database_managers.database_manager import DEFAULT_LOCAL_ROOT_DIR, DEFAULT_HISTORICAL_LOGS_DIR +from .database_managers.database_manager import DEFAULT_HISTORICAL_LOGS_DIR class PlatformInformationHandler: def __init__(self, strategy): diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index 9645f493f..8ece823da 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -20,8 +20,8 @@ from autosubmit.job.job import Job from bscearth.utils.date import date2str from autosubmit.job.job_common import Status, Type -from log.log import Log, AutosubmitError, AutosubmitCritical -from collections.abc import Iterable +from log.log import AutosubmitCritical + class DicJobs: """ Class to create jobs from conf file and to find jobs by start date, member and chunk diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index af80e2158..35f91e4fb 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -1576,12 +1576,12 @@ class JobList(object): select_jobs_by_name = jobs_to_check[0] unparsed_jobs = jobs_to_check[1] if not ";" in unparsed_jobs: - if '[': + if '[' in unparsed_jobs: select_all_jobs_by_section = unparsed_jobs filter_jobs_by_section = "" else: select_all_jobs_by_section = "" - filter_jobs_by_section = unparsed_jbos + filter_jobs_by_section = unparsed_jobs else: aux = unparsed_jobs.split(';') select_all_jobs_by_section = aux[0] diff --git a/autosubmit/job/job_packager.py b/autosubmit/job/job_packager.py index 67e833e27..810dcf9c8 100644 --- a/autosubmit/job/job_packager.py +++ b/autosubmit/job/job_packager.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . import collections -from log.log import Log, AutosubmitCritical, AutosubmitError +from log.log import Log, AutosubmitCritical from autosubmit.job.job_common import Status, Type from bscearth.utils.date import sum_str_hours from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical, JobPackageHorizontal, \ @@ -124,7 +124,6 @@ class JobPackager(object): self._maxTotalProcessors = 0 def compute_weight(self, job_list): - job = self jobs_by_section = dict() held_jobs = self._jobs_list.get_held_jobs() jobs_held_by_section = dict() diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index ebdbf3d7c..567631ac3 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -98,7 +98,6 @@ class JobPackageBase(object): for job in jobs: if str(job.check).lower() == str(Job.CHECK_ON_SUBMISSION).lower(): if only_generate: - exit_ = True break if not os.path.exists(os.path.join(configuration.get_project_dir(), job.file)): lock.acquire() diff --git a/autosubmit/monitor/monitor.py b/autosubmit/monitor/monitor.py index f1de48885..e1e011817 100644 --- a/autosubmit/monitor/monitor.py +++ b/autosubmit/monitor/monitor.py @@ -480,8 +480,8 @@ class Monitor: Log.result('Status txt created at {0}', output_file) def write_output_txt_recursive(self, job, output_file, level, path): - log_out = "" - log_err = "" + # log_out = "" + # log_err = "" # + " " + log_out + " " + log_err + "\n" output = level + job.name + " " + \ Status().VALUE_TO_KEY[job.status] + "\n" diff --git a/autosubmit/platforms/ecplatform.py b/autosubmit/platforms/ecplatform.py index 3c4110f00..b029c6070 100644 --- a/autosubmit/platforms/ecplatform.py +++ b/autosubmit/platforms/ecplatform.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . -import locale import os import subprocess from autosubmit.platforms.paramiko_platform import ParamikoPlatform, ParamikoPlatformException diff --git a/autosubmit/platforms/locplatform.py b/autosubmit/platforms/locplatform.py index 7f41060eb..201f2c956 100644 --- a/autosubmit/platforms/locplatform.py +++ b/autosubmit/platforms/locplatform.py @@ -27,7 +27,7 @@ from autosubmit.platforms.headers.local_header import LocalHeader from autosubmitconfigparser.config.basicconfig import BasicConfig from time import sleep -from log.log import Log, AutosubmitError, AutosubmitCritical +from log.log import Log, AutosubmitError class LocalPlatform(ParamikoPlatform): """ @@ -232,8 +232,6 @@ class LocalPlatform(ParamikoPlatform): os.rename(os.path.join(path_root, src),os.path.join(path_root, dest)) return True except IOError as e: - raise AutosubmitError('File {0} does not exists, something went wrong with the platform'.format( - path_root), 6004, str(e)) if must_exist: raise AutosubmitError("File {0} does not exists".format( os.path.join(path_root,src)), 6004, str(e)) diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index 58582bc0b..74ef11412 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -1,5 +1,4 @@ import locale -from binascii import hexlify from contextlib import suppress from time import sleep import sys @@ -7,7 +6,6 @@ import socket import os import paramiko import datetime -import time import select import re from datetime import timedelta @@ -15,7 +13,6 @@ import random from autosubmit.job.job_common import Status from autosubmit.job.job_common import Type from autosubmit.platforms.platform import Platform -from bscearth.utils.date import date2str from log.log import AutosubmitError, AutosubmitCritical, Log from paramiko.ssh_exception import (SSHException) import Xlib.support.connect as xlib_connect diff --git a/autosubmit/platforms/paramiko_submitter.py b/autosubmit/platforms/paramiko_submitter.py index ce8c9b358..e575e0869 100644 --- a/autosubmit/platforms/paramiko_submitter.py +++ b/autosubmit/platforms/paramiko_submitter.py @@ -21,7 +21,7 @@ import os from collections import defaultdict -from log.log import Log,AutosubmitCritical,AutosubmitError +from log.log import Log,AutosubmitError from autosubmitconfigparser.config.basicconfig import BasicConfig from autosubmitconfigparser.config.configcommon import AutosubmitConfig from .submitter import Submitter diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 95fea2bcd..2c654afa0 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -1,6 +1,5 @@ import locale import os -from pathlib import Path import traceback from autosubmit.job.job_common import Status @@ -8,7 +7,7 @@ from typing import List, Union from autosubmit.helpers.parameters import autosubmit_parameter from log.log import AutosubmitCritical, AutosubmitError, Log -import getpass + class Platform(object): """ Class to manage the connections to the different platforms. diff --git a/autosubmit/platforms/slurmplatform.py b/autosubmit/platforms/slurmplatform.py index acfaaf7ba..b9085ac3e 100644 --- a/autosubmit/platforms/slurmplatform.py +++ b/autosubmit/platforms/slurmplatform.py @@ -386,7 +386,6 @@ class SlurmPlatform(ParamikoPlatform): detailed_data[name] = extra_data steps.append(name) submit = start = finish = energy = nnodes = ncpus = 0 - status = "UNKNOWN" # Take first line as source line = lines[0].strip().split() ncpus = int(line[2] if len(line) > 2 else 0) -- GitLab