diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index f81ceb80498dbc9aeeadd76543530b0b03c8888a..489b8ff9593df38e973c387188c3ec10d2cebc59 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1455,6 +1455,7 @@ class Autosubmit: return True except portalocker.AlreadyLocked: + Autosubmit.show_lock_warning(expid) except WrongTemplateException: @@ -3164,6 +3165,10 @@ class Autosubmit: return True # catching Exception except (KeyboardInterrupt, Exception) as e: + all_threads = threading.enumerate() + for thread in all_threads: + thread.join() + # Setting signal handler to handle subsequent CTRL-C signal.signal(signal.SIGINT, signal_handler_create) # Terminating locking as sugested by the portalocker developer diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index 101026694a1b96b53f249af47b3866fe90242826..928171da19400a7bd4a22ce58ac5e7fbcf383564 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1362,14 +1362,16 @@ done def _is_over_wallclock(self, start_time, wallclock): elapsed = datetime.datetime.now() - parse_date(start_time) - wallclock = datetime.datetime.strptime(wallclock, '%H:%M') - total = 0.0 - if wallclock.hour > 0: - total = wallclock.hour - if wallclock.minute > 0: - total += wallclock.minute/60.0 - if wallclock.second > 0: - total += wallclock.second/60.0/60.0 + splited = wallclock.split(':') + if len(splited) == 3: + total = int(splited[0]) + int(splited[1]) + int(splited[2]) + elif len(splited) == 2: + total = int(splited[0]) + int(splited[1]) + elif len(splited) == 1: + total = int(splited[0]) + else: + total = 0 + total = total * 1.15 hour = int(total) minute = int((total - int(total)) * 60.0) diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index 03995f76a2f3bd4662ed7f47aa6bb62476f15141..875910c3a419c19ab03ab1042ed039d3b0b844c6 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -65,7 +65,7 @@ class ParamikoPlatform(Platform): retries = 2 retry = 0 connected = False - while connected == False and retry < retries: + while connected is False and retry < retries: if self.connect(True): connected = True retry += 1 @@ -513,6 +513,46 @@ class ParamikoPlatform(Platform): :rtype: str """ raise NotImplementedError + def exec_command(self, command, bufsize=-1, timeout=None, get_pty=False,retries=3,x11=False): + """ + Execute a command on the SSH server. A new `.Channel` is opened and + the requested command is executed. The command's input and output + streams are returned as Python ``file``-like objects representing + stdin, stdout, and stderr. + + :param str command: the command to execute + :param int bufsize: + interpreted the same way as by the built-in ``file()`` function in + Python + :param int timeout: + set command's channel timeout. See `Channel.settimeout`.settimeout + :return: + the stdin, stdout, and stderr of the executing command, as a + 3-tuple + + :raises SSHException: if the server fails to execute the command + """ + while retries > 0: + try: + chan = self._ssh._transport.open_session() + if get_pty: + chan.get_pty() + if x11: + chan.request_x11() + chan.settimeout(timeout) + chan.exec_command(command) + stdin = chan.makefile('wb', bufsize) + stdout = chan.makefile('r', bufsize) + stderr = chan.makefile_stderr('r', bufsize) + return stdin, stdout, stderr + except paramiko.SSHException as e: + if str(e) in "SSH session not active": + self._ssh = None + self.restore_connection() + timeout = timeout + 60 + retries = retries - 1 + if retries <= 0: + return False , False, False def send_command(self, command, ignore_log=False): """ @@ -527,16 +567,16 @@ class ParamikoPlatform(Platform): if not self.restore_connection(): return False if "-rP" in command or "find" in command or "convertLink" in command: - # Max Wait 1hour if the command is a copy or simbolic links ( migrate can trigger long times) timeout = 60*60 elif "rm" in command: timeout = 60/2 else: timeout = 60*2 try: - stdin, stdout, stderr = self._ssh.exec_command(command) + stdin, stdout, stderr = self.exec_command(command,timeout=timeout) + if not stdin and not stdout and not stderr: + raise channel = stdout.channel - channel.settimeout(timeout) stdin.close() channel.shutdown_write() stdout_chunks = []