From fef1e4144df62b63a1f9783b92b948add2160b56 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 2 Jul 2024 13:53:07 +0200 Subject: [PATCH 1/8] Fixed migrate command --- autosubmit/autosubmit.py | 6 +++--- autosubmit/platforms/ecplatform.py | 2 +- autosubmit/platforms/locplatform.py | 2 +- autosubmit/platforms/paramiko_platform.py | 20 +++++++++++++++++--- autosubmit/platforms/platform.py | 2 +- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index e5f07216a..1d27a7d73 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3073,9 +3073,9 @@ class Autosubmit: Log.result( "[OPTIONAL] HOST_TO directive not found. The directive HOST will remain unchanged") p = submitter.platforms[platform] - if p.temp_dir not in already_moved: + if p.root_dir not in already_moved: if p.root_dir != p.temp_dir and len(p.temp_dir) > 0: - already_moved.add(p.temp_dir) + already_moved.add(p.root_dir) # find /home/bsc32/bsc32070/dummy3 -type l -lname '/*' -printf ' ln -sf "$(realpath -s --relative-to="%p" $(readlink "%p")")" \n' > script.sh # command = "find " + p.root_dir + " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n'" Log.info( @@ -3112,7 +3112,7 @@ class Autosubmit: Log.info( "Moving remote files/dirs on {0}", platform) p.send_command("chmod 777 -R " + p.root_dir) - if not p.move_file(p.root_dir, os.path.join(p.temp_dir, experiment_id), False): + if not p.move_file(p.root_dir, os.path.join(p.temp_dir, experiment_id), False, path_root=""): Log.result("No data found in {0} for [{1}]\n".format( p.root_dir, platform)) except IOError as e: diff --git a/autosubmit/platforms/ecplatform.py b/autosubmit/platforms/ecplatform.py index 9515692e1..5fa756879 100644 --- a/autosubmit/platforms/ecplatform.py +++ b/autosubmit/platforms/ecplatform.py @@ -236,7 +236,7 @@ class EcPlatform(ParamikoPlatform): raise AutosubmitError('Could not send file {0} to {1}'.format(os.path.join(self.tmp_path, filename),os.path.join(self.get_files_path(), filename)),6005,e.message) return True - def move_file(self, src, dest, must_exist = False): + def move_file(self, src, dest, must_exist = False, path_root = None): command = "ecaccess-file-move {0}:{1} {0}:{2}".format(self.host,os.path.join(self.remote_log_dir,src) , os.path.join(self.remote_log_dir,dest)) try: retries = 0 diff --git a/autosubmit/platforms/locplatform.py b/autosubmit/platforms/locplatform.py index a51b8882a..20c1b7b20 100644 --- a/autosubmit/platforms/locplatform.py +++ b/autosubmit/platforms/locplatform.py @@ -198,7 +198,7 @@ class LocalPlatform(ParamikoPlatform): Log.debug('Could not remove file {0}'.format(os.path.join(self.tmp_path, filename))) return False return True - def move_file(self, src, dest, must_exist=False): + def move_file(self, src, dest, must_exist=False, path_root=None): """ Moves a file on the platform (includes .err and .out) :param src: source name diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index dcad9cdeb..fd65af9bb 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -380,7 +380,7 @@ class ParamikoPlatform(Platform): raise AutosubmitCritical( "Wrong User or invalid .ssh/config. Or invalid user in platform.conf or public key not set ", 7051, e.message) - def move_file(self, src, dest, must_exist=False): + def move_file(self, src, dest, must_exist=False, path_root=None): """ Moves a file on the platform (includes .err and .out) :param src: source name @@ -389,8 +389,9 @@ class ParamikoPlatform(Platform): :param must_exist: ignore if file exist or not :type dest: str """ - try: + if path_root is None: path_root = self.get_files_path() + try: src = os.path.join(path_root, src) dest = os.path.join(path_root, dest) self._ftpChannel.rename(src,dest) @@ -399,11 +400,24 @@ class ParamikoPlatform(Platform): except IOError as e: if str(e) in "Garbage": raise AutosubmitError('File {0} does not exists, something went wrong with the platform'.format(os.path.join(path_root,src)), 6004, e.message) + if e.message.lower() in "failure": + try: + Log.warning("FTP Channel did not work, trying with bash command") + bash_command = "mv {0} {1}".format(src, dest) + self.send_command(bash_command) + if "file exists" in self.get_ssh_output_err().lower() or "move failed" in self.get_ssh_output_err().lower(): + Log.warning("Bash: File {0} was already moved or couldn't be moved".format(src)) + else: + Log.result("Bash: Move command was successful") + return True + except BaseException as e: + raise AutosubmitError('File {0} does not exists, something went wrong with the platform'.format( + os.path.join(path_root, src)), 6004, e.message) if must_exist: raise AutosubmitError("File {0} does not exists".format( os.path.join(path_root,src)), 6004, e.message) else: - Log.debug("File {0} doesn't exists ".format(path_root)) + Log.debug("File {0} was already moved or couldn't be moved".format(src)) return False except Exception as e: if str(e) in "Garbage": diff --git a/autosubmit/platforms/platform.py b/autosubmit/platforms/platform.py index 5c61eb93d..e8f9dd723 100644 --- a/autosubmit/platforms/platform.py +++ b/autosubmit/platforms/platform.py @@ -151,7 +151,7 @@ class Platform(object): """ raise NotImplementedError - def move_file(self, src, dest): + def move_file(self, src, dest, must_exists = False, path_root = None): """ Moves a file on the platform :param src: source name -- GitLab From b5ef9c339138e753cd15c064905b427269b4c7cd Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 2 Jul 2024 16:41:25 +0200 Subject: [PATCH 2/8] Fixed migrate commandm change mv for rsync --- autosubmit/autosubmit.py | 56 ++--------------------- autosubmit/helpers/utils.py | 51 ++++++++++++++++++++- autosubmit/job/job.py | 1 - autosubmit/platforms/locplatform.py | 1 + autosubmit/platforms/paramiko_platform.py | 14 +++--- test/unit/test_autosubmit_config.py | 2 +- test/unit/test_machinefiles_wrapper.py | 2 +- 7 files changed, 64 insertions(+), 63 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 1d27a7d73..6f3cf46dc 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with Autosubmit. If not, see . from __future__ import print_function - +from helpers.utils import as_rsync import requests import threading import traceback @@ -2967,6 +2967,8 @@ class Autosubmit: return True + + @staticmethod def migrate(experiment_id, offer, pickup, only_remote): """ @@ -3235,55 +3237,9 @@ class Autosubmit: "Copying remote files/dirs on {0}", platform) Log.info("Copying from {0} to {1}", os.path.join( p.temp_dir, experiment_id), p.root_dir) - finished = False - limit = 150 - rsync_retries = 0 try: - # Avoid infinite loop unrealistic upper limit, only for rsync failure - while not finished and rsync_retries < limit: - finished = False - pipeline_broke = False - Log.info( - "Rsync launched {0} times. Can take up to 150 retrials or until all data is transfered".format( - rsync_retries + 1)) - try: - p.send_command( - "rsync --timeout=3600 --bwlimit=20000 -aq --remove-source-files " + os.path.join( - p.temp_dir, experiment_id) + " " + p.root_dir[:-5]) - except BaseException as e: - Log.debug("{0}".format(str(e))) - rsync_retries += 1 - try: - if p.get_ssh_output_err() == "": - finished = True - elif p.get_ssh_output_err().lower().find("no such file or directory") == -1: - finished = True - else: - finished = False - except: - finished = False - pipeline_broke = True - if not pipeline_broke: - if p.get_ssh_output_err().lower().find("no such file or directory") == -1: - finished = True - elif p.get_ssh_output_err().lower().find( - "warning: rsync") != -1 or p.get_ssh_output_err().lower().find( - "closed") != -1 or p.get_ssh_output_err().lower().find( - "broken pipe") != -1 or p.get_ssh_output_err().lower().find( - "directory has vanished") != -1: - rsync_retries += 1 - finished = False - elif p.get_ssh_output_err() == "": - finished = True - else: - error = True - finished = False - break - p.send_command( - "find {0} -depth -type d -empty -delete".format( - os.path.join(p.temp_dir, experiment_id))) - Log.result( - "Empty dirs on {0} have been successfully deleted".format(p.temp_dir)) + finished = as_rsync(p, os.path.join( + p.temp_dir, experiment_id), p.root_dir[:-5]) if finished: p.send_command("chmod 755 -R " + p.root_dir) Log.result( @@ -3296,8 +3252,6 @@ class Autosubmit: Log.printlog("The files/dirs on {0} cannot be copied to {1}.".format( os.path.join(p.temp_dir, experiment_id), p.root_dir), 6012) error = True - break - except IOError as e: raise AutosubmitError( "I/O Issues", 6016, e.message) diff --git a/autosubmit/helpers/utils.py b/autosubmit/helpers/utils.py index 0ce27ab8a..8b62bc9a8 100644 --- a/autosubmit/helpers/utils.py +++ b/autosubmit/helpers/utils.py @@ -27,4 +27,53 @@ def check_experiment_ownership(expid, basic_config, raise_error=False, logger=No is_eadmin = False if not is_owner and raise_error: raise AutosubmitCritical("You don't own the experiment {0}.".format(expid), 7012) - return is_owner, is_eadmin, current_owner_name \ No newline at end of file + return is_owner, is_eadmin, current_owner_name + + +def as_rsync(p, src, dest): + finished = False + limit = 150 + rsync_retries = 0 + # Avoid infinite loop unrealistic upper limit, only for rsync failure + while not finished and rsync_retries < limit: + finished = False + pipeline_broke = False + Log.info( + "Rsync launched {0} times. Can take up to 150 retrials or until all data is transfered".format( + rsync_retries + 1)) + try: + p.send_command( + "rsync --timeout=3600 --bwlimit=20000 -aqz --remove-source-files " + src + " " + dest) + except BaseException as e: + Log.debug("{0}".format(str(e))) + rsync_retries += 1 + try: + if p.get_ssh_output_err() == "": + finished = True + elif p.get_ssh_output_err().lower().find("no such file or directory") == -1: + finished = True + else: + finished = False + except: + finished = False + pipeline_broke = True + if not pipeline_broke: + if p.get_ssh_output_err().lower().find("no such file or directory") == -1: + finished = True + elif p.get_ssh_output_err().lower().find( + "warning: rsync") != -1 or p.get_ssh_output_err().lower().find( + "closed") != -1 or p.get_ssh_output_err().lower().find( + "broken pipe") != -1 or p.get_ssh_output_err().lower().find( + "directory has vanished") != -1: + rsync_retries += 1 + finished = False + elif p.get_ssh_output_err() == "": + finished = True + else: + error = True + finished = False + break + p.send_command("find {0} -depth -type d -empty -delete".format(src)) + Log.result( + "Empty dirs on {0} have been successfully deleted".format(src)) + return finished \ No newline at end of file diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index b451e321c..ab2e566d3 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1112,7 +1112,6 @@ class Job(object): self.hyperthreading = str(as_conf.get_hyperthreading(self.section)).lower() if self.hyperthreading is 'none': self.hyperthreading = str(job_platform.hyperthreading).lower() - if job_platform.processors_per_node is not None and int(self.tasks) > int(job_platform.processors_per_node): self.tasks = job_platform.processors_per_node self.tasks = str(self.tasks) diff --git a/autosubmit/platforms/locplatform.py b/autosubmit/platforms/locplatform.py index 20c1b7b20..d6b043283 100644 --- a/autosubmit/platforms/locplatform.py +++ b/autosubmit/platforms/locplatform.py @@ -206,6 +206,7 @@ class LocalPlatform(ParamikoPlatform): :param dest: destination name :param must_exist: ignore if file exist or not :type dest: str + """ try: path_root = self.get_files_path() diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index fd65af9bb..e3793ccad 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -18,7 +18,7 @@ from paramiko.ssh_exception import (SSHException, BadAuthenticationType, ChannelException, ProxyCommandFailure) import Xlib.support.connect as xlib_connect from threading import Thread - +from autosubmit.helpers.utils import as_rsync class ParamikoPlatform(Platform): @@ -388,6 +388,8 @@ class ParamikoPlatform(Platform): :param dest: destination name :param must_exist: ignore if file exist or not :type dest: str + :param path_root: root path + :type path_root: str """ if path_root is None: path_root = self.get_files_path() @@ -402,13 +404,9 @@ class ParamikoPlatform(Platform): raise AutosubmitError('File {0} does not exists, something went wrong with the platform'.format(os.path.join(path_root,src)), 6004, e.message) if e.message.lower() in "failure": try: - Log.warning("FTP Channel did not work, trying with bash command") - bash_command = "mv {0} {1}".format(src, dest) - self.send_command(bash_command) - if "file exists" in self.get_ssh_output_err().lower() or "move failed" in self.get_ssh_output_err().lower(): - Log.warning("Bash: File {0} was already moved or couldn't be moved".format(src)) - else: - Log.result("Bash: Move command was successful") + Log.warning("FTP Channel did not work due an inter-device operation... Rsync will be used") + finished = as_rsync(self, src, dest) + if finished and not self._ftpChannel.stat(src) and self._ftpChannel.stat(dest): return True except BaseException as e: raise AutosubmitError('File {0} does not exists, something went wrong with the platform'.format( diff --git a/test/unit/test_autosubmit_config.py b/test/unit/test_autosubmit_config.py index 00e624406..dc2beb171 100644 --- a/test/unit/test_autosubmit_config.py +++ b/test/unit/test_autosubmit_config.py @@ -119,7 +119,7 @@ class TestAutosubmitConfig(TestCase): def test_get_tasks(self): # arrange expected_value = '99999' - default_value = '0' + default_value = '1' config, parser_mock = self._arrange_config(expected_value) # act returned_value = config.get_tasks(self.section) diff --git a/test/unit/test_machinefiles_wrapper.py b/test/unit/test_machinefiles_wrapper.py index b3c91cebc..0f1a1ac5d 100644 --- a/test/unit/test_machinefiles_wrapper.py +++ b/test/unit/test_machinefiles_wrapper.py @@ -37,7 +37,7 @@ class TestMachinefiles(TestCase): machinefiles_dict[job] = machines """).format(nodes, cores_list, self.job_scripts, wrapper_builder._indent(machinefiles_code, 4)) - exec (script, result) + exec(script, result) machinefiles_dict = result["machinefiles_dict"] all_machines = list() -- GitLab From 50cccbcade6846191703795d31ba1dfd0f6ee102 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 3 Jul 2024 08:17:36 +0200 Subject: [PATCH 3/8] update version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7e32265b8..6e0567827 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.15.18 \ No newline at end of file +3.15.19 \ No newline at end of file -- GitLab From a3a90bbfe138f2fa718b81865b60cc398b7daa39 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 3 Jul 2024 09:16:50 +0200 Subject: [PATCH 4/8] Fix pipeline --- autosubmit/platforms/wrappers/wrapper_builder.py | 8 ++++++-- test/unit/test_job.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/autosubmit/platforms/wrappers/wrapper_builder.py b/autosubmit/platforms/wrappers/wrapper_builder.py index 3e8c67bf2..98db510a3 100644 --- a/autosubmit/platforms/wrappers/wrapper_builder.py +++ b/autosubmit/platforms/wrappers/wrapper_builder.py @@ -244,8 +244,10 @@ processors_per_node = int(jobs_resources['PROCESSORS_PER_NODE']) if node: machines += node +"_NEWLINE_" cores -= 1 + else: + break # Break the loop if cores is still greater than 0 - if cores > 0: + else: break for rest in range(processors_per_node-tasks): if len(all_cores) > 0: @@ -760,8 +762,10 @@ processors_per_node = int(jobs_resources['PROCESSORS_PER_NODE']) if node: machines += node +"_NEWLINE_" cores -= 1 + else: + break # Break the loop if cores is still greater than 0 - if cores > 0: + else: break for rest in range(processors_per_node-tasks): if len(all_cores) > 0: diff --git a/test/unit/test_job.py b/test/unit/test_job.py index 59e1f51fc..93a6076a6 100644 --- a/test/unit/test_job.py +++ b/test/unit/test_job.py @@ -292,10 +292,10 @@ class TestJob(TestCase): self.job.parameters['PROJECT_TYPE'] = "none" dummy_serial_platform = Mock() dummy_serial_platform.name = 'serial' - dummy_platform = Mock() + dummy_platform = MagicMock() dummy_platform.serial_platform = dummy_serial_platform dummy_platform.custom_directives = '["whatever"]' - + dummy_platform.proccesors_per_node = MagicMock(return_value=None) self.job._platform = dummy_platform # Act -- GitLab From eac7e97afa86ce82722b31eccff06fb8d412308d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 4 Jul 2024 13:24:43 +0200 Subject: [PATCH 5/8] Fix migrate --- autosubmit/autosubmit.py | 16 ++++++++++------ autosubmit/platforms/paramiko_platform.py | 14 ++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 6f3cf46dc..187f45de7 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3082,11 +3082,11 @@ class Autosubmit: # command = "find " + p.root_dir + " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n'" Log.info( "Converting the absolute symlinks into relatives on platform {0} ", platform) - command = "find " + p.root_dir + \ + link_finder = "find " + p.root_dir + \ " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n' " try: - p.send_command(command, True) - if p.get_ssh_output().startswith("var="): + p.send_command(link_finder, True) + while p.get_ssh_output().startswith("var="): convertLinkPath = os.path.join( BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, 'convertLink.sh') @@ -3098,6 +3098,7 @@ class Autosubmit: command = "chmod +x " + convertLinkPathRemote + " && " + \ convertLinkPathRemote + " && rm " + convertLinkPathRemote p.send_command(command, True) + p.send_command(link_finder, True) else: Log.result("No links found in {0} for [{1}] ".format( p.root_dir, platform)) @@ -3114,9 +3115,12 @@ class Autosubmit: Log.info( "Moving remote files/dirs on {0}", platform) p.send_command("chmod 777 -R " + p.root_dir) - if not p.move_file(p.root_dir, os.path.join(p.temp_dir, experiment_id), False, path_root=""): - Log.result("No data found in {0} for [{1}]\n".format( - p.root_dir, platform)) + if not p.move_file(p.root_dir, os.path.join(p.temp_dir,experiment_id), False, path_root=""): + if not as_rsync(p,p.root_dir, p.temp_dir): + error = True + break + Log.result("Data on {0} has been successfully moved".format(p.root_dir)) + except IOError as e: Log.printlog("The files/dirs on {0} cannot be moved to {1}.".format(p.root_dir, os.path.join(p.temp_dir, diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index e3793ccad..3bb4b8c49 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -380,6 +380,7 @@ class ParamikoPlatform(Platform): raise AutosubmitCritical( "Wrong User or invalid .ssh/config. Or invalid user in platform.conf or public key not set ", 7051, e.message) + def move_file(self, src, dest, must_exist=False, path_root=None): """ Moves a file on the platform (includes .err and .out) @@ -396,21 +397,14 @@ class ParamikoPlatform(Platform): try: src = os.path.join(path_root, src) dest = os.path.join(path_root, dest) - self._ftpChannel.rename(src,dest) + self._ftpChannel.rename(src, dest) return True - except IOError as e: if str(e) in "Garbage": raise AutosubmitError('File {0} does not exists, something went wrong with the platform'.format(os.path.join(path_root,src)), 6004, e.message) if e.message.lower() in "failure": - try: - Log.warning("FTP Channel did not work due an inter-device operation... Rsync will be used") - finished = as_rsync(self, src, dest) - if finished and not self._ftpChannel.stat(src) and self._ftpChannel.stat(dest): - return True - except BaseException as e: - raise AutosubmitError('File {0} does not exists, something went wrong with the platform'.format( - os.path.join(path_root, src)), 6004, e.message) + Log.warning("FTP Channel did not work due an inter-device operation... Rsync will be used") + return False if must_exist: raise AutosubmitError("File {0} does not exists".format( os.path.join(path_root,src)), 6004, e.message) -- GitLab From 92a1bf7102a0ac39062be763bbef2255e518420e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 4 Jul 2024 13:27:39 +0200 Subject: [PATCH 6/8] Fix migrate --- autosubmit/autosubmit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 187f45de7..d59ea6654 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3117,8 +3117,11 @@ class Autosubmit: p.send_command("chmod 777 -R " + p.root_dir) if not p.move_file(p.root_dir, os.path.join(p.temp_dir,experiment_id), False, path_root=""): if not as_rsync(p,p.root_dir, p.temp_dir): - error = True - break + Log.printlog("The files/dirs on {0} cannot be moved to {1}.".format(p.root_dir, + os.path.join( + p.temp_dir, + experiment_id), + 6012)) Log.result("Data on {0} has been successfully moved".format(p.root_dir)) except IOError as e: -- GitLab From bd3cb5b46552787cffb6761153e70ce4d5ccdb16 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 4 Jul 2024 16:10:00 +0200 Subject: [PATCH 7/8] retriasl added --- autosubmit/autosubmit.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index d59ea6654..c70153662 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3086,12 +3086,14 @@ class Autosubmit: " -type l -lname \'/*\' -printf 'var=\"$(realpath -s --relative-to=\"%p\" \"$(readlink \"%p\")\")\" && var=${var:3} && ln -sf $var \"%p\" \\n' " try: p.send_command(link_finder, True) - while p.get_ssh_output().startswith("var="): + retrials = 2 + while p.get_ssh_output().startswith("var=") and retrials > 0: convertLinkPath = os.path.join( BasicConfig.LOCAL_ROOT_DIR, experiment_id, BasicConfig.LOCAL_TMP_DIR, 'convertLink.sh') with open(convertLinkPath, 'w') as convertLinkFile: convertLinkFile.write(p.get_ssh_output()) + Log.debug("Link\n{0}", p.get_ssh_output()) p.send_file("convertLink.sh") convertLinkPathRemote = os.path.join( p.remote_log_dir, "convertLink.sh") @@ -3102,6 +3104,7 @@ class Autosubmit: else: Log.result("No links found in {0} for [{1}] ".format( p.root_dir, platform)) + retrials -= 1 except IOError: Log.debug( -- GitLab From 9283c48dbe56f92e44c1aa65ced7ca5a3a002e61 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 4 Jul 2024 16:10:47 +0200 Subject: [PATCH 8/8] retriasl added --- autosubmit/autosubmit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index c70153662..958303eaa 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -3093,7 +3093,10 @@ class Autosubmit: 'convertLink.sh') with open(convertLinkPath, 'w') as convertLinkFile: convertLinkFile.write(p.get_ssh_output()) - Log.debug("Link\n{0}", p.get_ssh_output()) + try: + Log.debug("Link\n{0}", p.get_ssh_output()) + except: + pass p.send_file("convertLink.sh") convertLinkPathRemote = os.path.join( p.remote_log_dir, "convertLink.sh") @@ -3101,10 +3104,10 @@ class Autosubmit: convertLinkPathRemote + " && rm " + convertLinkPathRemote p.send_command(command, True) p.send_command(link_finder, True) + retrials = retrials - 1 else: Log.result("No links found in {0} for [{1}] ".format( p.root_dir, platform)) - retrials -= 1 except IOError: Log.debug( -- GitLab