From 14a8172ed5aca7e70bd9def3b6ba1bb440d1964e Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 30 Jan 2024 13:38:41 +0100 Subject: [PATCH 1/6] now all regex includes _.- --- VERSION | 2 +- autosubmit/autosubmit.py | 4 ++-- autosubmit/job/job.py | 6 +++--- autosubmit/job/job_dict.py | 2 +- autosubmit/job/job_list.py | 2 +- autosubmit/platforms/paramiko_platform.py | 1 + 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/VERSION b/VERSION index 2536c7c28..b3db1e620 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.0.104 +4.0.105 diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 0d76c7d18..8a4866d8b 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4052,7 +4052,7 @@ class Autosubmit: shutil.copyfile(template_path, backup_path) template_content = open(template_path, 'r', encoding=locale.getlocale()[1]).read() # Look for %_% - variables = re.findall('%(? -1: - template_content = re.sub('(?m)^( )*(EXPID:)( )*[a-zA-Z0-9]*(\n)*', "", template_content, flags=re.I) + template_content = re.sub('(?m)^( )*(EXPID:)( )*[a-zA-Z0-9.-_]*(\n)*', "", template_content, flags=re.I) # Write final result open(template_path, "w").write(template_content) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index c16924e1f..95a4e44b1 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -1948,7 +1948,7 @@ class Job(object): #TODO regresion test for additional_file, additional_template_content in zip(self.additional_files, additional_templates): # append to a list all names don't matter the location, inside additional_template_content that starts with % and ends with % - placeholders_inside_additional_template = re.findall('%(? Date: Tue, 30 Jan 2024 14:14:01 +0100 Subject: [PATCH 2/6] changed requeriments --- requeriments.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requeriments.txt b/requeriments.txt index 11bef0c11..47fd8ccfb 100644 --- a/requeriments.txt +++ b/requeriments.txt @@ -1,7 +1,7 @@ zipp>=3.1.0 setuptools>=60.8.2 cython -autosubmitconfigparser==1.0.50 +autosubmitconfigparser==1.0.53 paramiko>=2.9.2 bcrypt>=3.2 PyNaCl>=1.5.0 -- GitLab From 34c7784d255b471bba315c54ec61753323b734d6 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 30 Jan 2024 14:35:01 +0100 Subject: [PATCH 3/6] moved - at the end --- autosubmit/autosubmit.py | 2 +- autosubmit/job/job_list.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 8a4866d8b..89f4d7bbf 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -4078,7 +4078,7 @@ class Autosubmit: # write_it # Deletes unused keys from confs if template_path.name.lower().find("autosubmit") > -1: - template_content = re.sub('(?m)^( )*(EXPID:)( )*[a-zA-Z0-9.-_]*(\n)*', "", template_content, flags=re.I) + template_content = re.sub('(?m)^( )*(EXPID:)( )*[a-zA-Z0-9._-]*(\n)*', "", template_content, flags=re.I) # Write final result open(template_path, "w").write(template_content) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 390fe7fcc..07150d964 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -763,7 +763,7 @@ class JobList(object): if element == "": continue # Get only the first alphanumeric part and [:] chars - parsed_element = re.findall(r"([\[:\]a-zA-Z0-9-._]+)", element)[0].lower() + parsed_element = re.findall(r"([\[:\]a-zA-Z0-9._-]+)", element)[0].lower() extra_data = element[len(parsed_element):] parsed_element = JobList._parse_filter_to_check(parsed_element, value_list = value_list, level_to_check = filter_type) # convert list to str -- GitLab From 3204e945e07bb399b6246b212811d8015e0d0e03 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 30 Jan 2024 14:43:05 +0100 Subject: [PATCH 4/6] added .get --- autosubmit/job/job_dict.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autosubmit/job/job_dict.py b/autosubmit/job/job_dict.py index 5116e5e67..ca98e92a0 100644 --- a/autosubmit/job/job_dict.py +++ b/autosubmit/job/job_dict.py @@ -392,7 +392,7 @@ class DicJobs: job.ec_queue = str(parameters[section].get("EC_QUEUE", "")) if job.ec_queue == "": - job.ec_queue = str(self.experiment_data["PLATFORMS"][job.platform_name].get("EC_QUEUE","hpc")) + job.ec_queue = str(self.experiment_data["PLATFORMS"].get(job.platform_name,{}).get("EC_QUEUE","hpc")) job.partition = str(parameters[section].get( "PARTITION", "")) job.check = str(parameters[section].get( "CHECK", "true")).lower() -- GitLab From 9cde0b35e444037e0ab1ff5ce7a69acbbda6a954 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 31 Jan 2024 11:22:03 +0100 Subject: [PATCH 5/6] removed submit_cmd from update_cmds --- autosubmit/platforms/ecplatform.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autosubmit/platforms/ecplatform.py b/autosubmit/platforms/ecplatform.py index 3c4110f00..95471693b 100644 --- a/autosubmit/platforms/ecplatform.py +++ b/autosubmit/platforms/ecplatform.py @@ -95,8 +95,6 @@ class EcPlatform(ParamikoPlatform): self._checkjob_cmd = "ecaccess-job-list " self._checkhost_cmd = "ecaccess-certificate-list" self._checkvalidcert_cmd = "ecaccess-gateway-connected" - self._submit_cmd = ("ecaccess-job-submit -distant -queueName " + self.ec_queue + " " + self.host + ":" + - self.remote_log_dir + "/") self._submit_command_name = "ecaccess-job-submit" self.put_cmd = "ecaccess-file-put" self.get_cmd = "ecaccess-file-get" @@ -115,7 +113,8 @@ class EcPlatform(ParamikoPlatform): def get_mkdir_cmd(self): return self.mkdir_cmd - def set_submit_cmd(self,ec_queue="hpc"): + + def set_submit_cmd(self,ec_queue): self._submit_cmd = ("ecaccess-job-submit -distant -queueName " + ec_queue + " " + self.host + ":" + self.remote_log_dir + "/") -- GitLab From 30c8e40f7e8f03f8a16c8748d2f7b4b6dfb2d61f Mon Sep 17 00:00:00 2001 From: dbeltran Date: Wed, 31 Jan 2024 13:35:38 +0100 Subject: [PATCH 6/6] added retrial, command is failing sometimes --- autosubmit/platforms/ecplatform.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/autosubmit/platforms/ecplatform.py b/autosubmit/platforms/ecplatform.py index 95471693b..12fb7443e 100644 --- a/autosubmit/platforms/ecplatform.py +++ b/autosubmit/platforms/ecplatform.py @@ -227,14 +227,21 @@ class EcPlatform(ParamikoPlatform): lang = locale.getdefaultlocale()[1] if lang is None: lang = 'UTF-8' - try: - output = subprocess.check_output(command, shell=True).decode(lang) - except subprocess.CalledProcessError as e: - if command.find("ecaccess-job-submit") != -1: - raise AutosubmitError("bad parameters. Error submitting job.") - if not ignore_log: - raise AutosubmitError('Could not execute command {0} on {1}'.format(e.cmd, self.host),7500,str(e)) - return False + retries = 3 + success = False + err_message = "" + output = "" + while not success and retries > 0: + try: + output = subprocess.check_output(command, shell=True).decode(lang) + success = True + except Exception as e: + err_message = str(e) + success = False + retries = retries - 1 + sleep(1) + if not success: + raise AutosubmitError(f'Could not execute command {command} on {self.host}',7500, str(err_message)) self._ssh_output = output return True -- GitLab