From dcadfba2f1e36f8db1a43870c139025a42812b90 Mon Sep 17 00:00:00 2001 From: Luiggi Tenorio Date: Wed, 24 Apr 2024 15:58:22 +0200 Subject: [PATCH 1/4] fix joblist db schema --- autosubmit/job/job_list_persistence.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/autosubmit/job/job_list_persistence.py b/autosubmit/job/job_list_persistence.py index 791de7864..48c638db2 100644 --- a/autosubmit/job/job_list_persistence.py +++ b/autosubmit/job/job_list_persistence.py @@ -124,10 +124,22 @@ class JobListPersistenceDb(JobListPersistence): VERSION = 3 JOB_LIST_TABLE = 'job_list' - TABLE_FIELDS = ['name', 'id', 'status', 'priority', - 'section', 'date', 'member', 'chunk', - 'local_out', 'local_err', - 'remote_out', 'remote_err'] + TABLE_FIELDS = [ + "name", + "id", + "status", + "priority", + "section", + "date", + "member", + "chunk", + "split", + "local_out", + "local_err", + "remote_out", + "remote_err", + "wrapper_type", + ] def __init__(self, persistence_path, persistence_file): self.db_manager = DbManager(persistence_path, persistence_file, self.VERSION) -- GitLab From 78f9603b91e54fb51d820d899b6486c3c4a5d5c7 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 9 May 2024 09:47:27 +0200 Subject: [PATCH 2/4] Changed expid regex --- autosubmit/autosubmit.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 34106dc75..2dfa89cb2 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1194,6 +1194,20 @@ class Autosubmit: yaml.dump(yaml_data, output) @staticmethod + def replace_expid_in_default(content, new_expid): + # Find the DEFAULT section + default_section_match = re.search(r'(DEFAULT:[\s\S]*?)(?=\w+:|$)', content) + + if default_section_match: + default_section = default_section_match.group(1) + + # Replace EXPID in the DEFAULT section + new_default_section = re.sub(rf'(EXPID:).*', rf'\1 " {new_expid}"', default_section) + # Replace the old DEFAULT section + content = content.replace(default_section, new_default_section) + + return content + @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ Replace default values in as_conf files @@ -1220,9 +1234,7 @@ class Autosubmit: search = re.search('TO: .*', content, re.MULTILINE) if search is not None: content = content.replace(search.group(0), "TO: \"\"") - search = re.search('EXPID: .*', content, re.MULTILINE) - if search is not None: - content = content.replace(search.group(0),"EXPID: \""+exp_id+"\"") + Autosubmit.replace_expid_in_default(content, exp_id) search = re.search('HPCARCH: .*', content, re.MULTILINE) if search is not None: content = content.replace(search.group(0),"HPCARCH: \""+hpc+"\"") -- GitLab From 8fa0f364481ef3255bd73039103f95b723d4678d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 9 May 2024 14:19:30 +0200 Subject: [PATCH 3/4] made it more general --- autosubmit/autosubmit.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 2dfa89cb2..96ed4f348 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1194,18 +1194,20 @@ class Autosubmit: yaml.dump(yaml_data, output) @staticmethod - def replace_expid_in_default(content, new_expid): - # Find the DEFAULT section - default_section_match = re.search(r'(DEFAULT:[\s\S]*?)(?=\w+:|$)', content) - - if default_section_match: - default_section = default_section_match.group(1) - - # Replace EXPID in the DEFAULT section - new_default_section = re.sub(rf'(EXPID:).*', rf'\1 " {new_expid}"', default_section) - # Replace the old DEFAULT section - content = content.replace(default_section, new_default_section) - + def replace_parameter_inside_section(content, parameter, new_value, section): + # same but for any section any parameter, not only EXPID case insensitive + # Find the any section + if section: + section_match = re.search(rf'({section}:[\s\S]*?{parameter}:.*?)(?=\n|$)', content, re.IGNORECASE) + if section_match: + section = section_match.group(1) + # Replace parameter in the section + new_section = re.sub(rf'({parameter}:).*', rf'\1 "{new_value}"', section) + # Replace the old section + content = content.replace(section, new_section) + else: + # replace only the parameter + content = re.sub(rf'({parameter}:).*', rf'\1 "{new_value}"', content) return content @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): @@ -1234,7 +1236,7 @@ class Autosubmit: search = re.search('TO: .*', content, re.MULTILINE) if search is not None: content = content.replace(search.group(0), "TO: \"\"") - Autosubmit.replace_expid_in_default(content, exp_id) + content = Autosubmit.replace_parameter_inside_section(content, "EXPID", exp_id, "DEFAULT") search = re.search('HPCARCH: .*', content, re.MULTILINE) if search is not None: content = content.replace(search.group(0),"HPCARCH: \""+hpc+"\"") @@ -1251,9 +1253,6 @@ class Autosubmit: search = re.search('PROJECT_BRANCH: .*', content, re.MULTILINE) if search is not None: content = content.replace(search.group(0), "PROJECT_BRANCH: \""+git_branch+"\"") - - - with open(os.path.join(BasicConfig.LOCAL_ROOT_DIR, exp_id,"conf", as_conf_file), 'w') as f: f.write(content) -- GitLab From 1990dd28aa2495ca152354efe5db6cf84716c634 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 9 May 2024 15:13:10 +0200 Subject: [PATCH 4/4] made it more general --- autosubmit/autosubmit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index 96ed4f348..520b45f9a 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -1209,6 +1209,7 @@ class Autosubmit: # replace only the parameter content = re.sub(rf'({parameter}:).*', rf'\1 "{new_value}"', content) return content + @staticmethod def as_conf_default_values(exp_id,hpc="local",minimal_configuration=False,git_repo="",git_branch="main",git_as_conf=""): """ -- GitLab