From 3545691d45e1f70cb6747d6bd61bd5cf8eaebfea Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 11:24:34 +0200 Subject: [PATCH 1/4] fixes 949 --- autosubmit/job/job_packages.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index ed9544028..bbb17adbc 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -18,7 +18,7 @@ # along with Autosubmit. If not, see . - +import json import os import random import time @@ -417,6 +417,23 @@ class JobPackageThread(JobPackageBase): else: self.exclusive = jobs[0].exclusive wr_custom_directives = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section,{}).get("CUSTOM_DIRECTIVES",[]) + # parse custom_directives + if type(wr_custom_directives) is list: + wr_custom_directives = json.dumps(wr_custom_directives) + wr_custom_directives = wr_custom_directives.replace("\'", "\"").strip("[]").strip(", ") + if wr_custom_directives == '': + if jobs[0].custom_directives is None: + jobs[0].custom_directives = '' + wr_custom_directives = jobs[0].custom_directives.replace("\'", "\"").strip("[]").strip(", ") + if wr_custom_directives != '': + if wr_custom_directives[0] != "\"": + wr_custom_directives = "\""+wr_custom_directives + if wr_custom_directives[-1] != "\"": + wr_custom_directives = wr_custom_directives+"\"" + wr_custom_directives = "[" + wr_custom_directives + "]" + wr_custom_directives = json.loads(wr_custom_directives) + else: + wr_custom_directives = [] if len(str(wr_custom_directives)) > 0: self.custom_directives = wr_custom_directives else: -- GitLab From 50b0f74c1b1a063d15f09347b5434be135630418 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 11:46:33 +0200 Subject: [PATCH 2/4] fixed test --- autosubmit/job/job_packages.py | 5 ++++- test/unit/test_job_package.py | 26 +++++++++----------------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index bbb17adbc..1dd1efc16 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -424,7 +424,10 @@ class JobPackageThread(JobPackageBase): if wr_custom_directives == '': if jobs[0].custom_directives is None: jobs[0].custom_directives = '' - wr_custom_directives = jobs[0].custom_directives.replace("\'", "\"").strip("[]").strip(", ") + wr_custom_directives = jobs[0].custom_directives + if type(wr_custom_directives) is list: + wr_custom_directives = json.dumps(wr_custom_directives) + wr_custom_directives = wr_custom_directives.replace("\'", "\"").strip("[]").strip(", ") if wr_custom_directives != '': if wr_custom_directives[0] != "\"": wr_custom_directives = "\""+wr_custom_directives diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index 81a650dad..091269302 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -1,29 +1,19 @@ from unittest import TestCase -import os import inspect -from copy import deepcopy -from mock import Mock,MagicMock, mock_open , call +import tempfile +from mock import MagicMock from mock import patch -from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical from autosubmit.job.job import Job from autosubmit.job.job_common import Status - -import shutil -import tempfile - -from unittest import TestCase -from mock import MagicMock -from autosubmit.job.job_packager import JobPackager from autosubmit.job.job_list import JobList -from autosubmit.job.job_dict import DicJobs -from autosubmit.job.job_utils import Dependency -from autosubmitconfigparser.config.yamlparser import YAMLParserFactory from autosubmit.job.job_list_persistence import JobListPersistenceDb -from random import randrange -from collections import OrderedDict +from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + + class FakeBasicConfig: def __init__(self): pass @@ -80,6 +70,7 @@ class TestJobPackage(TestCase): self.jobs[0].custom_directives = "dummy_directives" self.jobs[0].processors = "9" self.jobs[0]._processors = "9" + self.jobs[0]._platform = self.platform self.jobs[0].retrials = 0 self.jobs[1].wallclock = "00:00" self.jobs[1].threads = "" @@ -90,6 +81,7 @@ class TestJobPackage(TestCase): self.jobs[1].custom_directives = "dummy_directives2" self.jobs[1].processors = "9" self.jobs[1]._processors = "9" + self.jobs[1]._platform = self.platform self.wrapper_type = options.get('TYPE', 'vertical') @@ -153,7 +145,7 @@ class TestJobPackage(TestCase): self.assertEqual(self.job_package_wrapper._wrapper_data["PARTITION"], "bsc32") self.assertEqual(self.job_package_wrapper._wrapper_data["THREADS"], "30") self.assertEqual(self.job_package_wrapper._wrapper_data["TASKS"], "40") - self.assertEqual(self.job_package_wrapper._wrapper_data["CUSTOM_DIRECTIVES"], "['#SBATCH --mem=1000']") + self.assertEqual(self.job_package_wrapper._wrapper_data["CUSTOM_DIRECTIVES"], ['#SBATCH --mem=1000']) -- GitLab From d4e241656b9c60c87bf3b12d8ad205c6aea218e4 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 11:24:34 +0200 Subject: [PATCH 3/4] fixes 949 --- autosubmit/job/job_packages.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index 6fe699275..db7cc2bed 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -18,7 +18,7 @@ # along with Autosubmit. If not, see . - +import json import os import random import time @@ -422,6 +422,23 @@ class JobPackageThread(JobPackageBase): else: self.exclusive = jobs[0].exclusive wr_custom_directives = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section,{}).get("CUSTOM_DIRECTIVES",[]) + # parse custom_directives + if type(wr_custom_directives) is list: + wr_custom_directives = json.dumps(wr_custom_directives) + wr_custom_directives = wr_custom_directives.replace("\'", "\"").strip("[]").strip(", ") + if wr_custom_directives == '': + if jobs[0].custom_directives is None: + jobs[0].custom_directives = '' + wr_custom_directives = jobs[0].custom_directives.replace("\'", "\"").strip("[]").strip(", ") + if wr_custom_directives != '': + if wr_custom_directives[0] != "\"": + wr_custom_directives = "\""+wr_custom_directives + if wr_custom_directives[-1] != "\"": + wr_custom_directives = wr_custom_directives+"\"" + wr_custom_directives = "[" + wr_custom_directives + "]" + wr_custom_directives = json.loads(wr_custom_directives) + else: + wr_custom_directives = [] if len(str(wr_custom_directives)) > 0: self.custom_directives = wr_custom_directives else: -- GitLab From fbd4ed41195c8a4e6083751d521c641886e06865 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 27 Jul 2023 11:46:33 +0200 Subject: [PATCH 4/4] fixed test --- autosubmit/job/job_packages.py | 5 ++++- test/unit/test_job_package.py | 25 +++++++++---------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/autosubmit/job/job_packages.py b/autosubmit/job/job_packages.py index db7cc2bed..4090c6d84 100644 --- a/autosubmit/job/job_packages.py +++ b/autosubmit/job/job_packages.py @@ -429,7 +429,10 @@ class JobPackageThread(JobPackageBase): if wr_custom_directives == '': if jobs[0].custom_directives is None: jobs[0].custom_directives = '' - wr_custom_directives = jobs[0].custom_directives.replace("\'", "\"").strip("[]").strip(", ") + wr_custom_directives = jobs[0].custom_directives + if type(wr_custom_directives) is list: + wr_custom_directives = json.dumps(wr_custom_directives) + wr_custom_directives = wr_custom_directives.replace("\'", "\"").strip("[]").strip(", ") if wr_custom_directives != '': if wr_custom_directives[0] != "\"": wr_custom_directives = "\""+wr_custom_directives diff --git a/test/unit/test_job_package.py b/test/unit/test_job_package.py index 884c307d8..c1acdf4ca 100644 --- a/test/unit/test_job_package.py +++ b/test/unit/test_job_package.py @@ -3,28 +3,19 @@ from unittest import TestCase import os from pathlib import Path import inspect -from copy import deepcopy -from mock import Mock,MagicMock, mock_open , call +import tempfile +from mock import MagicMock from mock import patch -from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical from autosubmit.job.job import Job from autosubmit.job.job_common import Status - -import shutil -import tempfile - -from unittest import TestCase -from mock import MagicMock -from autosubmit.job.job_packager import JobPackager from autosubmit.job.job_list import JobList -from autosubmit.job.job_dict import DicJobs -from autosubmit.job.job_utils import Dependency -from autosubmitconfigparser.config.yamlparser import YAMLParserFactory from autosubmit.job.job_list_persistence import JobListPersistenceDb -from random import randrange -from collections import OrderedDict +from autosubmit.job.job_packages import JobPackageSimple, JobPackageVertical from autosubmitconfigparser.config.configcommon import AutosubmitConfig +from autosubmitconfigparser.config.yamlparser import YAMLParserFactory + + class FakeBasicConfig: def __init__(self): pass @@ -82,6 +73,7 @@ class TestJobPackage(TestCase): self.jobs[0].custom_directives = "dummy_directives" self.jobs[0].processors = "9" self.jobs[0]._processors = "9" + self.jobs[0]._platform = self.platform self.jobs[0].retrials = 0 self.jobs[1].wallclock = "00:00" self.jobs[1].threads = "" @@ -92,6 +84,7 @@ class TestJobPackage(TestCase): self.jobs[1].custom_directives = "dummy_directives2" self.jobs[1].processors = "9" self.jobs[1]._processors = "9" + self.jobs[1]._platform = self.platform self.wrapper_type = options.get('TYPE', 'vertical') @@ -155,7 +148,7 @@ class TestJobPackage(TestCase): self.assertEqual(self.job_package_wrapper._wrapper_data["PARTITION"], "bsc32") self.assertEqual(self.job_package_wrapper._wrapper_data["THREADS"], "30") self.assertEqual(self.job_package_wrapper._wrapper_data["TASKS"], "40") - self.assertEqual(self.job_package_wrapper._wrapper_data["CUSTOM_DIRECTIVES"], "['#SBATCH --mem=1000']") + self.assertEqual(self.job_package_wrapper._wrapper_data["CUSTOM_DIRECTIVES"], ['#SBATCH --mem=1000']) -- GitLab