From 6a1a85d3a4b9c23ad09fd7d35da8394f4b6f75f3 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 20 Jan 2021 17:37:17 +0100 Subject: [PATCH 1/2] Implemented #607. The user can now supply a date range for the setstatus options -f --- autosubmit/autosubmit.py | 54 +++++++++++++++++++++++++++++++++------- simple_test.py | 5 ++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/autosubmit/autosubmit.py b/autosubmit/autosubmit.py index a677f8c19..24aa4d09c 100644 --- a/autosubmit/autosubmit.py +++ b/autosubmit/autosubmit.py @@ -2644,8 +2644,10 @@ class Autosubmit: limit = 100 rsync_retries = 0 try: - while not finished and rsync_retries < limit: # Avoid infinite loop unrealistic upper limit, only for rsync failure - p.send_command("rsync -ah --remove-source-files " + os.path.join(p.temp_dir, experiment_id) + " " + p.root_dir[:-5]) + # Avoid infinite loop unrealistic upper limit, only for rsync failure + while not finished and rsync_retries < limit: + p.send_command("rsync -ah --remove-source-files " + os.path.join( + p.temp_dir, experiment_id) + " " + p.root_dir[:-5]) if "no such file or directory" in p.get_ssh_output_err().lower(): finished = True elif "warning: rsync" in p.get_ssh_output_err().lower() or "connection unexpectedly closed" in p.get_ssh_output_err().lower(): @@ -2656,11 +2658,15 @@ class Autosubmit: else: error = True finished = True - raise AutosubmitError("{0}".format(p.get_ssh_output_err().lower()), 6012) + raise AutosubmitError("{0}".format( + p.get_ssh_output_err().lower()), 6012) p.send_command("chmod 755 -R " + p.root_dir) - Log.result("Files/dirs on {0} have been successfully picked up", platform) - p.send_command("find {0} -depth -type d -empty -delete".format(os.path.join(p.temp_dir, experiment_id))) - Log.result("Empty dirs on {0} have been successfully deleted".format(p.temp_dir)) + Log.result( + "Files/dirs on {0} have been successfully picked up", platform) + p.send_command( + "find {0} -depth -type d -empty -delete".format(os.path.join(p.temp_dir, experiment_id))) + Log.result( + "Empty dirs on {0} have been successfully deleted".format(p.temp_dir)) except BaseException as e: error = True @@ -4088,6 +4094,7 @@ class Autosubmit: # New feature : Change status by section, member, and chunk; freely. # Including inner validation. Trying to make it independent. + # 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16-30] ] ],SIM,SIM2,SIM3 if filter_type_chunk: validation_message = "## -ftc Validation Message ##" filter_is_correct = True @@ -4601,14 +4608,43 @@ class Autosubmit: data = [] # text = "[ 19601101 [ fc0 [1 2 3 4] fc1 [1] ] 16651101 [ fc0 [1-30 31 32] ] ]" + def parse_date(datestring): + result = [] + startindex = datestring.find('(') + endindex = datestring.find(')') + if startindex > 0 and endindex > 0: + try: + startstring = datestring[:startindex] + startrange = datestring[startindex + 1:].split('-')[0] + endrange = datestring[startindex:-1].split('-')[1] + startday = int(startrange[-2:]) + endday = int(endrange[-2:]) + + frommonth = int(startrange[:2]) + tomonth = int(endrange[:2]) + + for i in range(frommonth, tomonth + 1): + for j in range(startday, endday + 1): + result.append(startstring + "%02d" % + i + "%02d" % j) + except Exception as exp: + raise AutosubmitCritical( + "Autosubmit couldn't parse your input format. Exception: {0}".format(exp)) + + else: + result = [datestring] + return result + out = nestedExpr('[', ']').parseString(text).asList() # noinspection PyUnusedLocal for element in out[0]: if count % 2 == 0: - sd = {'sd': out[0][count], 'ms': Autosubmit._get_members( - out[0][count + 1])} - data.append(sd) + datelist = parse_date(out[0][count]) + for item in datelist: + sd = {'sd': item, 'ms': Autosubmit._get_members( + out[0][count + 1])} + data.append(sd) count += 1 else: count += 1 diff --git a/simple_test.py b/simple_test.py index df1bee2d6..94ead23dd 100644 --- a/simple_test.py +++ b/simple_test.py @@ -11,7 +11,12 @@ # from autosubmit.job.job_list import JobList # from autosubmit.database.db_jobdata import JobDataStructure # from bscearth.utils.log import Log +# from autosubmit.autosubmit import Autosubmit +# from pyparsing import nestedExpr +# text = "[ 1960(0605-1206) [ fc0 [1 2 3 4] fc1 [1] ] 16651101 [ fc0 [1-30 31 32] ] ]" +# out = Autosubmit._create_json(text) +# print(out) # def test_retrieve_energy(): # BasicConfig.read() -- GitLab From 65622499578e2ef182ac515a5988e6cdc7efeb40 Mon Sep 17 00:00:00 2001 From: Wilmer Uruchi Ticona Date: Wed, 20 Jan 2021 17:37:32 +0100 Subject: [PATCH 2/2] Added documentation --- docs/source/usage/setstatus.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/source/usage/setstatus.rst b/docs/source/usage/setstatus.rst index 4d4d0bf15..db57fbd1e 100644 --- a/docs/source/usage/setstatus.rst +++ b/docs/source/usage/setstatus.rst @@ -39,8 +39,9 @@ Options: List of types to be changed -ftc FILTER_TYPE_CHUNK --filter_type_chunk Accepts a string with the formula: "[ 19601101 [ fc0 [1 2 3 4] Any [1] ] 19651101 [ fc0 [16 30] ] ],SIM,SIM2" - Where SIM, SIM2 are section (or job types) names that also accept the keyword "Any" so the changes apply to all sections. + Where SIM, SIM2 are section (job types) names that also accept the keyword "Any" so the changes apply to all sections. Starting Date (19601101) does not accept the keyword "Any", so you must specify the starting dates to be changed. + You can also specify date ranges to apply the change to a range on dates. Member names (fc0) accept the keyword "Any", so the chunks ([1 2 3 4]) given will be updated for all members. Chunks must be in the format "[1 2 3 4]" where "1 2 3 4" represent the numbers of the chunks in the member, no range format is allowed. @@ -65,6 +66,30 @@ Examples: autosubmit setstatus cxxx -ft TRANSFER -t SUSPENDED -s autosubmit setstatus cxxx -ftc "[ 19601101 [ fc1 [1] ], SIM" -t SUSPENDED -s +Date (month) range example: +:: + + autosubmit setstatus cxxx -ftc "[ 1960(1101-1201) [ fc1 [1] ], SIM" -t SUSPENDED -s + +This example will result changing the following jobs: +:: + + cxxx_19601101_fc1_1_SIM + cxxx_19601201_fc1_1_SIM + +Date (day) range example: +:: + + autosubmit setstatus cxxx -ftc "[ 1960(1101-1105) [ fc1 [1] ], SIM" -t SUSPENDED -s + +Result: +:: + cxxx_19601101_fc1_1_SIM + cxxx_19601102_fc1_1_SIM + cxxx_19601103_fc1_1_SIM + cxxx_19601104_fc1_1_SIM + cxxx_19601105_fc1_1_SIM + This script has two mandatory arguments. The -t where you must specify the target status of the jobs you want to change to: -- GitLab