From 9c99c520e738025cc4cfeef0349703d22a6a240b Mon Sep 17 00:00:00 2001 From: dbeltran Date: Mon, 3 Feb 2020 14:19:48 +0100 Subject: [PATCH 1/5] Unfinished select_chunks --- autosubmit/job/job_list.py | 36 +++++++++++++++++++++++++++--------- autosubmit/job/job_utils.py | 5 +++-- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 8900f7aa2..52e4db8e8 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -208,19 +208,37 @@ class JobList: dependency_running_type = dic_jobs.get_option(section, 'RUNNING', 'once').lower() delay = int(dic_jobs.get_option(section, 'DELAY', -1)) select_chunks_opt = dic_jobs.get_option(job_section, 'SELECT_CHUNKS', None) - select_chunks = [] + selected_chunks = [] if select_chunks_opt is not None: if '*' in select_chunks_opt: - if ',' in select_chunks_opt: - sections_chunks= select_chunks_opt.split(' ') - else: - sections_chunks = select_chunks_opt.split(',') + sections_chunks = select_chunks_opt.split(' ') for section_chunk in sections_chunks: info=section_chunk.split('*') if info[0] in key: - select_chunks.append(info[1]) - - dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits, select_chunks) + for location in info[1].split('-'): + auxiliar_chunk_list = [] + location = location.strip('[').strip(']') + if ':' in location: + if len(location) == 3: + for chunk in range(int(location[0]),int(location[2])): + auxiliar_chunk_list.append(chunk) + elif len(location) == 2: + if ':' == location[0]: + for chunk in range(0, int(location[1])): + auxiliar_chunk_list.append(chunk) + elif ':' == location[1]: + for chunk in range(int(location[0]),dic_jobs._chunk_list.__len__-1): + auxiliar_chunk_list.append(chunk) + elif ',' in location: + for chunk in location.strip(','): + auxiliar_chunk_list.append(chunk) + selected_chunks.append(auxiliar_chunk_list) + if len(selected_chunks) == 2: + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,selected_chunks[0],selected_chunks[1] ) #[]select_chunks_dest,select_chunks_orig + elif len(selected_chunks) == 1: + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,selected_chunks[0],[]) #[]select_chunks_dest,select_chunks_orig + else: + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,[],[] ) #[]select_chunks_dest,select_chunks_orig dependencies[key] = dependency return dependencies @@ -262,7 +280,7 @@ class JobList: else: if dependency.splits is not None: parent = filter(lambda _parent: _parent.split in dependency.splits, parent) - if len(dependency.select_chunks) == 0 or str(parent.chunk) in dependency.select_chunks: + if len(dependency.select_chunks_dest) == 0 or str(parent.chunk) in dependency.select_chunks_dest: job.add_parent(parent) JobList._add_edge(graph, job, parent) diff --git a/autosubmit/job/job_utils.py b/autosubmit/job/job_utils.py index 941bd8c42..bf720b90b 100644 --- a/autosubmit/job/job_utils.py +++ b/autosubmit/job/job_utils.py @@ -44,12 +44,13 @@ class Dependency(object): """ - def __init__(self, section, distance=None, running=None, sign=None, delay=-1, splits=None, select_chunks=list()): + def __init__(self, section, distance=None, running=None, sign=None, delay=-1, splits=None, select_chunks_dest=list(),select_chunks_orig=list() ): self.section = section self.distance = distance self.running = running self.sign = sign self.delay = delay self.splits = splits - self.select_chunks = select_chunks + self.select_chunks_dest = select_chunks_dest + self.select_chunks_orig = select_chunks_orig -- GitLab From 2c9213697b74a53911453de18316c7660712e939 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Thu, 6 Feb 2020 17:04:04 +0100 Subject: [PATCH 2/5] Improved select_chunk_list now you can select multiple jobs more info on the issue related --- autosubmit/job/job_list.py | 91 +++++++++++++++++++++++-------------- autosubmit/job/job_utils.py | 10 ++-- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 52e4db8e8..364f6f31e 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -23,7 +23,7 @@ except ImportError: # noinspection PyCompatibility from ConfigParser import SafeConfigParser import json - +import re import os import pickle from time import localtime, strftime @@ -215,30 +215,33 @@ class JobList: for section_chunk in sections_chunks: info=section_chunk.split('*') if info[0] in key: - for location in info[1].split('-'): - auxiliar_chunk_list = [] - location = location.strip('[').strip(']') - if ':' in location: - if len(location) == 3: - for chunk in range(int(location[0]),int(location[2])): - auxiliar_chunk_list.append(chunk) - elif len(location) == 2: - if ':' == location[0]: - for chunk in range(0, int(location[1])): - auxiliar_chunk_list.append(chunk) - elif ':' == location[1]: - for chunk in range(int(location[0]),dic_jobs._chunk_list.__len__-1): + for relation in range(1,len(info)): + auxiliar_relation_list=[] + for location in info[relation].split('-'): + auxiliar_chunk_list = [] + location = location.strip('[').strip(']') + if ':' in location: + if len(location) == 3: + for chunk in range(int(location[0]),int(location[2])): auxiliar_chunk_list.append(chunk) - elif ',' in location: - for chunk in location.strip(','): - auxiliar_chunk_list.append(chunk) - selected_chunks.append(auxiliar_chunk_list) - if len(selected_chunks) == 2: - dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,selected_chunks[0],selected_chunks[1] ) #[]select_chunks_dest,select_chunks_orig - elif len(selected_chunks) == 1: - dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,selected_chunks[0],[]) #[]select_chunks_dest,select_chunks_orig + elif len(location) == 2: + if ':' == location[0]: + for chunk in range(0, int(location[1]+1)): + auxiliar_chunk_list.append(chunk) + elif ':' == location[1]: + for chunk in range(int(location[0]+1),dic_jobs._chunk_list.__len__-1): + auxiliar_chunk_list.append(chunk) + elif ',' in location: + for chunk in location.split(','): + auxiliar_chunk_list.append(int(chunk)) + elif re.match('^[0-9]+$',location): + auxiliar_chunk_list.append(int(location)) + auxiliar_relation_list.append(auxiliar_chunk_list) + selected_chunks.append(auxiliar_relation_list) + if len(selected_chunks) == 1: + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,selected_chunks) #[]select_chunks_dest,select_chunks_orig else: - dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,[],[] ) #[]select_chunks_dest,select_chunks_orig + dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,[]) #[]select_chunks_dest,select_chunks_orig dependencies[key] = dependency return dependencies @@ -271,19 +274,37 @@ class JobList: dependency) if skip: continue - parents_jobs=dic_jobs.get_jobs(dependency.section, date, member, chunk) - for parent in parents_jobs: - if dependency.delay == -1 or chunk > dependency.delay: - if isinstance(parent, list): - if job.split is not None: - parent = filter(lambda _parent: _parent.split == job.split, parent)[0] + if len(dependency.select_chunks_orig) > 0: # find chunk relation + relation_found = False + relation_indx = 0 + while not relation_found and relation_indx < len(dependency.select_chunks_orig): + if job.chunk in dependency.select_chunks_orig[relation_indx]: + relation_found=True + relation_indx+=1 + if not relation_found: + relation_indx = 0 + while not relation_found and relation_indx < len(dependency.select_chunks_orig): + if len(dependency.select_chunks_orig[relation_indx]) == 0: + relation_found + relation_indx+=1 + relation_indx-=1 + if len(dependency.select_chunks_orig) == 0 or job.chunk is None or relation_found : #If doesn't contain select_chunks or running isn't chunk . ... + parents_jobs=dic_jobs.get_jobs(dependency.section, date, member, chunk) + for parent in parents_jobs: + if dependency.delay == -1 or chunk > dependency.delay: + if isinstance(parent, list): + if job.split is not None: + parent = filter(lambda _parent: _parent.split == job.split, parent)[0] + else: + if dependency.splits is not None: + parent = filter(lambda _parent: _parent.split in dependency.splits, parent) + if len(dependency.select_chunks_dest) == 0 or parent.chunk is None or parent.chunk in dependency.select_chunks_dest[relation_indx]: + Log.warning("Parent:{0} actual_job:{1}", parent.name, job.name) + Log.warning("{0} is in {1}",parent.chunk,dependency.select_chunks_dest) + job.add_parent(parent) + JobList._add_edge(graph, job, parent) else: - if dependency.splits is not None: - parent = filter(lambda _parent: _parent.split in dependency.splits, parent) - if len(dependency.select_chunks_dest) == 0 or str(parent.chunk) in dependency.select_chunks_dest: - job.add_parent(parent) - JobList._add_edge(graph, job, parent) - + pass JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph) diff --git a/autosubmit/job/job_utils.py b/autosubmit/job/job_utils.py index bf720b90b..751e0892f 100644 --- a/autosubmit/job/job_utils.py +++ b/autosubmit/job/job_utils.py @@ -44,13 +44,17 @@ class Dependency(object): """ - def __init__(self, section, distance=None, running=None, sign=None, delay=-1, splits=None, select_chunks_dest=list(),select_chunks_orig=list() ): + def __init__(self, section, distance=None, running=None, sign=None, delay=-1, splits=None, select_chunks=list()): self.section = section self.distance = distance self.running = running self.sign = sign self.delay = delay self.splits = splits - self.select_chunks_dest = select_chunks_dest - self.select_chunks_orig = select_chunks_orig + self.select_chunks_dest = list() + self.select_chunks_orig = list() + for chunk_relation in select_chunks: + self.select_chunks_dest.append(chunk_relation[0]) + self.select_chunks_orig.append(chunk_relation[1]) + -- GitLab From 1710176b53b33efa4ac4e36c19b859d3bf67a849 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 7 Feb 2020 12:18:49 +0100 Subject: [PATCH 3/5] select_chunk working , but not all at same time --- autosubmit/job/job_list.py | 27 ++++++++++++++------------- autosubmit/job/job_utils.py | 7 ++++++- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 364f6f31e..977e6befa 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -222,15 +222,15 @@ class JobList: location = location.strip('[').strip(']') if ':' in location: if len(location) == 3: - for chunk in range(int(location[0]),int(location[2])): - auxiliar_chunk_list.append(chunk) + for chunk_number in range(int(location[0]),int(location[2])+1): + auxiliar_chunk_list.append(chunk_number) elif len(location) == 2: if ':' == location[0]: - for chunk in range(0, int(location[1]+1)): - auxiliar_chunk_list.append(chunk) + for chunk_number in range(0, int(location[1])+1): + auxiliar_chunk_list.append(chunk_number) elif ':' == location[1]: - for chunk in range(int(location[0]+1),dic_jobs._chunk_list.__len__-1): - auxiliar_chunk_list.append(chunk) + for chunk_number in range(int(location[0])+1,dic_jobs._chunk_list.__len__-1): + auxiliar_chunk_list.append(chunk_number) elif ',' in location: for chunk in location.split(','): auxiliar_chunk_list.append(int(chunk)) @@ -238,7 +238,7 @@ class JobList: auxiliar_chunk_list.append(int(location)) auxiliar_relation_list.append(auxiliar_chunk_list) selected_chunks.append(auxiliar_relation_list) - if len(selected_chunks) == 1: + if len(selected_chunks) >= 1: dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,selected_chunks) #[]select_chunks_dest,select_chunks_orig else: dependency = Dependency(section, distance, dependency_running_type, sign, delay, splits,[]) #[]select_chunks_dest,select_chunks_orig @@ -267,13 +267,13 @@ class JobList: graph): for key in dependencies_keys: dependency = dependencies[key] - skip, (chunk, member, date) = JobList._calculate_dependency_metadata(job.chunk, chunk_list, job.member, member_list, job.date, date_list, dependency) if skip: continue + relation_indx = -1 if len(dependency.select_chunks_orig) > 0: # find chunk relation relation_found = False relation_indx = 0 @@ -281,13 +281,15 @@ class JobList: if job.chunk in dependency.select_chunks_orig[relation_indx]: relation_found=True relation_indx+=1 + relation_indx -= 1 if not relation_found: relation_indx = 0 while not relation_found and relation_indx < len(dependency.select_chunks_orig): if len(dependency.select_chunks_orig[relation_indx]) == 0: - relation_found + relation_found=True relation_indx+=1 - relation_indx-=1 + relation_indx-=1 + if len(dependency.select_chunks_orig) == 0 or job.chunk is None or relation_found : #If doesn't contain select_chunks or running isn't chunk . ... parents_jobs=dic_jobs.get_jobs(dependency.section, date, member, chunk) for parent in parents_jobs: @@ -298,17 +300,16 @@ class JobList: else: if dependency.splits is not None: parent = filter(lambda _parent: _parent.split in dependency.splits, parent) - if len(dependency.select_chunks_dest) == 0 or parent.chunk is None or parent.chunk in dependency.select_chunks_dest[relation_indx]: + + if len(dependency.select_chunks_dest) == 0 or parent.chunk is None or (relation_indx >= 0 and (parent.chunk in dependency.select_chunks_dest[relation_indx] or len(dependency.select_chunks_dest[relation_indx]) == 0) ): Log.warning("Parent:{0} actual_job:{1}", parent.name, job.name) Log.warning("{0} is in {1}",parent.chunk,dependency.select_chunks_dest) job.add_parent(parent) JobList._add_edge(graph, job, parent) else: pass - JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph) - @staticmethod def _calculate_dependency_metadata(chunk, chunk_list, member, member_list, date, date_list, dependency): skip = False diff --git a/autosubmit/job/job_utils.py b/autosubmit/job/job_utils.py index 751e0892f..14cc0030e 100644 --- a/autosubmit/job/job_utils.py +++ b/autosubmit/job/job_utils.py @@ -55,6 +55,11 @@ class Dependency(object): self.select_chunks_orig = list() for chunk_relation in select_chunks: self.select_chunks_dest.append(chunk_relation[0]) - self.select_chunks_orig.append(chunk_relation[1]) + if len(chunk_relation) > 1: + self.select_chunks_orig.append(chunk_relation[1]) + else: + self.select_chunks_orig.append([]) + + -- GitLab From 73b4dbfddd274c854089942169404a6b0d274fbf Mon Sep 17 00:00:00 2001 From: dbeltran Date: Fri, 7 Feb 2020 13:06:26 +0100 Subject: [PATCH 4/5] Still more work to do --- autosubmit/job/job_list.py | 41 +++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 977e6befa..75fd700f9 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -29,7 +29,6 @@ import pickle from time import localtime, strftime from sys import setrecursionlimit from shutil import move - from autosubmit.job.job import Job from bscearth.utils.log import Log from autosubmit.job.job_dict import DicJobs @@ -273,24 +272,16 @@ class JobList: dependency) if skip: continue - relation_indx = -1 - if len(dependency.select_chunks_orig) > 0: # find chunk relation - relation_found = False + chunk_relations_to_add=list() + if len(dependency.select_chunks_orig) > 0: # find chunk relation relation_indx = 0 - while not relation_found and relation_indx < len(dependency.select_chunks_orig): - if job.chunk in dependency.select_chunks_orig[relation_indx]: - relation_found=True + while relation_indx < len(dependency.select_chunks_orig): + if len(dependency.select_chunks_orig[relation_indx]) == 0 or job.chunk in dependency.select_chunks_orig[relation_indx]: + chunk_relations_to_add.append(relation_indx) relation_indx+=1 relation_indx -= 1 - if not relation_found: - relation_indx = 0 - while not relation_found and relation_indx < len(dependency.select_chunks_orig): - if len(dependency.select_chunks_orig[relation_indx]) == 0: - relation_found=True - relation_indx+=1 - relation_indx-=1 - - if len(dependency.select_chunks_orig) == 0 or job.chunk is None or relation_found : #If doesn't contain select_chunks or running isn't chunk . ... + + if len(dependency.select_chunks_orig) == 0 or job.chunk is None or len(chunk_relations_to_add) > 0 : #If doesn't contain select_chunks or running isn't chunk . ... parents_jobs=dic_jobs.get_jobs(dependency.section, date, member, chunk) for parent in parents_jobs: if dependency.delay == -1 or chunk > dependency.delay: @@ -300,14 +291,18 @@ class JobList: else: if dependency.splits is not None: parent = filter(lambda _parent: _parent.split in dependency.splits, parent) - - if len(dependency.select_chunks_dest) == 0 or parent.chunk is None or (relation_indx >= 0 and (parent.chunk in dependency.select_chunks_dest[relation_indx] or len(dependency.select_chunks_dest[relation_indx]) == 0) ): - Log.warning("Parent:{0} actual_job:{1}", parent.name, job.name) - Log.warning("{0} is in {1}",parent.chunk,dependency.select_chunks_dest) - job.add_parent(parent) - JobList._add_edge(graph, job, parent) + if len(dependency.select_chunks_dest) <= 0 or parent.chunk is None: + job.add_parent(parent) + JobList._add_edge(graph, job, parent) else: - pass + visited_parents=set() + for relation_indx in chunk_relations_to_add: + if parent.chunk in dependency.select_chunks_dest[relation_indx] or len(dependency.select_chunks_dest[relation_indx]) == 0: + if parent not in visited_parents: + job.add_parent(parent) + JobList._add_edge(graph, job, parent) + visited_parents.add(parent) + JobList.handle_frequency_interval_dependencies(chunk, chunk_list, date, date_list, dic_jobs, job, member, member_list, dependency.section, graph) @staticmethod -- GitLab From b51e86cb2126f9c57298d0e5947a0c90aff65fd8 Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 11 Feb 2020 13:03:07 +0100 Subject: [PATCH 5/5] Fixed issue #479 --- autosubmit/job/job_list.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autosubmit/job/job_list.py b/autosubmit/job/job_list.py index 75fd700f9..b5d7d2ea9 100644 --- a/autosubmit/job/job_list.py +++ b/autosubmit/job/job_list.py @@ -228,7 +228,7 @@ class JobList: for chunk_number in range(0, int(location[1])+1): auxiliar_chunk_list.append(chunk_number) elif ':' == location[1]: - for chunk_number in range(int(location[0])+1,dic_jobs._chunk_list.__len__-1): + for chunk_number in range(int(location[0])+1,len(dic_jobs._chunk_list)-1): auxiliar_chunk_list.append(chunk_number) elif ',' in location: for chunk in location.split(','): @@ -276,12 +276,12 @@ class JobList: if len(dependency.select_chunks_orig) > 0: # find chunk relation relation_indx = 0 while relation_indx < len(dependency.select_chunks_orig): - if len(dependency.select_chunks_orig[relation_indx]) == 0 or job.chunk in dependency.select_chunks_orig[relation_indx]: + if len(dependency.select_chunks_orig[relation_indx]) == 0 or job.chunk in dependency.select_chunks_orig[relation_indx] or job.chunk is None: chunk_relations_to_add.append(relation_indx) relation_indx+=1 relation_indx -= 1 - if len(dependency.select_chunks_orig) == 0 or job.chunk is None or len(chunk_relations_to_add) > 0 : #If doesn't contain select_chunks or running isn't chunk . ... + if len(dependency.select_chunks_orig) <= 0 or job.chunk is None or len(chunk_relations_to_add) > 0 : #If doesn't contain select_chunks or running isn't chunk . ... parents_jobs=dic_jobs.get_jobs(dependency.section, date, member, chunk) for parent in parents_jobs: if dependency.delay == -1 or chunk > dependency.delay: -- GitLab