Commit 02609c13 authored by Oriol Tintó-Prims's avatar Oriol Tintó-Prims
Browse files

Added assertion for members path, a fix on writing forced variable list and a...

Added assertion for members path, a fix on writing forced variable list and a limited retrial for failed jobs.
parent c0ddacd3
......@@ -374,6 +374,8 @@ class BinarySearchJob(Job):
self.thresholds = {}
self.limit_file = limit_file
self.local_folder = local_folder
self.max_retrials = 1
self.retrials = 0
if assertion_template:
self.asserting = True
self.assertion_template = assertion_template
......@@ -434,8 +436,9 @@ class BinarySearchJob(Job):
save_variable_list(self.analysis_set, analysis_variables_filename,self.vault)
# Save list of forced variables
forced_variables_filename = join(destination_folder, "forced_variables.txt")
save_variable_list(self.forced_variables, forced_variables_filename, self.vault)
if self.forced_variables is not None:
forced_variables_filename = join(destination_folder, "forced_variables.txt")
save_variable_list(self.forced_variables, forced_variables_filename, self.vault)
# Save RMSE file.
rmse_filename = join(destination_folder, "result.pkl")
......@@ -532,9 +535,10 @@ class BinarySearchJob(Job):
def check_running(self, queue):
remote_status = self.remote_status
if remote_status == "COMPLETED":
self.retrials = 0
success = self.evaluate_success()
# Create plots
self.create_plots()
# self.create_plots()
if success:
self.qualify(queue)
else:
......@@ -542,7 +546,11 @@ class BinarySearchJob(Job):
elif remote_status == "FAILED":
# TODO: Implement limited retry
self.submit(force=True)
if self.retrials < self.max_retrials:
self.submit(force=True)
self.retrials += 1
else:
raise AssertionError("Job failed!")
# self.fail(queue)
elif remote_status == "RUNNING":
pass
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment