diff --git a/autosubmit/platforms/paramiko_platform.py b/autosubmit/platforms/paramiko_platform.py index 57989c19a71e8493438834d736eaeb2ae7c8f014..96b3f8c472a3a57647ff62755bf03914cdef7daf 100644 --- a/autosubmit/platforms/paramiko_platform.py +++ b/autosubmit/platforms/paramiko_platform.py @@ -140,15 +140,17 @@ class ParamikoPlatform(Platform): log_dir = os.path.join(self.tmp_path, 'LOG_{0}'.format(self.expid)) multiple_delete_previous_run = os.path.join(log_dir,"multiple_delete_previous_run.sh") - open(multiple_delete_previous_run, 'w+').write("rm -f "+filenames) - os.chmod(multiple_delete_previous_run, 0o770) - self.send_file(multiple_delete_previous_run, False) - command = os.path.join(self.get_files_path(),"multiple_delete_previous_run.sh") - - if self.send_command(command, ignore_log=True): - return self._ssh_output - else: - return "" + if os.path.exists(multiple_delete_previous_run): + open(multiple_delete_previous_run, 'w+').write("rm -f "+filenames) + os.chmod(multiple_delete_previous_run, 0o770) + self.send_file(multiple_delete_previous_run, False) + command = os.path.join(self.get_files_path(),"multiple_delete_previous_run.sh") + + if self.send_command(command, ignore_log=True): + return self._ssh_output + else: + return "" + return "" def send_file(self, filename, check=True): """ diff --git a/autosubmit/platforms/wrappers/wrapper_builder.py b/autosubmit/platforms/wrappers/wrapper_builder.py index 97490e0949d676bf0a06d3c8c11f3a60bcaef824..03eb1a29ff448499d0285b63f3915a98e6436b52 100644 --- a/autosubmit/platforms/wrappers/wrapper_builder.py +++ b/autosubmit/platforms/wrappers/wrapper_builder.py @@ -18,7 +18,7 @@ # along with Autosubmit. If not, see . import textwrap - +import math class WrapperDirector: """ @@ -58,6 +58,8 @@ class WrapperBuilder(object): self.machinefiles_indent = 0 self.exit_thread = '' + + def build_header(self): return textwrap.dedent(self.header_directive) + self.build_imports() @@ -747,8 +749,40 @@ class SrunVerticalHorizontalWrapperBuilder(SrunWrapperBuilder): declare -a scripts_list={0} declare -a scripts_index={1} """).format(str(scripts_array_vars),str(scripts_array_index), '\n'.ljust(13)) + + total_threads = float(len(self.job_scripts)) + n_threads = float(self.threads) + cpu_values = [] + for thread in range(int(n_threads)): + cpu_values.append(0x0) + horizontal_wrapper_size=int(total_threads) + thr_mask = 0x1 + srun_mask_values = [] + for job in range(horizontal_wrapper_size): + job_mask = 0x0 + cpu_values[0] = thr_mask + for thread in range(1, int(n_threads)): + thr_mask = thr_mask + thr_mask + cpu_values[thread] = thr_mask + #print "#{0} cpu-mask is {1}: ".format(thread, hex(thr_mask)) + for thr_mask in cpu_values: + job_mask = job_mask + thr_mask + srun_mask_values.append(str(hex(job_mask))) + #print "#{0} cpu-id is {1}: ".format(0, hex(thr_mask)) + thr_mask = job_mask + 0x1 + #print "{0} mask-id is {1}: ".format(job, hex(job_mask)) + mask_array = "( " + for mask in srun_mask_values: + mask_array += str("\"" + mask + "\"") + " " + mask_array += ")" + scripts_bash += textwrap.dedent(""" + declare -a job_mask_array={0} + """).format(mask_array, '\n'.ljust(13)) + return scripts_bash + + def build_srun_launcher(self, jobs_list, footer=True): srun_launcher = textwrap.dedent(""" suffix=".cmd" @@ -780,12 +814,11 @@ class SrunVerticalHorizontalWrapperBuilder(SrunWrapperBuilder): #prev_template=${{prev_horizontal_scripts[$job_index]}} prev_template=${{scripts[((job_index-1))]}} fi - echo "$as_index = $job_index*$horizontal_size+$i_list $out" completed_filename=${{prev_template%"$suffix"}} completed_filename="$completed_filename"_COMPLETED completed_path=${{PWD}}/$completed_filename if [ $job_index -eq 0 ] || [ -f "$completed_path" ]; then #If first horizontal wrapper or last wrapper is completed - srun -N1 --ntasks=1 --cpus-per-task={1} $template > $out 2> $err & + srun -N1 --ntasks=1 --cpus-per-task={1} --cpu-bind=verbose,mask_cpu:job_mask_array[$job_index] --distribution=block:block $template > $out 2> $err & job_index=$(($job_index+1)) else