#!/usr/bin/expect
############################################################################
# Purpose: Timing test for 2000 jobs.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Danny Auble <da@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://www.schedmd.com/slurmdocs/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
############################################################################
source ./globals

set test_id	"9.9"
set exit_code   0
set job_cnt   2000

print_header $test_id

# if {[test_front_end] != 0 || $enable_memory_leak_debug != 0} {
# 	set job_cnt 2
# }

proc _submit_jobs { job_name test_file } {
	global exit_code job_cnt sbatch srun number

	log_user 0
	for {set inx 0} {$inx < $job_cnt} {incr inx} {
		spawn -nottyinit -nottycopy $sbatch -J $job_name -o $test_file --wrap $job_name
		#spawn $srun -J $job_name -o $test_file $job_name
		expect {
# 			-re "Submitted batch job ($number)" {
# 				exp_continue
# 			}
# 			-re "Unable to contact" {
# 				send_user "\nFAILURE: slurm appears to be down\n"
# 				exp_continue
# 			}
# 			timeout {
# 				send_user "\nFAILURE: sbatch not responding\n"
# 				slow_kill $sbatch_pid
# 				set exit_code 1
# 			}
			eof {
				wait
			}
		}
	}
	log_user 1
	sleep 1
	if {[wait_for_all_jobs $job_name 0] != 0} {
		send_user "\nFAILURE: some submitted jobs failed to terminate\n"
		set exit_code 1
	}
}

set time_took [string trim [time {_submit_jobs "hostname" "/dev/null"}] " per iteration microseconds"]
set time_took_sec [expr $time_took / 1000000]
set time_per_job [expr $job_cnt / $time_took_sec]
send_user "test 1 took $time_took microseconds == $time_took_sec seconds, $time_per_job jobs a sec\n"

if { $exit_code != 0 } {
	exit $exit_code
}

send_user "\nSUCCESS\n"
exit $exit_code
