#!/usr/bin/expect
############################################################################
# Purpose: Test of BLUEGENE SLURM functionality
#          Test sacct functionality and accuracy.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2006-2007 The Regents of the University of California.
# Copyright (C) 2008 Lawrence Livermore National Security.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Danny Auble <da@llnl.gov>
# LLNL-CODE-402394.
# 
# This file is part of SLURM, a resource management program.
# For details, see <http://www.llnl.gov/linux/slurm/>.
#  
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
# 
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# 
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id     "8.6"
set exit_code   0
set file_in     "test$test_id.input"
set job_id      0
set matches     0
set 32node_block_cnt 16
set 128node_block_cnt 8
set 512node_block_cnt 8
set 1knode_block_cnt  8
set 4knode_block_cnt  8
set 8knode_block_cnt  8
set 16knode_block_cnt 8
set 32knode_block_cnt 8

# job paramters
set sleep_time  5

print_header $test_id

if {[test_bluegene] == 0} {
	send_user "\nWARNING: This test is only compatable with bluegene systems\n"
	exit $exit_code
}

make_bash_script $file_in "$bin_sleep $sleep_time"
set timeout [expr $max_job_delay + $sleep_time]

# make a bunch of blocks of the specified size (node_cnt)
proc run_batch_jobs { node_cnt job_cnt file_in } {
	global sbatch srun number kill_srun
	set start_cnt 0	
	for {set inx 0} {$inx < $job_cnt} {incr inx} {
		set sbatch_pid [spawn $sbatch --output=/dev/null -t5 -N$node_cnt-$node_cnt $file_in]
		expect {
			-re "Node count specification invalid" {
				send_user "This error was expected, no worries\n"
				return -1
			}
			-re "More processors requested than permitted" {
				send_user "This error was expected, no worries\n"
				return -1
			}
			-re "Requested node configuration is not available" {
				send_user "This error was expected, no worries\n"
				return -1
			}
			-re "Submitted batch job ($number)" {
				incr start_cnt
				exp_continue
			}
			-re "Unable to contact" {
				send_user "\nFAILURE: slurm appears to be down\n"
				exp_continue
			}
			timeout {
				send_user "\nFAILURE: sbatch not responding\n"
				slow_kill $sbatch_pid
				return 0
			}
			eof {
				wait
			}
		}
	}
	if { $start_cnt != $job_cnt } {
		return 0
	} else {
		return 1
	}	
}

# Wait up to 900 seconds for all jobs to terminate
# Return 0 if all jobs done, remainin job count otherwise
proc wait_for_all_jobs { job_name } {
	global scancel squeue bin_sleep file_in

	set last_matches 0
	send_user "Waiting for all jobs to terminate\n"
	for {set inx 0} {$inx < 600} {incr inx} {
		set matches 0
		log_user 0
		spawn $squeue -o %j
		expect {
			-re "$job_name" {
				incr matches
				exp_continue
			}
			-re "error" {
				set matches -1
			}
			timeout {
				send_user "No response from squeue\n"
				set matches -1
			}
			eof {
				wait
			}
		}
		log_user 1
		if {$matches == 0} { 
			send_user "All jobs complete\n"
			break
		}
		if {$matches > 0} {
			send_user "  $matches jobs remaining\n"
#			Moab can slow throughput down a lot, 
#			so don't return here
#			if {$matches == $last_matches} {
#				send_user "Running jobs hung\n"
#				break
#			}
#			set last_matches $matches
			exec $bin_sleep 15
		}
	}
	exec $scancel -n $file_in
	return $matches
}

#
# Run the jobs here. We can be reasonably sure of running 
# 512 cnode jobs (one midplane). Other sizes may not be supported
# due to architecture (I/O node count) and/or slurm configuration
#
if {[run_batch_jobs 32 $32node_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 32 cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 128 $128node_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 128 cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 512 $512node_block_cnt $file_in ] != 1} {
	send_user "\nFAILURE: 512 cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 1k $1knode_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 1k cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 4k $4knode_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 4k cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 8k $8knode_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 8k cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 16k $16knode_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 16k cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 32k $32knode_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 32k cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 1k $1knode_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 1k cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 512 $512node_block_cnt $file_in ] != 1} {
	send_user "\nFAILURE: 512 cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 128 $128node_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 128 cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 32 $32node_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 32 cnodes can't be created\n"
	set exit_code 1	
}

if {[run_batch_jobs 512 $512node_block_cnt $file_in ] == 0} {
	send_user "\nFAILURE: 512 cnodes can't be created\n"
	set exit_code 1	
}

if {[wait_for_all_jobs $file_in] != 0} {
	send_user "\nFAILURE: some submitted jobs failed to terminate\n"
	set exit_code 1
}

if {$exit_code == 0} {
	exec rm -f $file_in
	send_user "\nSUCCESS\n"
}
exit $exit_code
