#!/usr/bin/env expect
############################################################################
# Purpose: Test of SLURM functionality
#          Bluegene/Q only: Test to make sure if a nodeboard goes unavailable
#	   while another block is using it for passthrough to make sure the
#	   job using the block with passthrough fails.
#
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2011 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id	"8.9"
set exit_code	0
set file_in 	"test$test_id.input"
set job_id      0
set  geom       ""
set a_size      ""
set x_size      ""
set y_size      ""
set z_size      ""
set job_fini1   ""
set job_fini2   ""
set job_fini3   ""
set job_fini4   ""
set first_node  ""
set connection  ""
set a_flag      0
set x_flag      0
set y_flag      0
set z_flag      0
set tmp         0
set block       0

print_header $test_id


proc submit_sbatch {sbatch_perams} {
	global sbatch
	global alpha_numeric_under number
	global job_id
	global file_in
	global bin_sleep

make_bash_script $file_in "$bin_sleep 10"

	eval spawn $sbatch $sbatch_perams
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		-re "error" {
			send_user "\nFAILURE: Job was not submitted\n"
			exp_continue
			exit 1
		}
		timeout {
			send_user "\nFAILURE: sbatch is not responding\n"
			exit 1
		}
		eof {
			wait
		}
	}
}

proc check_job_state {state_perams} {
	global squeue
	global job_id
	global matches
	global last_node
	global change_subbp_stat

	set matches 0
	eval spawn $squeue $state_perams
	expect {
		-re "$job_id" {
			incr matches
			exp_continue
		}
		timeout {
			send_user "\nFAILURE:squeue is not responding\n"
			change_subbp_state $last_node "0" "free"
			exit 1
		}
		eof {
			wait
		}
	}

	if {$matches != 1} {
		send_user "\nFAILURE: Job $job_id did not fail\n"
		change_subbp_state $last_node "0" "free"
		exit 1
	}

}



#Test Starts
if {([test_bluegene] == 0) || [string compare [get_bluegene_type] "Q"]} {
	send_user "\nWARNING: This test is only compatible with bluegene systems\n"
	exit $exit_code
}

spawn $scontrol show node
expect {
	-re "NodeName=($alpha_numeric)($digit)($digit)($digit)($digit)" {
		set prefix $expect_out(1,string)
		set a_size $expect_out(2,string)
		set x_size $expect_out(3,string)
		set y_size $expect_out(4,string)
		set z_size $expect_out(5,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		exit 1
	}
	eof {
		wait
	}
}

if {$a_size>2} {
	set geom "2x1x1x1"
	set a_flag 1
} elseif {$x_size > 2} {
	set geom "1x2x1x1"
	set x_flag 1
} elseif {$y_size >2} {
	set geom "1x1x2x1"
	set y_flag 1
} elseif {$z_size >2} {
	set geom "1x1x1x2"
	set z_flag 1
} else {
	send_user "\nFAILURE: not enough dimenstions\n"
	exit 1
}



submit_sbatch "--no-requeue --geometry=$geom --conn-type=T,T,T,T --output=/dev/null --error=/dev/null $file_in"

wait_for_job $job_id "RUNNING"

spawn $scontrol show job $job_id
expect {
	-re "MidplaneList=($alpha_numeric).($alpha_numeric)x($digit)($digit)($digit)($digit)" {
		set prefix $expect_out(1,string)
		set first_node $expect_out(2,string)
		set job_fini1 $expect_out(3,string)
		set job_fini2 $expect_out(4,string)
		set job_fini3 $expect_out(5,string)
		set job_fini4 $expect_out(6,string)
		exp_continue

	}
	-re "Block_ID=($alpha_numeric)" {
		set block $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		exit 1
	}
	eof {
		wait
	}

}

if {$a_flag==1} {
	set last_node "$prefix$a_size$job_fini2$job_fini3$job_fini4"
	set last_job_node "$job_fini1$job_fini2$job_fini3$job_fini4"
} elseif {$x_flag==1} {
	set last_node "$prefix$job_fini1$x_size$job_fini3$job_fini4"
	set last_job_node "$job_fini1$job_fini2$job_fini3$job_fini4"
} elseif {$y_flag==1} {
	set last_node "$prefix$job_fini1$job_fini2$y_size$job_fini4"
	set last_job_node "$job_fini1$job_fini2$job_fini3$job_fini4"
} elseif {$z_flag==1} {
	set last_node "$prefix$job_fini1$job_fini2$job_fini3$z_size"
	set last_job_node "$job_fini1$job_fini2$job_fini3$job_fini4"
}

change_subbp_state $last_node "0" "error"

wait_for_job $job_id "DONE"

#checks if the job failed
check_job_state "--state=node_fail -o%i -j -h $job_id"

#Check if the block was destroyed
set matches 0
spawn $scontrol show block $block
expect {
	-re "not found" {
		send_user "\nBlock was not found, this is expected.\n"
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		change_subbp_state $last_node "0" "free"
	        exit 1
	}
	eof {
		wait
	}
}

if {$matches != 1} {
	send_user "\nFAILURE: Block $block is still around?  It should of been removed\n"
	change_subbp_state $last_node "0" "free"
	cancel_job $job_id
	exit 1
}

if {$a_flag==1} {
	set connection M,T,T,T
} elseif {$x_flag==1} {
	set connection T,M,T,T
} elseif {$y_flag==1} {
	set connection T,T,M,T
} elseif {$z_flag==1} {
	set connection T,T,T,M
}

set node_range "$prefix$first_node\,$prefix$last_job_node"
submit_sbatch "--no-requeue --nodelist=$node_range --geometry=$geom --conn-type=$connection --output=/dev/null --error=/dev/null $file_in"

if {[wait_for_job $job_id "RUNNING"]} {
	change_subbp_state $last_node "0" "free"
	cancel_job $job_id
	exec $bin_rm -f $file_in
	exit 1
}

#checks if the job is running
check_job_state "--state=running -o%i -j -h $job_id"

cancel_job $job_id
exec $bin_rm -f $file_in

#Sets bad block back to free
change_subbp_state $last_node "0" "free"

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
}

exit $exit_code
