############################################################################
# Purpose: Test of SLURM functionality
#          to be called from test3.11
#	   Several cases for core based reservations
#          Pluging select/cons_res needed
#
############################################################################
# Copyright (C) 2009 Lawrence Livermore National Security
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Dave Bremer <dbremer@llnl.gov>
# CODE-OCEC-09-009. All rights reserved.
#
#
# This file is part of SLURM, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

proc inc3_11_7 {} {
	global alpha_numeric_under bin_rm bin_sleep cluster_cpus cores_per_node
	global def_partition exit_code file_in number part_cores part_node
	global part_node_cnt part_node_cores res_name res_name res_nodes
	global res_thread_cnt sbatch scancel scontrol user_name wait_for_job

	send_user "\n+++++ STARTING TEST 7 (Within: inc3.11.7) +++++\n"

	# Make a reservation, just to get node size infomation
	set ret_code [create_res "StartTime=now Duration=1 Nodes=$part_node User=$user_name" 0]
	if {$ret_code != 0} {
		send_user "\n\033\[31mFAILURE: Unable to create a valid reservation (Within inc3.11.7)\033\[m\n"
		exit $ret_code
	}
	# Delete the reservation
	set ret_code [delete_res $res_name]
	if {$ret_code != 0} {
		exit $ret_code
	}

	# Now make a reservation using half the cores on that node
	# There is no way to specify a the Nodes in a reservation with CoreCnt,
	# so hope that we get a node with the same size
	set corecnt [ expr ($part_node_cores / 2) ]
	set ret_code [create_res "StartTime=now Duration=60 Nodes=$part_node CoreCnt=$corecnt User=$user_name" 0]
	if {$ret_code != 0} {
		send_user "\n\033\[31mFAILURE: Unable to create a valid reservation (Within: inc3.11.7)\033\[m\n"
		exit $ret_code
	}
	set thread_res_num [ expr $corecnt * $res_thread_cnt ]

	# Make the job script
	exec $bin_rm -f $file_in
	make_bash_script $file_in "$bin_sleep 100"

	# Sleep short time in case clocks are not synchronized
	sleep 5

	# (First test) Submit the batch job: a simple job using just 1 core inside the reservation
	set sbatch_pid [spawn $sbatch -n1 --reservation=$res_name --output=/dev/null $file_in]
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: sbatch not responding (Within: inc3.11.7)\033\[m\n"
			slow_kill $sbatch_pid
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$job_id == 0} {
		send_user "\n\033\[31mFAILURE: batch submit failure (Within: inc3.11.7)\033\[m\n"
		exit 1
	}

	if {[wait_for_job $job_id RUNNING] != 0} {
		send_user "\n\033\[31mFAILURE: Job $job_id failed to start (Within: inc3.11.7)\033\[m\n"
		set exit_code 1
	} else {
		send_user "\n\033\[32mJOB is running as expected (Within: inc3.11.7)\033\[m\n"
	}
	cancel_job $job_id

	sleep 1

	# (Second test) Submit the batch job: a job using all cores allocated by the reservation
	set sbatch_pid [spawn $sbatch -n$thread_res_num --reservation=$res_name --output=/dev/null $file_in]
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: sbatch not responding (Within: inc3.11.7)\033\[m\n"
			slow_kill $sbatch_pid
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$job_id == 0} {
		send_user "\n\033\[31mFAILURE: batch submit failure (Within: inc3.11.7)\033\[m\n"
		exit 1
	}

	# Show the job, make sure reservation tag is right
	if {[wait_for_job $job_id RUNNING] != 0} {
		send_user "\n\033\[31mFAILURE: Job $job_id failed to start (Within: inc3.11.7)\033\[m\n"
		set exit_code 1
	} else {
		send_user "\n\033\[32mJOB is running as expected (Within: inc3.11.7)\033\[m\n"
	}
	cancel_job $job_id

	sleep 1

	# (Third test) Submit the batch job: a job using all cores not allocated by the reservation
	set sbatch_pid [spawn $sbatch -n$thread_res_num --nodelist=$res_nodes --output=/dev/null $file_in]
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: sbatch not responding (Within: inc3.11.7)\033\[m\n"
			slow_kill $sbatch_pid
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$job_id == 0} {
		send_user "\n\033\[31mFAILURE: batch submit failure (Within: inc3.11.7)\033\[m\n"
		exit 1
	}

	# Show the job, make sure reservation tag is right
	if {[wait_for_job $job_id RUNNING] != 0} {
		send_user "\n\033\[31mFAILURE: Job $job_id failed to start (Within: inc3.11.7)\033\[m\n"
		set exit_code 1
	} else {
		send_user "\n\033\[32mJOB is running as expected (Within: inc3.11.7)\033\[m\n"
	}
	cancel_job $job_id

	# (Fourth test) Submit a batch job: a job using more cores than allocated by the reservation
	set thread_res_num [ expr ($thread_res_num + 1) ]
	set sbatch_pid [spawn $sbatch -n$thread_res_num --reservation=$res_name --output=/dev/null $file_in]
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: sbatch not responding (Within: inc3.11.7)\033\[m\n"
			slow_kill $sbatch_pid
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$job_id == 0} {
		send_user "\n\033\[31mFAILURE: batch submit failure (Within: inc3.11.7)\033\[m\n"
		exit 1
	}

	sleep 10
	# Show the job, make sure reservation tag is right
	spawn $scontrol show job $job_id
	expect {
		-re "Invalid job id specified" {
			send_user "\n\033\[31mFAILURE: Job $job_id not found (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
			exp_continue
		}
		-re "JobState=PENDING" {
			send_user "\n\033\[32mSUCCESS: Job $job_id is PENDING as expected (Within: inc3.11.7)\033\[m\n"
			exp_continue
		}
		-re "JobState=RUNNING" {
			send_user "\n\033\[31mFAILURE: Job $job_id is RUNNING but it should not (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: scontrol not responding (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	cancel_job $job_id

	# (Fifth test) Submit a batch job: a job specifying node in reservation and
	#              using more cores than allocated by the reservation
	set sbatch_pid [spawn $sbatch -n$thread_res_num --nodelist=$res_nodes --nodes=1 --output=/dev/null $file_in]
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: sbatch not responding (Within: inc3.11.7)\033\[m\n"
			slow_kill $sbatch_pid
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$job_id == 0} {
		send_user "\n\033\[31mFAILURE: batch submit failure (Within: inc3.11.7)\033\[m\n"
		exit 1
	}

	sleep 10
	# Show the job, make sure reservation tag is right
	spawn $scontrol show job $job_id
	expect {
		-re "Invalid job id specified" {
			send_user "\n\033\[31mFAILURE: Job $job_id not found (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
			exp_continue
		}
		-re "JobState=PENDING" {
			send_user "\n\033\[32mSUCCESS: Job $job_id is PENDING as expected (Within: inc3.11.7)\033\[m\n"
			exp_continue
		}
		-re "JobState=RUNNING" {
			send_user "\n\033\[31mFAILURE: Job $job_id is RUNNING but it should not (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: scontrol not responding (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	# Cancelling the job now. Let's see if it runs once reservation is deleted
	#
	# Delete the reservation
	set ret_code [delete_res $res_name]
	if {$ret_code != 0} {
		exit $ret_code
	}

	sleep 10
	# Show the job
	spawn $scontrol show job $job_id
	expect {
		-re "Invalid job id specified" {
			send_user "\n\033\[31mFAILURE: Job $job_id not found (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
			exp_continue
		}
		-re "JobState=PENDING" {
			send_user "\n\033\[31mFAILURE: Job $job_id is PENDING but it should not (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
			exp_continue
		}
		-re "JobState=RUNNING" {
			send_user "\n\033\[32mSUCCESS: Job $job_id is RUNNING as expected (Within: inc3.11.7)\033\[m\n"
			exp_continue
		}
		timeout {
			send_user "\n\033\[31mFAILURE: scontrol not responding (Within: inc3.11.7)\033\[m\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	cancel_job $job_id
}
