#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#         Test that checks if the QOS limits are enforced.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting
source ./inc21.30.1
source ./inc21.30.2
source ./inc21.30.3
source ./inc21.30.4
source ./inc21.30.5
source ./inc21.30.6
source ./inc21.30.7
source ./inc21.30.8
source ./inc21.30.9
source ./inc21.30.10
source ./inc21.30.11
source ./inc21.30.12
source ./inc21.30.13
source ./inc21.30.14
source ./inc21.30.15
source ./inc21.30.16

set test_id	"21.30"
set exit_code	0
set acct        test_acct
set user_name   ""
set qosname     name
set qostest     [format "%s_%s" $test_id "qosTest"]
set grn         GrpNodes
set grn_num     2
set grcpu       GrpCpus
set grcpu_num   10
set grpcpumin   GrpCPUMins
set grpcpumin_num  4
# Set grpcpurunmin_num to multiple of CPUs per core to work with most configurations
# Also make sure that it is at least 4 so we can add and subtract from it
set grpcpurunmin GrpCPURunMins
set grpcpurunmin_num 40
set grjobs      GrpJobs
set grjobs_num  2
set grpmem      GrpMem
set grpmem_num  100
set grsub       GrpSubmit
set grsub_num   2
set grpwall     GrpWall
set grpwall_num 1
set maxcpu      MaxCpus
set maxcpu_num  10
# Set maxcpumin_num to multiple of CPUs per core to work with most configurations
set maxcpumin   MaxCPUMins
set maxcpumin_num 2
set maxwall     MaxWall
set maxwall_num 2
set maxcpuspu   MaxCPUSPerUser
set maxcpuspu_num 2
set maxnodes    MaxNodes
set maxnode_num 10
set maxnodespu  MaxNodesPerUser
set maxnodespu_num 2
set maxjobs     MaxJobs
set maxjobs_num 2
set maxjobsub   MaxSubmitJobs
set maxjobsub_num 2
set time_spacing 1

print_header $test_id

# Checks the state of the job
proc check_state { job } {

	global scontrol job_id exit_code

	set state_match 0
	spawn $scontrol show job $job
	expect {
		"JobState=PENDING" {
			incr state_match
		}
		timeout {
			send_user "\nFAILURE scontrol not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if {$state_match != 1} {
		send_user "\nFAILURE job should be pending, but is not\n"
		set exit_code 1
	}

}

# modifies the QoS
proc mod_qos { node cpu job sub mcpu mnode mjobs mjobsub gcpumin gcpurunmin gmem gwall mcpumin mwall mcpupu mnodespu} {

	global sacctmgr exit_code qosname qostest grn grcpu grjobs grsub maxcpu maxnodes maxjobs maxjobsub grpcpumin grpcpurunmin grpmem grpwall maxcpumin maxwall maxcpumin maxwall maxcpuspu maxnodespu

	set change_cnt 0
	spawn $sacctmgr -i modify qos where $qosname=$qostest set $grn=$node $grcpu=$cpu $grjobs=$job $grsub=$sub $maxcpu=$mcpu $maxnodes=$mnode $maxjobs=$mjobs $maxjobsub=$mjobsub $grpcpumin=$gcpumin $grpcpurunmin=$gcpurunmin $grpmem=$gmem $grpwall=$gwall $maxcpumin=$mcpumin $maxwall=$mwall $maxcpuspu=$mcpupu $maxnodespu=$mnodespu
	expect {
		-re "Modified qos" {
			incr change_cnt
		}
		timeout {
			send_user "\nFAILURE sacctmgr not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}
	if {$change_cnt==0} {
		send_user "\nFAILURE: sacctmgr did not change qos $qostest\n"
		set exit_code 1
	}

}

proc endit { } {
	global sacctmgr qostest acct test_id exit_code
	# delete qos
	spawn $sacctmgr -i delete qos $qostest
	expect {
		-re "Deleting QOS(s)" {
			exp_continue
		}
		-re "Error" {
			send_user "\nFAILURE: QOS was not deleted\n"
		}
		timeout {
			send_user "\nFAILURE: sacctmgr is not responding\n"
		}
		eof {
			wait
		}
	}

	#delete account
	spawn $sacctmgr -i  delete account $acct
	expect {
		-re "Deleting accounts" {
			exp_continue
		}
		-re "Error" {
			send_user "\nFAILURE: account was not deleted\n"
			set exit_code 1
		}
		timeout {
			send_user "\nFAILURE: sacctmgr is not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if {$exit_code == 0} {
		print_success $test_id
	} else {
		send_user "\nFAILURE: test $test_id\n"
	}

	exit $exit_code
}

#
# Check accounting configuration and terminate if limits not enforced.
#
if { [test_account_storage] == 0 } {
	send_user "\nWARNING: This test can't be run without a usable AccountStorageType\n"
	exit 0
} elseif { [test_enforce_limits] == 0 } {
	send_user "\nWARNING: This test can't be run without a usable AccountingStorageEnforce\n"
	exit 0
}
if { [test_limits_enforced] == 0 } {
	send_user "\nWARNING: This test can't be run without enforcing limits\n"
	exit 0
}
if {[test_super_user] == 0} {
	send_user "\nWARNING Test can only be ran as SlurmUser\n"
	exit 0
}

#
# Some tests will not work properly when allocating whole nodes to jobs
#
set select_type [test_select_type]
if {![string compare $select_type "linear"]} {
	send_user "\nWARNING: This test is incompatible with select/$select_type\n"
	exit 0
}

# Remove any vesitgial accounts or qos
spawn $sacctmgr -i delete qos $qostest
expect {
	-re "Deleting QOS(s)" {
		exp_continue
	}
	-re "Error" {
		send_user "\nFAILURE: QOS was not deleted\n"
	}
	timeout {
		send_user "\nFAILURE: sacctmgr is not responding\n"
	}
	eof {
		wait
	}
}

# Delete account
spawn $sacctmgr -i  delete account $acct
expect {
	-re "Deleting accounts" {
		exp_continue
	}
	-re "Error" {
		send_user "\nFAILURE: account was not deleted\n"
		set exit_code 1
	}
	timeout {
		send_user "\nFAILURE: sacctmgr is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

# Gets user
spawn $bin_id -u -n
expect {
	-re "($alpha_numeric_under)" {
		set user_name $expect_out(1,string)
		exp_continue
	}
	eof {
		wait
	}
}

# add qos
set qosmatch 0
spawn $sacctmgr -i add qos $qosname=$qostest
expect {
	-re "Adding QOS" {
		incr qosmatch
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr did not add QOS\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

# Add account with qos
set acctmatch 0
spawn $sacctmgr -i add account $acct qos=$qostest
expect {
	-re "Adding Account" {
		incr acctmatch
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr is not responding\n"
		set exit_code 1
	}
	eof {
		wait

	}
}
if {$acctmatch != 1} {
	send_user "\nFAILURE: sacctmgr had a problem adding the account\n"
	exit 1
}

# Add user to account
spawn $sacctmgr -i create user name=$user_name account=$acct
expect {
	timeout {
		send_user "\nFAILURE: sacctmgr not responding\n"
	}
	eof {
		wait
	}
}

#
# Test GrpNode limit
#
mod_qos $grn_num -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_1
if {$exit_code != 0} {
	endit
}

#
# Test GrpCpus
#
mod_qos -1 $grcpu_num -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_2
if {$exit_code != 0} {
	endit
}

#
# test GrpJob limits
#
mod_qos -1 -1 $grjobs_num -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_3
if {$exit_code != 0} {
	endit
}

#
# test GrpSubmit
#
mod_qos -1 -1 -1 $grsub_num -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_4
if {$exit_code != 0} {
	endit
}

#
# Test MaxCpus limits
#
mod_qos -1 -1 -1 -1 $maxcpu_num -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_5
if {$exit_code != 0} {
	endit
}

#
# Test MaxNode limit
#
mod_qos -1 -1 -1 -1 -1 $maxnode_num -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_6
if {$exit_code != 0} {
	endit
}

#
# Test MaxJobs limit
#
mod_qos -1 -1 -1 -1 -1 -1 $maxjobs_num -1 -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_7
if {$exit_code != 0} {
	endit
}

#
# Test MaxJobsSubmits limit
#
mod_qos -1 -1 -1 -1 -1 -1 -1 $maxjobsub_num -1 -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_8
if {$exit_code != 0} {
	endit
}

#
# Test GroupCPUMins
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 $grpcpumin_num -1 -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_9
if {$exit_code != 0 } {
	endit
}

#
# Test GroupCPURunMins
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 -1 $grpcpurunmin_num -1 -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_10
if {$exit_code != 0 } {
	endit
}

#
# Test Group Memory
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 $grpmem_num -1 -1 -1 -1 -1
sleep $time_spacing
inc21_30_11
if {$exit_code != 0 } {
	endit
}

#
# Test Group wall
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 $grpwall_num -1 -1 -1 -1
sleep $time_spacing
inc21_30_12
if {$exit_code != 0 } {
	endit
}

#
# Test Max Cpu Mins
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 $maxcpumin_num -1 -1 -1
sleep $time_spacing
inc21_30_13
if {$exit_code != 0 } {
	endit
}

#
# Test Max Wall
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 $maxwall_num -1 -1
sleep $time_spacing
inc21_30_14
if {$exit_code != 0 } {
	endit
}

#
# Test Max CPUs Per User
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 $maxcpuspu_num -1
sleep $time_spacing
inc21_30_15
if {$exit_code != 0 } {
	endit
}
#
# Test MaxNodesPerUser
#
mod_qos -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 $maxnodespu_num
sleep $time_spacing
inc21_30_16
if {$exit_code != 0 } {
	endit
}

endit
