#!/usr/bin/env expect
############################################################################
# Purpose: Test of SLURM functionality
#          Test sacct filtering of records by account and job name.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2012 SchedMD LLC.
# Copyright (C) 2008-2010 Lawrence Livermore National Security.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Joseph Donaghy <donaghy1@llnl.gov> and
#            Nathan Yee <nyee32@schedmd.com>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting
source ./inc12.3.1
source ./inc12.3.2

set test_id     "12.3"
set exit_code   0
set job_name_1  "JOB1"
set job_name_2  "JOB2"
set job_name_3  "JOB3"
set file_in     "test.$test_id.input"
set test_acct	"slurm_test-account.1"

print_header $test_id

#
# Checks if test is running on blue gene.
#
if {[test_front_end]} {
	send_user "\nWARNING: This test can't be run on a front end system\n"
	exit $exit_code
} elseif {[test_serial]} {
	send_user "\nWARNING: This test is incompatible with serial systems\n"
	exit $exit_code
}

if {[test_launch_poe]} {
	set step_id 1
} else {
	set step_id 0
}

#
# Check accounting config and bail if not found.
#
if { [test_account_storage] == 0 } {
	send_user "\nWARNING: This test can't be run without a usable AccountStorageType\n"
	exit $exit_code
}

if { [string compare [check_accounting_admin_level] "Administrator"] } {
	send_user "\nWARNING: This test can't be run without being an Accounting administrator.\n"
	send_user "Use sacctmgr mod user \$USER_NAME set admin=admin.\n"
	exit $exit_code
}

#
# Make sure we have enough nodes to test functionality
#
set partition [default_partition]
set node_count [get_node_cnt_in_part $partition]
if { $node_count < 3 } {
	send_user "WARNING: system must have at least 3 \
		   nodes to run this test on.  This system \
		   only has $node_count.\n"
	exit $exit_code
}

#
# Identify the user and his current default account
#
set acct_name ""
set user_name ""
spawn $bin_id -u -n
expect {
	 -re "($alpha_numeric_under)" {
		set user_name $expect_out(1,string)
		exp_continue
	}
	eof {
		wait
	}
}
set s_pid [spawn $sacctmgr show user $user_name]
expect {
	-re "$user_name *($alpha_numeric_under)" {
		set acct_name $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "FAILURE: sacctmgr add not responding\n"
		slow_kill $s_pid
		exit 1
	}
	eof {
		wait
	}
}

#
# Use sacctmgr to add a test account
#
set aamatches 0
set sadd_pid [spawn $sacctmgr -i add account $test_acct]
expect {
	-re "Adding Account" {
		incr aamatches
		exp_continue
	}
	-re "Nothing new added" {
		send_user "\nWARNING: vestigial account $test_acct found\n"
		incr aamatches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $sadd_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$aamatches != 1} {
	send_user "\nFAILURE:  sacctmgr had a problem adding account ($aamatches != 1)\n"
	exit 1
}

#
# Add self to this new account
#
set sadd_pid [spawn $sacctmgr -i create user name=$user_name account=$test_acct]
expect {
	 timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $sadd_pid
		set exit_code 1
	}
	eof {
		wait
	}
}

#
# Submit two slurm jobs to capture job info
#
set job_id_1 0
set job_1_node_inx 0
set job_1_node(0) "TBD"
set job_1_node(1) "TBD"
set timeout $max_job_delay
set srun_pid [spawn $srun -N2 -t1 -A $test_acct -J $job_name_1 -v $bin_id]
expect {
	-re "launching ($number).$step_id on host ($alpha_numeric_under)," {
		set job_id_1 $expect_out(1,string)
		set job_1_node($job_1_node_inx) $expect_out(2,string)
		incr job_1_node_inx
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		slow_kill $srun_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$job_1_node_inx != 2} {
	send_user "\nFAILURE: srun host count bad ($job_1_node_inx != 2)\n"
	set exit_code 1
}

#
# Second job explicitly excludes node zero from previous job
#
set job_id_2 0
set job_2_node_inx 0
set job_2_node(0) "TBD"
set job_2_node(1) "TBD"
set srun_pid [spawn $srun -N2 -x $job_1_node(0) -t1 -A $test_acct -J $job_name_2 -v $bin_id]
expect {

	-re "launching ($number).$step_id on host ($alpha_numeric_under)," {
		set job_id_2 $expect_out(1,string)
		set job_2_node($job_2_node_inx) $expect_out(2,string)
		incr job_2_node_inx
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		slow_kill $srun_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$job_2_node_inx != 2} {
	send_user "\nFAILURE: srun host count bad ($job_2_node_inx != 2)\n"
	set exit_code 1
}

#
# Third job explicitly excludes node 1 from previous job
#
set job_id_3 0
set job_3_node_inx 0
set job_3_node(0) "TBD"
set job_3_node(1) "TBD"
set srun_pid [spawn $srun -w$job_1_node(0),$job_2_node(1) -t1 -A $test_acct -J $job_name_3 -v $bin_id]
expect {

	-re "launching ($number).$step_id on host ($alpha_numeric_under)," {
		set job_id_3 $expect_out(1,string)
		set job_3_node($job_3_node_inx) $expect_out(2,string)
		incr job_3_node_inx
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: srun not responding\n"
		slow_kill $srun_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$job_3_node_inx != 2} {
	send_user "\nFAILURE: srun host count bad ($job_3_node_inx != 2)\n"
	set exit_code 1
}

#
# Wait for accounting data to be propagated to slurmdbd
#
sleep 5

#
# Execute sub-tests based upon these jobs
#
inc12_3_1 $job_id_1 $job_id_2 $job_name_1 $job_name_2 $test_acct

inc12_3_2 $job_id_1 $job_id_2 $job_id_3 $job_name_1 $job_name_2 $test_acct $job_1_node(0) $job_1_node(1) $job_2_node(0) $job_2_node(1)

#
# Use sacctmgr to delete the test account
#
set damatches 0
set sadel_pid [spawn $sacctmgr -i delete account $test_acct]
expect {
	-re "Deleting account" {
		incr damatches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr delete not responding\n"
		slow_kill $sadel_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$damatches != 1} {
	send_user "\nFAILURE: sacctmgr had a problem deleting account\n"
	set exit_code 1
}

if {$exit_code == 0} {
	print_success $test_id
} else {
	print_failure $test_id
}
exit $exit_code
