#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Test sreport cluster utilization.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2008 Lawrence Livermore National Security.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Danny Auble <da@llnl.gov>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of SLURM, a resource management program.
# For details, see <http://www.schedmd.com/slurmdocs/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting

source ./inc22.1.1
source ./inc22.1.2
source ./inc22.1.3
source ./inc22.1.4

set test_id     "test22.1"
set test_nu     "test22-1"
set exit_code   0
set matches     0
set not_support 0
set sql_in     "$test_id-in.sql"
set sql_rem    "$test_id-rem.sql"
set sql_out    "$test_id-out.sql"
set cluster    [format "%s%s" $test_nu "clus"]
set account1   [format "%s%s" $test_nu "acct1"]
set account2   [format "%s%s" $test_nu "acct2"]
set account3   [format "%s%s" $test_nu "acct3"]
set accounts   [format "%s,%s,%s" $account1 $account2 $account3]
set wckey1     [format "%s%s" $test_nu "wckey"]
set user1      [format "%s%s" $test_nu "user1"]
set user2      [format "%s%s" $test_nu "user2"]
set users      [format "%s,%s" $user1 $user2]
set node0      [format "%s%d" $cluster 0]
set node1      [format "%s%d" $cluster 1]
set node_list  [format "%s%s" $cluster "\[0-1\]"]
set node0_cpus 2
set node1_cpus 2
set cluster_cpus [expr $node0_cpus + $node1_cpus]
set access_err  0
set uid [get_my_uid]
set gid [get_my_gid]
set timeout 120

print_header $test_id

# Fri Thu Jan 31 00:00:00 2008 - Needed only for the 00:00:00 for timing purposes
set midnight 1201766400

# Mon Dec 31 23:00:00 2007
set period_start 1199170800
# Thu Jan 31 23:59:59 2008
set period_end 1201852799

set start_str [timestamp -format %Y-%m-%dT%X -seconds $period_start]
set end_str [timestamp -format %Y-%m-%dT%X -seconds $period_end]

#job1
set job1_start $period_start
set job1_run 3900
set job1_end [expr $job1_start+$job1_run]
# This will give us the correct time we ran for
set job1_diff_str [timestamp -format %X -seconds [expr $midnight+$job1_run]]
set job1_start_str [timestamp -format %Y-%m-%dT%X -seconds $job1_start]
set job1_end_str [timestamp -format %Y-%m-%dT%X -seconds $job1_end]
set job1_nodes $node1
set job1_cpus $node1_cpus
set job1_alloc [expr $job1_run * $job1_cpus]
set job1_acct $account1

#job2
#make job eligible an hour into the allocation
set job2_elig [expr $period_start+3600]
#start the job 65 minutes later so we can check reserved time
set job2_start [expr $job2_elig+3900]
#run for a day
set job2_run 86400
set job2_end [expr $job2_start+$job2_run]
# This will give us the correct time we ran for
set job2_diff_str "1-00:00:00"
set job2_start_str [timestamp -format %Y-%m-%dT%X -seconds $job2_start]
set job2_end_str [timestamp -format %Y-%m-%dT%X -seconds $job2_end]
set job2_nodes [format "%s\[%s\]" $cluster "0-1"]
set job2_cpus [expr $node0_cpus + $node1_cpus]
set job2_alloc [expr $job2_run * $job2_cpus]
set job2_acct $account3

#job3
#make job eligible an hour before the end of job2
set job3_elig [expr $job2_end-3600]
#start the job at the end of job2
set job3_start $job2_end
#run for 65 minutes
set job3_run 3900
set job3_end [expr $job3_start+$job3_run]
# This will give us the correct time we ran for
set job3_diff_str [timestamp -format %X -seconds [expr $midnight+$job1_run]]
set job3_start_str [timestamp -format %Y-%m-%dT%X -seconds $job3_start]
set job3_end_str [timestamp -format %Y-%m-%dT%X -seconds $job3_end]
#run on just node0
set job3_nodes $node0
set job3_cpus $node0_cpus
set job3_alloc [expr $job3_run * $job3_cpus]
set job3_acct $account2

set acct1_alloc $job1_alloc
set acct3_alloc $job2_alloc
set acct2_alloc [expr $acct3_alloc + $job3_alloc]
set total_alloc [expr $job1_alloc + $job2_alloc + $job3_alloc]

set wckey1_alloc [expr $job1_alloc + $job2_alloc + $job3_alloc]
set user1_wckey1_alloc [expr $job1_alloc + $job3_alloc]
set user2_wckey1_alloc $job2_alloc
#node0 down
set node0_down_start [expr $period_start+(60*45)]
set node0_down_end [expr $period_start+(60*75)]

#
# Check accounting config and bail if not found.
#
if { [test_account_storage] == 0 } {
	send_user "\nWARNING: This test can't be run without a usable AccountStorageType\n"
	exit 0
}

set wc_key_track [test_track_wckey]
if { $wc_key_track == 0 } {
	send_user "\nWARNING: Portions of this test will not work without TrackWCKey\n"
}

if { [string compare [check_accounting_admin_level] "Administrator"] } {
	send_user "\nWARNING: This test can't be run without being an Accounting administrator.\nUse sacctmgr mod user \$USER_NAME admin=admin.\n"
	exit 0
}

remove_user "" "" $users
remove_acct "" $accounts
remove_cluster "$cluster"
#add cluster
incr exit_code [add_cluster "$cluster" "" "" "" "" "" "" "" "" "" "" "" ""]
if { $exit_code } {
 	remove_cluster "$cluster"
	exit $exit_code
}

#add accounts
incr exit_code [add_acct "$cluster" "" "$account1,$account2" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ]
if { $exit_code } {
	remove_acct "" "$account1,$account2"
	remove_cluster "$cluster"
	exit $exit_code
}

#add accounts
incr exit_code [add_acct "$cluster" "$account2" "$account3" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ]
if { $exit_code } {
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}

#add users
incr exit_code [add_user "$cluster" $accounts $users "" "" "" $wckey1 "" "" "" "" "" "" "" "" "" "" "" "" "" "" "" ]
if { $exit_code } {
	remove_user "" "" $users
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}

#get the user association ids for the jobs we plan to add
set user1acct1 0
set user1acct2 0
set user1acct3 0
set user2acct1 0
set user2acct2 0
set user2acct3 0
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster format="User,account,id"]
expect {
	-re "There was a problem" {
		send_user "FAILURE: there was a problem with the sacctmgr command\n"
	    	incr exit_code 1
	}
	-re "$user1.$account1.($number)." {
		set user1acct1 $expect_out(1,string)
		exp_continue
	}
	-re "$user1.$account2.($number)." {
		set user1acct2 $expect_out(1,string)
		exp_continue
	}
	-re "$user1.$account3.($number)." {
		set user1acct3 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$account1.($number)." {
		set user2acct1 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$account2.($number)." {
		set user2acct2 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$account3.($number)." {
		set user2acct3 $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr list associations not responding\n"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {!$user1acct1 || !$user1acct2 || !$user1acct3 || !$user2acct1 || !$user2acct2|| !$user2acct3} {
	send_user "\nFAILURE:  Didn't get one of the user associations $user1acct1 $user1acct2 $user1acct3 $user2acct1 $user2acct2 $user2acct3.\n"
	incr exit_code 1
	remove_user "" "" $users
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}

#get the user association ids for the jobs we plan to add
set user1wckey1 0
set user2wckey1 0
set my_pid [eval spawn $sacctmgr -n -p list wckeys users=$users wckeys=$wckey1 cluster=$cluster format="user,wckey,id"]
expect {
	-re "There was a problem" {
		send_user "FAILURE: there was a problem with the sacctmgr command\n"
	    	incr exit_code 1
	}
	-re "$user1.$wckey1.($number)." {
		set user1wckey1 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$wckey1.($number)." {
		set user2wckey1 $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr list associations not responding\n"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {!$user1wckey1 || !$user1wckey1} {
	send_user "\nFAILURE:  Didn't get one of the user wckeys $user1wckey1 $user2wckey1.\n"
	incr exit_code 1
	remove_user "" "" $users
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}

exec $bin_rm -f $sql_in
set file [open $sql_in "w"]



# DON'T MESS WITH THIS UNLESS YOU REALLY UNDERSTAND WHAT YOU ARE DOING!!!!!
# THIS COULD SERIOUSLY MESS UP YOUR DATABASE IF YOU ALTER THIS INCORRECTLY
# JUST A FRIENDLY REMINDER ;)

# put in the cluster for back in the day before accounting was made here for us we are using 'Tue Jan  1 00:00:00 2008' = 1199174400 as the start

puts $file "insert into cluster_event_table (node_name, cluster, cpu_count, period_start, period_end, reason, cluster_nodes) values"
puts $file "('', '$cluster', $cluster_cpus, $period_start, $period_end, 'Cluster processor count', '$node_list' )"

#put a node down for 30 minutes starting at 45 minutes after the start to make sure our rollups work so we should get 15 minutes on one hour and 15 on the other
puts $file ", ('$node0', '$cluster', $node0_cpus, $node0_down_start, $node0_down_end, 'down','')"
#puts $file ", ('$node1', '$cluster', $node1_cpus, $period_start, $period_end, 'down')"
puts $file "on duplicate key update period_start=VALUES(period_start), period_end=VALUES(period_end);"

#now we will put in a job running for an hour and 5 minutes
puts $file "insert into job_table (jobid, associd, wckey, wckeyid, uid, gid, partition, blockid, cluster, account, eligible, submit, start, end, suspended, name, track_steps, state, comp_code, priority, req_cpus, alloc_cpus, nodelist, kill_requid, qos, deleted) values"
puts $file "('65537', '$user1acct1', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$job1_acct', $job1_start, $job1_start, $job1_start, $job1_end, '0', 'test_job1', '0', '3', '0', '$job1_cpus', '$job1_cpus', '$job1_cpus', '$job1_nodes', '0', '0', '0')"
puts $file ", ('65538', '$user2acct3', '$wckey1', '$user2wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$job2_acct', $job2_elig, $job2_elig, $job2_start, $job2_end, '0', 'test_job2', '0', '3', '0', '$job2_cpus', '$job2_cpus', '$job2_cpus', '$job2_nodes', '0', '0', '0')"
puts $file ", ('65539', '$user1acct2', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$job3_acct', $job3_elig, $job3_elig, $job3_start, $job3_end, '0', 'test_job3', '0', '3', '0', '$job3_cpus', '$job3_cpus', '$job3_cpus', '$job3_nodes', '0', '0', '0')"
puts $file "on duplicate key update id=LAST_INSERT_ID(id), eligible=VALUES(eligible), submit=VALUES(submit), start=VALUES(start), end=VALUES(end), associd=VALUES(associd), alloc_cpus=VALUES(alloc_cpus), wckey=VALUES(wckey), wckeyid=VALUES(wckeyid);";
close $file

exec $bin_rm -f $sql_rem
set file [open $sql_rem "w"]

puts $file [format "%s%s%s" "truncate table \"" $cluster "_event_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_job_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_step_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_day_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_hour_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_month_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_day_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_hour_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_month_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_day_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_hour_table\";"]
puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_month_table\";"]
close $file

#
# Use sacctmgr to load info
#
incr exit_code [archive_load $sql_in]

if { $exit_code } {
	incr exit_code [archive_load $sql_rem]
	remove_user "" "" $users
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}


#
# Use sacct to see if the job loaded
#
set matches 0
set my_pid [eval spawn $sacct -p -C $cluster --fields=cluster,account,associd,wckey,wckeyid,start,end,elapsed --noheader --start=$start_str --end=$end_str]
expect {
	-re "There was a problem" {
		send_user "FAILURE: there was a problem with the sacctmgr command\n"
	    	incr exit_code 1
	}
	-re "$cluster.$account1.$user1acct1.$wckey1.$user1wckey1.$job1_start_str.$job1_end_str.$job1_diff_str." {
		send_user "got 1\n"
		incr matches
		exp_continue
	}
	-re "$cluster.$account3.$user2acct3.$wckey1.$user2wckey1.$job2_start_str.$job2_end_str.$job2_diff_str." {
		send_user "got 2\n"
		incr matches
		exp_continue
	}
	-re "$cluster.$account2.$user1acct2.$wckey1.$user1wckey1.$job3_start_str.$job3_end_str.$job3_diff_str." {
		send_user "got 3\n"
		incr matches
		exp_continue
	}

	timeout {
		send_user "\nFAILURE: sacctmgr archive load not responding\n"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {$matches != 3} {
	send_user "\nFAILURE:  job wasn't loaded correctly with only $matches.\n"
	incr exit_code 1
}

if { $exit_code } {
	#incr exit_code [archive_load $sql_rem]
	remove_user "" "" $users
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}

#
# Use sacctmgr to roll up that time period
#
set matches 0
set my_pid [eval spawn $sacctmgr -i roll $start_str $end_str]
expect {
	-re "There was a problem" {
		send_user "FAILURE: there was a problem with the sacctmgr command\n"
	    	incr exit_code 1
	}
	-re "$cluster" {
		incr matches
		exp_continue
	}
	-re "SUCCESS" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr archive load not responding\n"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {$matches != 1} {
	send_user "\nFAILURE:  sacctmgr wasn't able to roll data.\n"
	incr exit_code 1
}

if { $exit_code } {
	incr exit_code [archive_load $sql_rem]
	remove_user "" "" $users
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}

#
# Execute sub-test
#
inc22_1_1

inc22_1_2

inc22_1_3

inc22_1_4

if {$matches != 1} {
	send_user "\nFAILURE:  sreport didn't give good info 25 $matches.\n"
	incr exit_code 1
}

if { $exit_code } {
	incr exit_code [archive_load $sql_rem]
	remove_user "" "" $users
	remove_acct "" $accounts
	remove_cluster "$cluster"
	exit $exit_code
}

if {$exit_code == 0} {
	incr exit_code [archive_load $sql_rem]
	incr exit_code [remove_user "" "" $users]
	incr exit_code [remove_acct "" $accounts]
	incr exit_code [remove_cluster "$cluster"]

 	exec $bin_rm -f $sql_in
 	exec $bin_rm -f $sql_rem
	if {$exit_code == 0} {
		send_user "\nSUCCESS\n"
	} else {
		send_user "\nFAILURE\n"
	}
} else {
	send_user "\nFAILURE\n"
}

exit $exit_code
