#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          sacctmgr federation functionality
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2016 SchedMD LLC.
# Written by Brian Christiansen <brian@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

source ./globals
source ./globals_accounting

set test_id     "37.1"
set test_id_2   "37_1"
set exit_code   0

set max_federations 64

set eol "\r\n"

# Test Clusters
set tc1		"tc1_$test_id_2"
set tc2		"tc2_$test_id_2"
set tc3		"tc3_$test_id_2"
set tc4		"tc4_$test_id_2"

set fed1 	"fed1_$test_id_2"
set fed2 	"fed2_$test_id_2"
set fed3 	"fed3_$test_id_2"

set timeout	60
print_header $test_id

#
# Check accounting config and bail if not found.
#
if { [test_account_storage] == 0 } {
	send_user "\nWARNING: This test can't be run without a usable AccountStorageType\n"
	exit 0
}

if { [string compare [check_accounting_admin_level] "Administrator"] } {
	send_user "\nWARNING: This test can't be run without being an Accounting administrator.\n"
	send_user "Use: sacctmgr mod user \$USER set admin=admin.\n"
	exit 0
}

proc delete_object { object names } {
	global sacctmgr
	set matches 0
	set rc 0
	set my_pid [spawn $sacctmgr -i delete $object $names]
	expect {
		-re "privilege to perform this action" {
			send_user "FAILURE: don't have privileges."
			incr rc
		}
		-re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" {
			send_user "FAILURE: there was a problem with the sacctmgr command\n"
			incr rc
		}
		-re "Problem getting" {
			send_user "FAILURE: there was a problem getting information from the database\n"
			incr rc
		}
		-re "Problem adding" {
			send_user "FAILURE: there was an unknown problem\n"
			incr rc
		}
		-re "No associations" {
			send_user "FAILURE: your command didn't return anything\n"
			incr rc
		}
		-re "Deleting $object" {
			incr matches
			exp_continue
		}
		-re " Nothing deleted" {
			incr matches
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: sacctmgr delete not responding\n"
			slow_kill $my_pid
			incr rc
		}
		eof {
			wait
		}
	}

	if {$matches != 1} {
		send_user "\nFAILURE: sacctmgr had a problem deleting $object got $matches\n"
		incr rc
	}

	return $rc
}

proc delete_clusters { names } {
	return [delete_object "cluster" $names]
}

proc delete_federations { names } {
	return [delete_object "federation" $names]
}

proc cleanup { } {
	global tc1 tc2 tc3 tc4 fed1 fed2 fed3 exit_code test_id_2 max_federations

	set tmp_clusters ""
	for {set i 0} {$i <= $max_federations} {incr i} {
		append  tmp_clusters "max${i}_$test_id_2,"
	}

	set exit_code [expr $exit_code + [delete_clusters "$tc1,$tc2,$tc3,$tc4,${tmp_clusters}"]]
	set exit_code [expr $exit_code + [delete_federations "$fed1,$fed2,$fed3"]]
}

proc end_it { exit_code } {
	global test_id
	cleanup
	if {$exit_code == 0} {
		print_success $test_id
	}
	exit $exit_code
}

#
# Use sacctmgr to delete the test cluster
#
set nothing 0
set matches 0

cleanup
if {$exit_code != 0} {
	exit $exit_code
}


######MAIN######
#####################################
# TEST: add federation with non-existant cluster(s)
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1 cluster=$tc1]
expect {
	-re "The cluster $tc1 doesn't exist. Please add first." {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches == 0} {
	send_user "FAILURE: adding invalid federation failed unpexectedly.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1 cluster=$tc1,$tc2]
expect {
	-re "The cluster $tc1 doesn't exist. Please add first." {
		incr matches
		exp_continue
	}
	-re "The cluster $tc2 doesn't exist. Please add first." {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "FAILURE: adding invalid federation failed unpexectedly.\n"
	end_it 1
}

#####################################
# TEST: add cluster to non-existant federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc1 federation=$fed1]
expect {
	-re "The federation $fed1 doesn't exist." {
		incr matches
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 1} {
	send_user "FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: add new federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format=federation%20]
expect {
	-re "Federation $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

#####################################
# TEST: add new federation - already exists
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1]
expect {
	-re "This federation $fed1 already exists.  Not adding.$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 1} {
	send_user "FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

#####################################
# TEST: add new federation with options
#####################################
set matches 0

delete_federations "$fed1"
set my_pid [spawn $sacctmgr -i add federation $fed1 flags=LLC]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Settings$eol" {
		incr matches
		exp_continue
	}
	-re "Flags\\s+=\\s+LLC$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format=federation%20,flags]
expect {
	-re "Federation\\s+Flags $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+LLC $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

#####################################
# TEST: add second federation and make sure that you can select only one federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed2]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation format=federation%20]
expect {
	-re "Federation $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 3} {
	send_user "$matches FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format=federation%20]
expect {
	-re "Federation $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: adding federation failed unpexectedly.\n"
	end_it 1
}

#####################################
# TEST: add new cluster to existing federation
# TEST: add new cluster with weight and state
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc1 federation=$fed1 weight=9 fedstate=drain]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+FedState\\s+=\\s+DRAIN$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Weight\\s+=\\s+9$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: add multiple new clusters to single federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc2 $tc3 federation=$fed2]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed1\\s+DRAIN\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+ACTIVE\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}


#Test show cluster WithFed
set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 WithFed]
expect {
	-re "Cluster\\s+.*?Federation\\s+ID\\s+Weight\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+.*?$fed1\\s+1\\s+9\\s+DRAIN $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+.*?$fed2\\s+\\s+1\\s+1\\s+ACTIVE $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+.*?$fed2\\s+\\s+2\\s+1\\s+ACTIVE $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,cluster%20"]
expect {
	-re "Federation\\s+Cluster $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+$tc1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$tc2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$tc3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# TEST tree option - shows one federation line per federation.
set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,cluster%20" tree]
expect {
	-re "Federation\\s+Cluster $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2 $eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: modify clusters with fed options
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 $tc3 set weight=5 fedstate=DRAIN]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+DRAIN$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Weight\\s+=\\s+5$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc2 set weight=9 fedstate=DRAIN+REMOVE]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+DRAIN\\+REMOVE$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Weight\\s+=\\s+9$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 set fedstate=ACTIVE]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+ACTIVE$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,weight"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+Weight $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed1\\s+ACTIVE\\s+5 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed2\\s+DRAIN\\+REMOVE\\s+9 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN\\s+5 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: modify federation with flags
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 $fed2 set flags+=LLC]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Flags\\s+\\+=\\s+LLC$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,flags,cluster%20,fedstate%20,weight"]
expect {
	-re "Federation\\s+Flags\\s+Cluster\\s+FedState\\s+Weight $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+LLC\\s+$tc1\\s+ACTIVE\\s+5 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+LLC\\s+$tc2\\s+DRAIN\\+REMOVE\\s+9 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+LLC\\s+$tc3\\s+DRAIN\\s+5 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,flags,cluster%20,fedstate%20,weight" tree]
expect {
	-re "Federation\\s+Flags\\s+Cluster\\s+FedState\\s+Weight $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+LLC\\s+ $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+ACTIVE\\s+5 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+LLC\\s+ $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+DRAIN\\+REMOVE\\s+9 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+DRAIN\\s+5 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: modify federation clear flags
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 $fed2 set flags-=LLC]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Flags\\s+-=\\s+LLC$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# Verify that LLC is removed and that only the federations are listed since no
# cluster print fields are requested.
set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,flags"]
expect {
	-re "Federation\\s+Flags $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+ $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+ $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 3} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: mod cluster to federation -- check ids
# create hole in fed2 ids
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc2 set federation=$fed1]
expect {
	-re "The cluster $tc2 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed1\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed1\\s+ACTIVE\\s+2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# move tc1 into whole.
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 set federation=$fed2]
expect {
	-re "The cluster $tc1 is assigned to federation $fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed1\\s+ACTIVE\\s+2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# move tc2 back to fed2 and get new id -- 3.
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc2 set federation=$fed2]
expect {
	-re "The cluster $tc2 is assigned to federation $fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed2\\s+ACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}


#####################################
# TEST: add federation with existing clusters
#####################################
#####################################
# TEST: add federation with flags
#####################################
set matches 0
set my_pid [spawn $sacctmgr add federation $fed3 flags=LLC clusters=$tc1,$tc2]
expect {
	-re "The cluster $tc1 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	-re "The cluster $tc2 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	"Are you sure" {
		incr matches
		exp_continue
	}
	"(N/y):" {
		incr matches
		send "y"
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: adding invalid federation failed unpexectedly.\n"
	end_it 1
}

set matches 0
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Settings$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Flags\\s+=\\s+LLC?$eol" {
		incr matches
		exp_continue
	}
	"Would you like" {
		incr matches
		exp_continue
	}
	"(N/y):" {
		incr matches
		send "y"
		expect eof
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 7} {
	send_user "$matches FAILURE: adding invalid federation failed unpexectedly.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 $fed3 format="federation%20,flags,cluster%20,fedstate%20,weight,id"]
expect {
	-re "Federation\\s+Flags\\s+Cluster\\s+FedState\\s+Weight\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$tc3\\s+DRAIN\\s+5\\s+2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed3\\s+LLC\\s+$tc1\\s+ACTIVE\\s+5\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed3\\s+LLC\\s+$tc2\\s+ACTIVE\\s+9\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# modify cluster, clear federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc3 set federation=]
expect {
	-re "Setting" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: test selecting clusters by federations
#####################################
set matches 0
set my_pid [spawn $sacctmgr show cluster where fed=$fed3 format="cluster%20,federation%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed3\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed3\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 3} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}



#####################################
# TEST: test adding new cluster and verify state and change states
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc4]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# New clusters should have fed_id=0, federation="", fed_state=NA
set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# Test adding cluster to federation sets state to ACTIVE
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set federation=$fed3]
expect {
	-re "Setting" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+ACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}


# Modifying cluster to same federation shouldln't change fed_id or fed_state
set matches 0
# set state to something other than ACTIVE, it should stay the same.
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=DRAIN]
expect {
}
expect -re $
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set federation=$fed3]
expect {
	-re "The cluster $tc4 is already assigned to federation $fed3" {
		incr matches
		exp_continue
	}
	"Nothing to change" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+DRAIN\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#same thing for modifying the federaiton.
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed3 set clusters+=$tc4]
expect {
	-re "The cluster $tc4 is already assigned to federation $fed3" {
		incr matches
		exp_continue
	}
	"Nothing to change" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+DRAIN\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# changing from one federation to another should set the state to active
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set federation=$fed2]
expect {
	-re "The cluster $tc4 is assigned to federation $fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Setting" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}


# same thing for modifying federation - state should go to active
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=DRAIN]
expect {
}
expect -re $
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed3 set clusters+=$tc4]
expect {
	-re "The cluster $tc4 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc4$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+ACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# Test setting state to INACTIVE
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=INACTIVE]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+INACTIVE$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+INACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# Test invalid state
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=abcdefg]
expect {
	-re "Invalid FedState abcdefg" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 1} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}



#####################################
# TEST: modify federation change clusters - must remove others
#####################################
#sacctmgr modify federation fed set clusters=test3,test4
#sacctmgr modify federation fed set clusters=test1,test2
#sacctmgr modify federation fed set clusters+=test1,test2
#sacctmgr modify federation fed set clusters-=test1,test2
# unset clusters
set my_pid [spawn $sacctmgr -i modify cluster $tc1 $tc2 $tc3 set federation=]
expect {
}
expect -re $

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=$tc1,$tc2 flags=LLC]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Flags\\s+=\\s+LLC?$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc1$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20,flags"]
expect {
	-re "Federation\\s+Cluster\\s+FedState\\s+Flags $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE\\s+LLC $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc1] ||
		    [string equal $match $tc2]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 3} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=$tc1,$tc3]
expect {
	-re "The cluster $tc1 is already assigned to federation $fed1" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc3$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20,flags"]
expect {
	-re "Federation\\s+Cluster\\s+FedState\\s+Flags $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE\\s+LLC $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc1] ||
		    [string equal $match $tc3]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 3} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters+=$tc2]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc1] ||
		    [string equal $match $tc2] ||
		    [string equal $match $tc3]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters-=$tc1,$tc2]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc1$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc3]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}


set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters-=$tc1,$tc3]
expect {
	-re "The cluster $tc1 isn't assigned to federation $fed1" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc3$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+ $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: error checking on using +, - and =
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set cluster=$tc1,+$tc2]
expect {
	"sacctmgr: error: You can't use '=' and '+' or '-' in the same line" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 1} {
	send_user "FAILURE: failed testing mod with + and =\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set cluster=$tc1,-$tc2]
expect {
	"sacctmgr: error: You can't use '=' and '+' or '-' in the same line" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 1} {
	send_user "FAILURE: failed testing mod with - and =\n"
	end_it 1
}


#####################################
# TEST: modify federation, clear clusters
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 $tc2 $tc3 set federation=$fed1]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr mod not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc1$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc3$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 6} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+ $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 2} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

# Verify clusters fed_id=0, federation="", fed_state=NA after being removed from
# federation
set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+($tc1|$tc2|$tc3)\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}


#####################################
# TEST: test deleting cluster with where federation= clause
#####################################
set my_pid [spawn $sacctmgr -i delete federation $fed2 $fed3]
expect {
}
expect -re $

set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed2 clusters=$tc1,$tc2]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Settings$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: adding federation with clusters failed.\n"
	end_it 1
}

# add second cluster to make sure selectin only on federation
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed3 clusters=$tc3,$tc4]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Settings$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: adding federation with clusters failed.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i delete cluster where federation=$fed2]
expect {
	-re "Deleting clusters...$eol" {
		incr matches
		exp_continue
	}
	-re "$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 3} {
	send_user "$matches FAILURE: deleting clusters with federation=.\n"
	end_it 1
}

# add back clusters back to federation to verify both federation clusters are
# selected.
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc1,$tc2 federation=$fed2]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i delete cluster where federation=$fed2,$fed3]
expect {
	-re "Deleting clusters...$eol" {
		incr matches
		exp_continue
	}
	-re "$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "$tc3$eol" {
		incr matches
		exp_continue
	}
	-re "$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr delete cluster not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 5} {
	send_user "$matches FAILURE: deleting clusters with federation=.\n"
	end_it 1
}


#####################################
# TEST: test adding more than 63 clusters to a federation
#####################################
# unset federations
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=]
expect {
}
expect -re $

for {set i 1} {$i <= $max_federations} {incr i} {
	set matches 0
	set tmpc "max${i}_$test_id_2"
	set my_pid [spawn $sacctmgr -i add cluster $tmpc federation=$fed1]
	if {$i < $max_federations} {
		expect {
			-re "Adding Cluster\\(s\\)$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+$tmpc$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+Setting$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+Federation\\s+=\\s+$fed1$eol" {
				incr matches
				exp_continue
			}
			timeout {
				send_user "\nFAILURE: sacctmgr add not responding\n"
				slow_kill $my_pid
				set exit_code 1
			}
			eof {
				wait
			}
		}
		if {$exit_code || $matches != 4} {
			send_user "$matches FAILURE: unexpected error.\n"
			end_it 1
		}
	} else {
		expect {
			-re "Adding Cluster\\(s\\)$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+$tmpc$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+Setting$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+Federation\\s+=\\s+$fed1$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+Problem adding clusters: Too many clusters in federation?" {
				incr matches
				exp_continue
			}
			timeout {
				send_user "\nFAILURE: sacctmgr add not responding\n"
				slow_kill $my_pid
				set exit_code 1
			}
			eof {
				wait
			}
		}
		if {$exit_code || $matches != 5} {
			send_user "$matches FAILURE: unexpected error.\n"
			end_it 1
		}

		set matches 0
		#####################################
		# TEST: modify cluster to exceed max clusters in federation
		#####################################
		#add last cluster without federation
		set my_pid [spawn $sacctmgr -i add cluster $tmpc]
		expect {
			-re "Adding Cluster\\(s\\)$eol" {
				incr matches
				exp_continue
			}
			-re "\\s+$tmpc$eol" {
				incr matches
				exp_continue
			}
			timeout {
				send_user "\nFAILURE: sacctmgr add not responding\n"
				slow_kill $my_pid
				set exit_code 1
			}
			eof {
				wait
			}
		}
		if {$exit_code || $matches != 2} {
			send_user "$matches FAILURE: unexpected error.\n"
			end_it 1
		}

		set matches 0
		set my_pid [spawn $sacctmgr -i modify cluster $tmpc set federation=$fed1]
		expect {
			-re "Setting$eol" {
				incr matches
				exp_continue
			}
			-re "^\\s+Federation\\s+=\\s+$fed1$eol" {
				incr matches
				exp_continue
			}
			-re "sacctmgr: error: Too many clusters in federation$eol" {
				incr matches
				exp_continue
			}
			timeout {
				send_user "\nFAILURE: sacctmgr add not responding\n"
				slow_kill $my_pid
				set exit_code 1
			}
			eof {
				wait
			}
		}
		if {$exit_code || $matches != 3} {
			send_user "$matches FAILURE: unexpected error.\n"
			end_it 1
		}
	}

}
set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20"]
expect {
	-re "Federation\\s+Cluster $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+max\[1-6\]{0,1}\\d{1}_$test_id_2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != $max_federations} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

#####################################
# TEST: delete cluster - should delete it from federation.
#####################################
delete_clusters "max1_$test_id_2"

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20"]
expect {
	-re "\\s+$fed1\\s+max1_$test_id_2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches } {
	send_user "$matches FAILURE: found cluster still on federation when it should have been deleted.\n"
	end_it 1
}


#####################################
# TEST: delete federation - should clean clusters from federation.
#####################################
delete_federations $fed1

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20"]
expect {
	-re "\\s+$fed1.*?$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches } {
	send_user "$matches FAILURE: found federation when it should have been deleted.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr show cluster format="cluster%20,federation%20"]
expect {
	-re "$fed1" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches } {
	send_user "$matches FAILURE: found federation on cluster when it was deleted.\n"
	end_it 1
}


# Verify clusters fed_id=0, federation="", fed_state=NA after federation is deleted.

set tmp_clusters ""
for {set i 0} {$i <= $max_federations} {incr i} {
	append  tmp_clusters "max${i}_$test_id_2,"
}
set matches 0
set my_pid [spawn $sacctmgr show cluster $tmp_clusters format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+max\[1-6\]{0,1}\\d{1}_$test_id_2\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr add not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != $max_federations} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

end_it 0
