#!/usr/bin/expect
############################################################################
# Purpose: Test federated submissions
#
# Reqs:    1. Using slurmdbd accounting storage type and is up
#          2. fed_slurm_base is defined in globals.local - set to directory that
#          has access to each federation configure (fedc1, fedc2, fedc3).
#          Eg.
#          fedr/slurm/ (src)
#          fedr/fed1/bin
#          fedr/fed1/sbin
#          fedr/fed1/etc
#          fedr/fed1/...
#          fedr/fed2/...
#          fedr/fed3/...
#          3. controllers are up and running.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2016 SchedMD LLC.
# Written by Brian Christiansen <brian@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

source ./globals
source ./globals_accounting
source ./globals_federation

set test_id     "37.4"
set exit_code   0
set fed_name    "feda"
set file_in     "test$test_id.in"
set user_name   ""
set srun_job_cnt 0

array set job_counts {}

set eol "\r\n"

print_header $test_id

#
# Check accounting config and bail if not found.
#
if { [test_account_storage] == 0 } {
	log_warn "This test can't be run without a usable AccountStorageType"
	exit 0
}

if { [string compare [check_accounting_admin_level] "Administrator"] } {
	log_warn "This test can't be run without being an Accounting administrator.\n \
	 	  Use: sacctmgr mod user \$USER set admin=admin."
	exit 0
}

proc cancel_all_jobs { } {
	global scancel user_name fedc1 fedc2 fedc3

	spawn $scancel -M$fedc1,$fedc2,$fedc3 --user $user_name
	expect {
		eof {
			wait
		}
	}
	sleep 5
}

proc cleanup { } {
	global scancel fed_name user_name bin_rm file_in fedc1 fedc2 fedc3
	global test_id bin_bash

	cancel_all_jobs
	exec $bin_rm -f $file_in
	exec $bin_bash -c "$bin_rm -f test$test_id*.out"

	return [delete_federations $fed_name];
}

proc end_it { exit_code } {
	global test_id
	cleanup
	if {$exit_code == 0} {
		print_success $test_id
	}
	exit $exit_code
}

proc submit_fed_batch_job { cname expected_origin expected_sib extra_args expected_state } {
	global fed_slurm_base file_in node_count number squeue

	set submit_cluster ""
	set job_id 0
	set my_sbatch "${fed_slurm_base}/$cname/bin/sbatch"
	set command "$my_sbatch -N$node_count --exclusive --output=/dev/null --error=/dev/null -t3"
	if {$extra_args ne ""} {
		append command " $extra_args"
	}
	append command " $file_in"
	set sbatch_pid [spawn {*}$command]
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
			exp_continue
		}
		-re "on cluster (\\S+)" {
			set submit_cluster $expect_out(1,string)
			exp_continue
		}
		timeout {
			log_error "sbatch not responding"
			slow_kill $sbatch_pid
			end_it 1
		}
		eof {
			wait
		}
	}
	if {$job_id == 0} {
		log_error "batch submit failure"
		end_it 1
	}

	if {[string compare $expected_state ""]} {
		set run_cluster [wait_for_fed_job $job_id $expected_state ""]
		if {[string compare $run_cluster ""] == 0} {
			log_error "Didn't find cluster with $expected_state job"
				end_it 1
		}
	}

	return [validate_fed_job $cname $job_id $expected_origin $expected_sib $submit_cluster]
}

proc submit_fed_srun_job { cname expected_origin expected_sib extra_args check_env expected_state } {
	global fed_slurm_base file_in node_count number squeue test_id srun_job_cnt

	set submit_cluster ""
	set job_id 0
	set file_out "test${test_id}_${srun_job_cnt}.out"
	incr srun_job_cnt
	set my_srun "${fed_slurm_base}/$cname/bin/srun"
	set command "$my_srun -N$node_count --exclusive -t3 -o $file_out"
	if {$extra_args ne ""} {
		append command " $extra_args"
	}
	append command " $file_in"
	set srun_pid [spawn {*}$command]
	expect {
		-re "job ($number) queued and waiting for resources" {
			set job_id $expect_out(1,string)
		}
		timeout {
			log_error "srun not responding"
			end_it 1
		}
		eof {
			wait
		}
	}

	if {[string compare $expected_state ""]} {
		set run_cluster [wait_for_fed_job $job_id $expected_state ""]
		if {[string compare $run_cluster ""] == 0} {
			log_error "Didn't find cluster with $expected_state job"
				end_it 1
		}
	}

	if {$check_env} {
		if {[wait_for_file $file_out]} {
			log_error "failed waiting for file '$file_out'"
			end_it 1
		}

		spawn grep SLURM_CLUSTER_NAME $file_out
		expect {
			-re "SLURM_CLUSTER_NAME=(\\S+)" {
				set submit_cluster $expect_out(1,string)
				if {$submit_cluster ne $expected_sib} {
					log_error "SLURM_CLUSTER_NAME=$submit_cluster != $expected_sib"
					end_it 1
				}
			}
			eof {
				wait
			}
		}
	}

	if {$job_id == 0} {
		log_error "srun submit failure"
		end_it 1
	}

	return [validate_fed_job $cname $job_id $expected_origin $expected_sib $submit_cluster]
}

proc submit_fed_salloc_job { cname expected_origin expected_sib extra_args check_env expected_state } {
	global fed_slurm_base file_in node_count number squeue test_id srun_job_cnt

	set submit_cluster ""
	set job_id 0
	set file_out "test${test_id}_${srun_job_cnt}.out"
	incr srun_job_cnt
	set my_salloc "${fed_slurm_base}/$cname/bin/salloc"
	set my_srun   "${fed_slurm_base}/$cname/bin/srun"
	set command "$my_salloc -N$node_count --exclusive -t3"
	if {$extra_args ne ""} {
		append command " $extra_args"
	}
	append command " $my_srun -o $file_out $file_in &"
	set salloc_pid [spawn {*}$command]
	expect {
		-re "salloc: Granted job allocation ($number)" {
			set job_id $expect_out(1,string)
		}
		-re "salloc: Pending job allocation ($number)" {
			set job_id $expect_out(1,string)
		}
		timeout {
			log_error "salloc not responding"
			end_it 1
		}
		eof {
			wait
		}
	}

	if {[string compare $expected_state ""]} {
		set run_cluster [wait_for_fed_job $job_id $expected_state ""]
		if {[string compare $run_cluster ""] == 0} {
			log_error "Didn't find cluster with $expected_state job"
				end_it 1
		}
	}

	if {$check_env} {
		if {[wait_for_file $file_out]} {
			log_error "failed waiting for file '$file_out'"
			end_it 1
		}

		spawn grep SLURM_CLUSTER_NAME $file_out
		expect {
			-re "SLURM_CLUSTER_NAME=(\\S+)" {
				set submit_cluster $expect_out(1,string)
				if {$submit_cluster ne $expected_sib} {
					log_error "SLURM_CLUSTER_NAME=$submit_cluster != $expected_sib"
					end_it 1
				}
			}
			eof {
				wait
			}
		}
	}

	if {$job_id == 0} {
		log_error "salloc submit failure"
		end_it 1
	}

	return [validate_fed_job $cname $job_id $expected_origin $expected_sib $submit_cluster]
}

proc validate_fed_job { cname job_id expected_origin expected_sib submit_cluster } {
	global fed_slurm_base file_in node_count number squeue

	set origin  ""
	set sibling ""

	sleep 3

	set my_squeue "${fed_slurm_base}/$cname/bin/squeue"
	if {$submit_cluster ne ""} {
		set my_squeue "${fed_slurm_base}/$submit_cluster/bin/squeue"
	}
	spawn $my_squeue -a --local --jobs=$job_id --noheader -Oorigin,siblingsactive
	expect {
		-re "(\\S+)\\s+(\\S+)" {
			set origin  $expect_out(1,string)
			set sibling $expect_out(2,string)
		}
	}

	log_info "origin:$origin sibling:$sibling"

	if {($expected_origin ne "") && ($origin ne $expected_origin)} {
		log_error "origin:$origin != expected_origin:$expected_origin"
		end_it 1
	}

	if {($expected_sib ne "") && ($sibling ne $expected_sib)} {
		log_error "sibling:$sibling != expected_sib:$expected_sib"
		end_it 1
	}


	# Verify that siblings have the job as well.
	foreach tmp_sib [split $sibling ","] {
		if {$tmp_sib eq  $origin} {
			continue
		}
		set my_squeue "${fed_slurm_base}/$tmp_sib/bin/squeue"
		spawn $my_squeue -a --local --jobs=$job_id --noheader -Oorigin,siblingsactive
		set match 0
		expect {
			-re "(\\S+)\\s+(\\S+)" {
				set match 1
				if {$origin ne $expect_out(1,string)} {
					log_error "origin not the same on $sibling"
					end_it 1
				}
				if {$expect_out(2,string) ne "NA" && $sibling ne $expect_out(2,string)} {
					log_error "sibling not the same on $sibling"
					end_it 1
				}
			}
			timeout {
				log_error "$my_squeue not responding"
					end_it 1
			}
			eof {
				wait
			}
		}

		if {!$match} {
			log_error "didn't find origin or sibling from job"
			end_it 1
		}
	}

	return $sibling
}

proc validate_cluster_features { cname job_id expected_features } {
	global fed_slurm_base bin_bash bin_grep

	set my_scontrol "${fed_slurm_base}/$cname/bin/scontrol"
	set matches 0
	set scontrol_pid [spawn $bin_bash -c "$my_scontrol show jobid=$job_id | $bin_grep ClusterFeatures="]
	expect {
		"ClusterFeatures=$expected_features" {
			incr matches
		}
		timeout {
			log_error "scontrol not responding"
			slow_kill $scontrol_pid
			end_it 1
		}
		eof {
			wait
		}
	}
	if {$matches != 1} {
		log_error "$matches didn't find ClusterFeatures=$expected_features on job"
		end_it 1
	}
}

if {[test_federation_setup]} {
	log_warn "WARNING: This test can't be run without fed_slurm_base,\
		fedc1, fedc2, fedc3 setup in globals.local."
	exit 0
}

if {[test_all_up]} {
	exit 0
}

set user_name [get_my_user_name]

# Remove existing setup
if {[cleanup] != 0} {
	log_error "failed to cleanup"
	end_it 1
}

# add clusters to federation
if {[setup_federation $fed_name]} {
	log_error "failed to setup federation"
	end_it 1
}

# get number of nodes per cluster
# devide by 2 to get 2 jobs per clusters
set node_count [expr [available_nodes "" ""] / 2]

make_bash_script $file_in "env; $bin_sleep 300"


# Test needs to change to count that it gets two job per cluster instead of making sure that it packs on each. Because it's not guaranteed that the cluster will pack because the job gets submitted to each cluster and it's a free for all.
send_user "\n################################################################\n"
send_user "Test packing across clusters with sbatch"
send_user "\n################################################################\n"

unset job_counts

for {set i 0} {$i < 6} {incr i} {
	set ret_sib [submit_fed_batch_job $fedc1 $fedc1 "" "" RUNNING]
	incr job_counts($ret_sib)
}
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 "" "" PENDING]
incr job_counts($ret_sib)

parray job_counts

if {$job_counts($fedc1) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc3) != 2} {
	log_error "$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc1,$fedc2,$fedc3) != 1} {
	log_error "$fedc1,$fedc2,$fedc3 don't have a pending job on it"
	end_it 1
}


send_user "\n################################################################\n"
send_user "Test packing across clusters with srun"
send_user "\n################################################################\n"
cancel_all_jobs
unset job_counts

for {set i 0} {$i < 6} {incr i} {
	set ret_sib [submit_fed_srun_job $fedc1 $fedc1 "" "" 0 RUNNING]
	incr job_counts($ret_sib)
}
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 "" "" 0 PENDING]
incr job_counts($ret_sib)

parray job_counts

if {$job_counts($fedc1) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc3) != 2} {
	log_error "$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc1,$fedc2,$fedc3) != 1} {
	log_error "$fedc1,$fedc2,$fedc3 don't have a pending job on it"
	end_it 1
}

send_user "\n################################################################\n"
send_user "Test packing across clusters with salloc"
send_user "\n################################################################\n"
cancel_all_jobs
unset job_counts

for {set i 0} {$i < 6} {incr i} {
	set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 "" "" 0 RUNNING]
	incr job_counts($ret_sib)
}
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 "" "" 0 PENDING]
incr job_counts($ret_sib)

parray job_counts

if {$job_counts($fedc1) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc3) != 2} {
	log_error "$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}
if {$job_counts($fedc1,$fedc2,$fedc3) != 1} {
	log_error "$fedc1,$fedc2,$fedc3 don't have a pending job on it"
	end_it 1
}

send_user "\n################################################################\n"
send_user "Test -M<clusters> with federated jobs with sbatch"
send_user "\n################################################################\n"
cancel_all_jobs
unset job_counts

# Submit job to only fed1
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc1 "-M$fedc1" RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc1 "-M$fedc1" RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc1 "-M$fedc1" PENDING]
incr job_counts($ret_sib);

# Submit job to only fed1,fed2
# Will go to fed2 since fed1 is full and third job should go to both
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc2 "-M$fedc1,$fedc2" RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc2 "-M$fedc1,$fedc2" RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 "$fedc1,$fedc2" "-M$fedc1,$fedc2" PENDING]
incr job_counts($ret_sib);

# Submit job to fed2,fed3.
# Should choose fed2 to be origin and submit
set ret_sib [submit_fed_batch_job $fedc1 $fedc2 $fedc3 "-M$fedc2,$fedc3" RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_batch_job $fedc1 $fedc2 $fedc3 "-M$fedc2,$fedc3" RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_batch_job $fedc1 $fedc2 "$fedc2,$fedc3" "-M$fedc2,$fedc3" PENDING]
incr job_counts($ret_sib);

parray job_counts

if {$job_counts($fedc1) != 3} {
	log_error "$fedc1 doesn't have 3 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc1,$fedc2) != 1} {
	log_error "$fedc1,$fedc2 doesn't have 1 jobs running on it";
	end_it 1
}

if {$job_counts($fedc3) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2,$fedc3) != 1} {
	log_error "$fedc2,$fedc3 doesn't have 1 jobs running on it";
	end_it 1
}

send_user "\n################################################################\n"
send_user "Test -M<clusters> with federated jobs with srun"
send_user "\n################################################################\n"
cancel_all_jobs
unset job_counts

# Submit job to only fed1
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc1 "-M$fedc1" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc1 "-M$fedc1" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc1 "-M$fedc1" 0 PENDING]
incr job_counts($ret_sib);

# Submit job to only fed1,fed2
# Will go to fed2 since fed1 is full and third job should go to both
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc2 "-M$fedc1,$fedc2" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc2 "-M$fedc1,$fedc2" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 "$fedc1,$fedc2" "-M$fedc1,$fedc2" 0 PENDING]
incr job_counts($ret_sib);

# Submit job to fed2,fed3.
# Should choose fed2 to be origin and submit
set ret_sib [submit_fed_srun_job $fedc1 $fedc2 $fedc3 "-M$fedc2,$fedc3" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_srun_job $fedc1 $fedc2 $fedc3 "-M$fedc2,$fedc3" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_srun_job $fedc2 $fedc2 "$fedc2,$fedc3" "-M$fedc2,$fedc3" 0 PENDING]
incr job_counts($ret_sib);

parray job_counts

if {$job_counts($fedc1) != 3} {
	log_error "$fedc1 doesn't have 3 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc1,$fedc2) != 1} {
	log_error "$fedc1,$fedc2 doesn't have 1 jobs running on it";
	end_it 1
}

if {$job_counts($fedc3) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2,$fedc3) != 1} {
	log_error "$fedc2,$fedc3 doesn't have 1 jobs running on it";
	end_it 1
}


send_user "\n################################################################\n"
send_user "Test -M<clusters> with federated jobs with salloc"
send_user "\n################################################################\n"
cancel_all_jobs
unset job_counts

# Submit job to only fed1
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc1 "-M$fedc1" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc1 "-M$fedc1" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc1 "-M$fedc1" 0 PENDING]
incr job_counts($ret_sib);

# Submit job to only fed1,fed2
# Will go to fed2 since fed1 is full and third job should go to both
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc2 "-M$fedc1,$fedc2" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc2 "-M$fedc1,$fedc2" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 "$fedc1,$fedc2" "-M$fedc1,$fedc2" 0 PENDING]
incr job_counts($ret_sib);

# Submit job to fed2,fed3.
# Should choose fed2 to be origin and submit
set ret_sib [submit_fed_salloc_job $fedc1 $fedc2 $fedc3 "-M$fedc2,$fedc3" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_salloc_job $fedc1 $fedc2 $fedc3 "-M$fedc2,$fedc3" 1 RUNNING]
incr job_counts($ret_sib);
set ret_sib [submit_fed_salloc_job $fedc2 $fedc2 "$fedc2,$fedc3" "-M$fedc2,$fedc3" 0 PENDING]
incr job_counts($ret_sib);

parray job_counts

if {$job_counts($fedc1) != 3} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc1,$fedc2) != 1} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc3) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2,$fedc3) != 1} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}


send_user "\n################################################################\n"
send_user "Test jobs with cluster features"
send_user "\n################################################################\n"
cancel_all_jobs

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $fedc1 set features=fa]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fa" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr mod not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $fedc2 set features=fb]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fb" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr mod not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $fedc3 set features=fc]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fc" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr mod not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

send_user "\n################################################################\n"
send_user "Test sbatch jobs with cluster features"
send_user "\n################################################################\n"
unset job_counts

set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc3 "--cluster-constraint=fc" RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc3 "--cluster-constraint=fc" RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc2 "--cluster-constraint=fb" RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc2 "--cluster-constraint=fb" RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc1 "--cluster-constraint=fa" RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 $fedc1 "--cluster-constraint=fa" RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_batch_job $fedc1 $fedc1 "$fedc1,$fedc2,$fedc3" "--cluster-constraint=fa,fb,fc" PENDING]
incr job_counts($ret_sib)

parray job_counts

if {$job_counts($fedc1) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc3) != 2} {
	log_error "$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc1,$fedc2,$fedc3) != 1} {
	log_error "$fedc1,$fedc2,$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}

send_user "\n################################################################\n"
send_user "Test salloc jobs with cluster features"
send_user "\n################################################################\n"
cancel_all_jobs
unset job_counts

set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc3 "--cluster-constraint=fc" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc3 "--cluster-constraint=fc" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc2 "--cluster-constraint=fb" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc2 "--cluster-constraint=fb" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc1 "--cluster-constraint=fa" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 $fedc1 "--cluster-constraint=fa" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_salloc_job $fedc1 $fedc1 "$fedc1,$fedc2,$fedc3" "--cluster-constraint=fa,fb,fc" 0 PENDING]
incr job_counts($ret_sib)

parray job_counts

if {$job_counts($fedc1) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc3) != 2} {
	log_error "$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc1,$fedc2,$fedc3) != 1} {
	log_error "$fedc1,$fedc2,$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}

send_user "\n################################################################\n"
send_user "Test srun jobs with cluster features"
send_user "\n################################################################\n"
cancel_all_jobs
unset job_counts

set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc3 "--cluster-constraint=fc" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc3 "--cluster-constraint=fc" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc2 "--cluster-constraint=fb" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc2 "--cluster-constraint=fb" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc1 "--cluster-constraint=fa" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 $fedc1 "--cluster-constraint=fa" 0 RUNNING]
incr job_counts($ret_sib)
set ret_sib [submit_fed_srun_job $fedc1 $fedc1 "$fedc1,$fedc2,$fedc3" "--cluster-constraint=fa,fb,fc" 0 PENDING]
incr job_counts($ret_sib)

parray job_counts

if {$job_counts($fedc1) != 2} {
	log_error "$fedc1 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc2) != 2} {
	log_error "$fedc2 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc3) != 2} {
	log_error "$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}

if {$job_counts($fedc1,$fedc2,$fedc3) != 1} {
	log_error "$fedc1,$fedc2,$fedc3 doesn't have 2 jobs running on it";
	end_it 1
}

send_user "\n################################################################\n"
send_user "Test jobs with invalid cluster features"
send_user "\n################################################################\n"
set matches 0
set my_sbatch "${fed_slurm_base}/$fedc1/bin/sbatch"
set sbatch_pid [spawn $my_sbatch --output=/dev/null --error=/dev/null -n1 -t3 --cluster-constraint=invalid $file_in]
expect {
	"sbatch: error: Batch job submission failed: Invalid cluster feature specification" {
		incr matches
	}
	timeout {
		log_error "sbatch not responding"
		slow_kill $sbatch_pid
		end_it 1
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	send_user "$matches FAILURE: invalid cluster features sbatch didn't fail.\n"
	end_it 1
}

# All Done
end_it 0
