#!/usr/bin/env expect
############################################################################
# Purpose:  Test gres.conf-specified and system-detected GPU device merging
#           logic
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2018 SchedMD LLC
# Written by Michael Hinton
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id		"39.18"
set exit_code		0
set fs1_err		"error: This GPU record was in gres.conf, but not found on the system."
set test_prog		"test$test_id.prog"
set cfgdir		"test${test_id}_configs"
set cfgpath		"[pwd]/$cfgdir"
set nodename_base	"tux"
set nodename		"${nodename_base}0"
set nodename_diff	"${nodename_base}1"
set test_ulong_prog	"test$test_id.ulong"
set file_in		"test$test_id.bash"
set file_out		"test$test_id.out"
# Assume unsigned long is 64 bits unless overwritten
set ulong_bits  64

print_header $test_id

if {[test_cray]} {
#	Many Cray-specific changes required in slurm.conf test files
	send_user "\nWARNING: This test can not run on a Cray system\n"
	exit $exit_code
}

# Run the test under valgrind if any arguments are given
set use_valgrind 0
set test_minor_in ""
if {[lindex $argv 0] == "v" || [lindex $argv 0] == "valgrind"} {
	set use_valgrind 1
	send_user "Running tests under valgrind!\n"
} elseif {[lindex $argv 0] != ""} {
	set test_minor_in [lindex $argv 0]
	send_user "Running test $test_id.$test_minor_in \n"
}

proc mkdir {dir} {
	exec mkdir -p $dir
}

proc touch_file {file} {
	exec touch $file
}

proc generate_file {text filepath} {
	exec echo $text > $filepath
}

# Create or overwrite the slurm.conf file used by the test runner program
proc create_slurm_conf {fast_schedule} {
	global test_id cfgpath
	# Contain slurm.conf data to this file
	# This is all we need to trigger loading the GRES GPU plugin
	set slurm_conf "
	# This file was autogenerated by test$test_id
	ControlMachine=test_machine
	ClusterName=test_cluster
	GresTypes=gpu,mps,nic,mic,tmpdisk
	FastSchedule=$fast_schedule
	"

	# NOTE: There are no node definitions - node name and node gres info
	# are passed into the test program directly, not through this slurm.conf

	generate_file $slurm_conf $cfgpath/slurm.conf
}

mkdir $cfgpath
# Set up dummy device files for testing. They just need to exist
set dev "$cfgpath/nvidia"
set dev0 "${dev}0"
set dev1 "${dev}1"
set dev2 "${dev}2"
set dev3 "${dev}3"
set dev4 "${dev}4"
set dev5 "${dev}5"
set dev6 "${dev}6"
set dev7 "${dev}7"
set dev8 "${dev}8"
touch_file $dev0
touch_file $dev1
touch_file $dev2
touch_file $dev3
touch_file $dev4
touch_file $dev5
touch_file $dev6
touch_file $dev7
touch_file $dev8


# Build test program
set build_cmd "$bin_cc ${test_prog}.c -g -pthread -export-dynamic -o ${test_prog} -I${src_dir} -I${build_dir} ${build_dir}/src/api/libslurm.o -ldl"
if {$use_valgrind} {
	set build_cmd "$build_cmd -DUSING_VALGRIND"
}
send_user "$build_cmd\n"
eval exec $build_cmd


# Build program to determine size of unsigned long in C
set build_cmd "$bin_cc ${test_ulong_prog}.c -o ${test_ulong_prog}"
send_user "$build_cmd\n"
eval exec $build_cmd

spawn ./$test_ulong_prog
expect {
	-re "($number)" {
		set ulong_bits $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "FAILURE: No response from $test_ulong_prog\n"
		exit 1
	}
	eof {
		wait
	}
}

send_user "sizeof(unsigned long) = $ulong_bits\n"


#
# Tests if $test_prog actual output matches expected output
# NOTE: The output strings are sorted so the order of lines is irrelevant
#
# test_minor -		The minor test number (39.18.X). If global variable
# 			"$test_minor_in" is set, then run only that test.
# gres_conf -		The gres.conf to use.
# fake_gpus_conf -	The fake_gpus.conf to use. This file tells Slurm to
# 			pretend to have devices detected by system, in effect
# 			emulating the result of AutoDetect=nvml.
# output_expected -	The expected "GRES_PARSABLE" output.
# err_msgs (optional) -	The error message regex to look for. Use | for multiple.
# 			If using with $output_expected, it will only reliably
# 			match if $err_msgs appear BEFORE lines with
# 			GRES_PARSABLE.
# errs_expected (optional) - The number of times you expect to see $err_msgs in
# 			the output. Defaults to 0. If this number isn't hit,
# 			then the test will fail.
# slurm_conf_gres (optional) - This is the `Gres=` value that would be in
# 			slurm.conf for this node (it's easier to pass it into
# 			the program directly than to regenerate a slurm.conf
# 			file with different node definitions).
#
set failed_tests [list]
proc check_configuration {test_minor gres_conf fake_gpus_conf output_expected {err_msgs ""} {errs_expected 0} {slurm_conf_gres ""}} {
	global cfgpath nodename test_prog alpha_numeric_special exit_code test_minor_in
	global test_id failed_tests use_valgrind bin_sort file_in file_out

	# Skip all tests not specified by the tester; force string comparison
	if {$test_minor_in != "" && [string compare "$test_minor" "$test_minor_in"] != 0} {
		return
	}

	set failed 0
	send_user "\n====test$test_id.$test_minor====\n\n"

	generate_file $gres_conf $cfgpath/gres.conf
	generate_file $fake_gpus_conf $cfgpath/fake_gpus.conf

	if {$use_valgrind} {
		make_bash_script $file_in "
valgrind --tool=memcheck --error-limit=no --leak-check=full --show-reachable=yes --max-stackframe=16777216 --num-callers=20 --child-silent-after-fork=yes --track-origins=yes ./$test_prog \"$cfgpath\" \"$nodename\" \"$slurm_conf_gres\" >./$file_out 2>&1
$bin_sort <./$file_out
rm -rf ./$file_out"
	} else {
		make_bash_script $file_in "
./$test_prog \"$cfgpath\" \"$nodename\" \"$slurm_conf_gres\" >./$file_out 2>&1
$bin_sort <./$file_out
rm -rf ./$file_out"
	}

	set sorted_actual ""
	set errs_actual 0
	set ignore_err_msgs 0
	# Doing a regex on an empty string stalls expect. So don't do it.
	if {$err_msgs == ""} {
		set err_msgs "bogus"
		set ignore_err_msgs 1
	}
	log_user 0
	spawn ./$file_in


	# Note that $err_msgs is checked before GRES_PARSABLE. This is to
	# prevent expect from throwing away $err_msgs if it comes before
	# GRES_PARSABLE in the same stdout chunk that expect is processing.
	# When expect matches something in a chunk, if exp_continue is called,
	# it throws away everything before the match and then continues
	# searching in the chunk.
	# So we are making the brittle assumption that all $err_msgs will come
	# before any GRES_PARSABLE lines.
	expect {
		-re "($err_msgs)" {
			send_user "Found '$err_msgs'\n"
			if {$ignore_err_msgs == 0} {
				incr errs_actual
			}
			exp_continue
		}
		-re "(GRES_PARSABLE$alpha_numeric_special)" {
			if {$sorted_actual == ""} {
				set sorted_actual "\n"
			}
			# Append string
			set sorted_actual "$sorted_actual$expect_out(1,string)\n"
			exp_continue
		}
		timeout {
			send_user "FAILURE: No response from $test_prog\n"
			set failed 1
		}
		eof {
			wait
		}
	}

	set sorted_expected ""
	make_bash_script $file_in "/bin/echo \"$output_expected\" | $bin_sort"
	spawn ./$file_in
	expect {
		-re "(GRES_PARSABLE$alpha_numeric_special)" {
			if {$sorted_expected == ""} {
				set sorted_expected "\n"
			}
			# Append string
			set sorted_expected "$sorted_expected$expect_out(1,string)\n"
			exp_continue
		}
		timeout {
			send_user "FAILURE: No response from $test_prog\n"
			set failed 1
		}
		eof {
			wait
		}
	}
	file delete ./$file_in
	log_user 1

	# Check the output with what is expected
	send_user "Actual (sorted): $sorted_actual\n"
	if {$sorted_actual != $sorted_expected} {
		send_user "Expected (sorted): $sorted_expected\n"
		log_error "Expected output does not match actual output for sub-test $test_minor"
		set failed 1
	} else {
		log_info "(This is OK) Expected output does match actual output for sub-test $test_minor"
	}

	#  Check for expected errors, if applicable
	if {$err_msgs != "bogus"} {
		set response "Errors: Expected $errs_expected, found $errs_actual for error regex '$err_msgs' in sub-test $test_minor"
		if {$errs_actual == $errs_expected} {
			set response "(This is OK) ${response}"
			log_info $response
		} else {
			log_error $response
			set failed 1
		}
	}

	if {$failed == 1} {
		set exit_code 1
		lappend failed_tests $test_minor
	}
}

################################################################################
# NOTES:
################################################################################
# fake_gpus.conf is of the following format, with each line representing one
# GPU device:
#
#     type|cpu_count|cpu_range|links|device_files
#
# Optionally, cpu_range allows special keywords starting in `~` to aid in
# testing the conversion from an NVML cpu affinity array to Slurm bitstring.
#
# The GRES_PARSABLE output is of the following format:
#
# GRES_PARSABLE[name](device_count):type|cpu_count|cpu_range|links|device_files
#
# To better decipher the tests, temporarily increase the log verbosity in
# test39.18.prog.c. E.g.:
#
#     opts.stderr_level = LOG_LEVEL_DEBUG2;
#
# Note that there is a known issue with Expect where the regex parsing fails
# when the output is somewhat large. So to avoid random errors, each test output
# should be kept concise.
#
################################################################################



# ##############################################################################
# ------------------------------------------------------------------------------
# Tests for FastSchedule = 0
# ------------------------------------------------------------------------------
# ##############################################################################
create_slurm_conf 0

# ##############################################################################
# # Test 0.2 - Empty gres.conf
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
tesla|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}4|
"
check_configuration 0.2 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.4 - Test empty and null identifiers in fake_gpus.conf
# 	       Also test that GPU and MPS are sorted by filename
# 	       Note: MPS ignores lines with empty files and nulls out links
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|(null)|(null)|(null)
tesla|4|(null)|(null)|${dev}1
tesla|4|(null)|0-1|(null)
tesla|4|(null)|0-1|${dev}3
tesla|4|0-1|(null)|(null)
tesla|4|0-1|(null)|${dev}2
tesla|4|0-1|0-1|(null)
tesla|4|0-1|0-1|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|0-1|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|0-1|(null)|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|0-1|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|0-1|${dev}4|
"
check_configuration 0.4 $gres_conf $fake_gpus_conf $expected_output



# ##############################################################################
# The following tests are NVML-specific.
# They will only execute if HAVE_NVML = 1 in config.h.
# In fake_gpus_conf, a CPU range starting with `~` will trigger the GPU plugin's
# gpu_p_test_cpu_conv(). In order for this to exercise gpu/nvml-specific code,
# explicitly specify `AutoDetect=nvml` in the ad-hoc gres.conf. If not,
# gpu/generic will be used and CPU ranges will be set to null, failing the tests
# ##############################################################################

if {[have_nvml] == 1} {
	if {$test_minor_in == ""} {
		log_info "Running NVML-specific tests:"
	}

# ##############################################################################
# # Test 0.6 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       Convert device CPU affinity of 20 ulongs with all bits set
# ##############################################################################

set arr_len 20
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.6 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.8 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       Convert device CPU affinity of 1 ulong with all bits set
# ##############################################################################

set arr_len 1
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.8 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.10 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity of 3 ulongs with all bits set
# ##############################################################################

set arr_len 3
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.10 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.12 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for max # of CPUs supported (2^15)
# ##############################################################################

set max_cpus [expr pow(2, 15)]
set arr_len [expr int($max_cpus/$ulong_bits)]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~max|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.12 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.14 - Test conversion of NVML-style CPU affinity array to Slurm bitstr
# 		Convert device CPU affinity for all zeros (expect this to fatal)
# ##############################################################################

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|0|~zero|(null)|${dev}1
"
set expected_output ""
set err_msgs "fatal:"
set expected_errs 1
check_configuration 0.14 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 0.16 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPU 0 set out of 16 total CPUs
# ##############################################################################

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~one|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|16|0|(null)|${dev}1|
"
check_configuration 0.16 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.18 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPUs 0-1 set out of 4 total CPUs
# ##############################################################################

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|~three|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|
"
check_configuration 0.18 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.20 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 		Convert device CPU affinity for CPUs 8-15 set out of 16 CPUs
# ##############################################################################

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~half|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|16|8-15|(null)|${dev}1|
"
check_configuration 0.20 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# End NVML-specific tests
# ##############################################################################

} else {
	send_user "\n====Omitting NVML-specific tests====\n\n"
}

# ##############################################################################
# # Test 0.22 - Test that when no GRES type is specified in slurm.conf,
#               the GRES type is NOT set to what the system detects, but rather
#               is set to NULL.
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}4|
"
check_configuration 0.22 $gres_conf $fake_gpus_conf $expected_output "" 0 "gpu:4"

# ##############################################################################
# # Test 0.24 - Test that the GRES type specified in slurm.conf overrides the
#               GRES type specified in the system, if it's a substring.
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):v100|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):v100|4|2-3|(null)|${dev}4|
"
check_configuration 0.24 $gres_conf $fake_gpus_conf $expected_output "" 0 "gpu:v100:4"


# ##############################################################################
# # Test 0.26 - Test that the GRES type specified in slurm.conf doesn't need
#               to match the case of the system-detected GRES type, and that it
#               can also match the first part.
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):TESLA_V100|4|2-3|(null)|${dev}4|
"
check_configuration 0.26 $gres_conf $fake_gpus_conf $expected_output "" 0 "gpu:TESLA_V100:4"


# ##############################################################################
# # Test 0.28 - Test that the GRES type specified in slurm.conf can match the
#               last part of the system-detected GRES type
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):pcie-16gb|4|2-3|(null)|${dev}4|
"
check_configuration 0.28 $gres_conf $fake_gpus_conf $expected_output "" 0 "gpu:pcie-16gb:4"

# ##############################################################################
# # Test 0.30 - Test that the GRES type specified in slurm.conf can match the
#               full system-detected GRES type
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3
tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4
tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla_v100-pcie-16gb|4|2-3|(null)|${dev}4|
"
check_configuration 0.30 $gres_conf $fake_gpus_conf $expected_output "" 0 "gpu:tesla_v100-pcie-16gb:4"

# ##############################################################################
# # Test 0.32 - Test that GRES types work as expected when one GRES type is a
#               substring of another GRES type (e.g. k20m + k20m1). See bug 7345
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla_k20m1|4|0-1|(null)|${dev}1
tesla_k20m|4|0-1|(null)|${dev}2
tesla_v100-sxm2-32gb|4|0-1|(null)|${dev}3
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):k20m1|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):v100|4|0-1|(null)|${dev}3|
"
check_configuration 0.32 $gres_conf $fake_gpus_conf $expected_output "" 0 "gpu:k20m:1,gpu:k20m1:1,gpu:v100:1"

# ##############################################################################
# # Test 0.34 - Test that only the first system device that matches a conf gres
#               type of count 1 has gres type synced (rather than any that *can*
#               match).
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
p100|4|0-1|(null)|${dev}2
p100|4|0-1|(null)|${dev}6
p100|4|0-1|(null)|${dev}3
p100|4|0-1|(null)|${dev}4
p100|4|0-1|(null)|${dev}1
p100|4|0-1|(null)|${dev}5
"
set expected_output "
GRES_PARSABLE\[gpu\](1):p100|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}6|
"
check_configuration 0.34 $gres_conf $fake_gpus_conf $expected_output "" 0 "gpu:p100:1"

# ##############################################################################
# # Test 0.36 - Test that the shorter GRES types don't match against the longer
#               GRES types.
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
aa|4|0-1|(null)|${dev}2
aaa|4|0-1|(null)|${dev}1
a|4|0-1|(null)|${dev}3
aaaa|4|0-1|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):aa|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):a|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):aaaa|4|0-1|(null)|${dev}4|
"
set slurm_conf_gres "gpu:aaaa:1,gpu:a:1,gpu:aa:1,gpu:aaa:1"

check_configuration 0.36 $gres_conf $fake_gpus_conf $expected_output "" 0 $slurm_conf_gres

# ##############################################################################
# # Test 0.38 - Test that the shorter GRES types don't match against the longer
#               GRES types.
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
k20|4|0-1|(null)|${dev}2
k20m|4|0-1|(null)|${dev}1
k20|4|0-1|(null)|${dev}3
k20m|4|0-1|(null)|${dev}4
k20|4|0-1|(null)|${dev}5
k20m|4|0-1|(null)|${dev}7
k20|4|0-1|(null)|${dev}6
k20m|4|0-1|(null)|${dev}8
"
set expected_output "
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):k20|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):k20m|4|0-1|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}6|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}8|
"
set slurm_conf_gres "gpu:k20:2,gpu:k20m:3"

check_configuration 0.38 $gres_conf $fake_gpus_conf $expected_output "" 0 $slurm_conf_gres


# ##############################################################################
# # Test 0.40 - Test that GRES types that are the same length get sorted
#               properly
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
test-bbb-test|4|0-1|(null)|${dev}3
test-aaa-test|4|0-1|(null)|${dev}2
test-ccc-test|4|0-1|(null)|${dev}5
test-aaa-test|4|0-1|(null)|${dev}1
test-bbb-test|4|0-1|(null)|${dev}4
test-ccc-test|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):bbb|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):bbb|4|0-1|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):ccc|4|0-1|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}6|
"
set slurm_conf_gres "gpu:aaa:3,gpu:bbb:2,gpu:ccc:1"

check_configuration 0.40 $gres_conf $fake_gpus_conf $expected_output "" 0 $slurm_conf_gres


# ##############################################################################
# # Test 0.42 - Test multiple identical GRES type specifications
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
test-aaa-test|4|0-1|(null)|${dev}4
test-aaa-test|4|0-1|(null)|${dev}3
test-aaa-test|4|0-1|(null)|${dev}2
test-aaa-test|4|0-1|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):aaa|4|0-1|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-1|(null)|${dev}4|
"
set slurm_conf_gres "gpu:aaa:1,gpu:aaa:1,gpu:aaa:1"

check_configuration 0.42 $gres_conf $fake_gpus_conf $expected_output "" 0 $slurm_conf_gres


# ##############################################################################
# ------------------------------------------------------------------------------
# Tests for FastSchedule = 1
# ------------------------------------------------------------------------------
# ##############################################################################
create_slurm_conf 1

# ##############################################################################
# # Test 1.2 - Test that all GPUs found on the system match gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 CPUs=0,1
Name=gpu Type=1080ti File=${dev}2 COREs=0,1
Name=mps Count=100 File=${dev}1
Name=mps Count=100 File=${dev}0
"

set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
1080ti|4|0-1|(null)|${dev}2
gtx560|4|0-1|(null)|${dev}0
"
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0-1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):1080ti|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[mps\](100):gtx560|4|0-1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):1080ti|4|0-1|(null)|${dev}2|
"

check_configuration 1.2 $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 1.4 - Test that errors are emitted when devices specified in
# 	       gres.conf are not found on the system
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=1080ti File=${dev}\[0-2\] COREs=0,1
"

set fake_gpus_conf "
# This file was autogenerated by test$test_id
1080ti|4|0-1|(null)|${dev}1
1080ti|4|0-1|(null)|${dev}2
"
# We're missing 2 of the 3 devices. See if that is detected

set expected_output "
GRES_PARSABLE\[gpu\](1):1080ti|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):1080ti|4|0-1|(null)|${dev}2|
"

set err_msgs $fs1_err
set expected_errs 1
check_configuration 1.4 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# # Test 1.6 - Test that errors are emitted when there are no devices on system
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=1080ti File=${dev}\[0-2\] COREs=0,1
"

set fake_gpus_conf ""
set expected_output ""

set err_msgs $fs1_err
set expected_errs 3
check_configuration 1.6 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs


# ##############################################################################
# # Test 1.8 - Test that the correct number of errors is emitted when no devices
# 	       are found on the system
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}\[0-1\] COREs=0,1
Name=gpu Type=tesla File=${dev}\[2-3\] CPUs=0,1 Links=0,1
Name=gpu Type=1080ti File=${dev}2 COREs=0,1
"

set fake_gpus_conf ""
set expected_output ""

set err_msgs $fs1_err
set expected_errs 5

check_configuration 1.8 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 1.9 - Test basic auto-fill of a single record
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu
"

set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|2-3|(null)|${dev}2
"

set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}2|
"

check_configuration 1.9 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 1.10 - Test basic auto-fill of a single record with Type specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla
"

set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|2-3|(null)|${dev}2
"

set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
"

check_configuration 1.10 $gres_conf $fake_gpus_conf $expected_output


# TODO: We need to create tests and modify behavior to do the following:
# 	* If Count only is specified, have Slurm grab the first GPUs it can find
# 	and fill in the rest of the info.
# 	* If there is a mix and match of Type, File, Links, or none of these,
# 	have it properly autofill



# # ##############################################################################
# # # Test 1.12 - Test that count automatically takes the first GPUs it finds
# # TODO: This functionality has not been coded yet
# # ##############################################################################

# set gres_conf "
# # This file was autogenerated by test$test_id
# Name=gpu Count=3
# "

# set fake_gpus_conf "
# # This file was autogenerated by test$test_id
# tesla|4|2-3|(null)|${dev}2
# "

# set expected_output "
# GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
# "

# check_configuration 1.12 $gres_conf $fake_gpus_conf $expected_output


# # ##############################################################################
# # # Test 1.14 - Test advanced auto-fill of multiple records with diff fields
# # TODO: This functionality has not been coded yet
# # ##############################################################################

# set gres_conf "
# # This file was autogenerated by test$test_id
# Name=gpu
# Name=gpu            File=${dev}3
# Name=gpu            File=${dev}4 CPUs=0,1
# Name=gpu Type=tesla
# Name=gpu Type=tesla              CPUs=0,1
# Name=gpu Type=tesla File=${dev}7
# Name=gpu Type=tesla File=${dev}8 CPUs=0,1
# "

# set fake_gpus_conf "
# # This file was autogenerated by test$test_id
# tesla|4|2-3|(null)|${dev}2
# "

# set expected_output "
# GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
# "

# check_configuration 1.14 $gres_conf $fake_gpus_conf $expected_output
 

# ##############################################################################
# ------------------------------------------------------------------------------
# Tests for FastSchedule = 2
# ------------------------------------------------------------------------------
# ##############################################################################
create_slurm_conf 2




# ##############################################################################
# # Test 2.2 - Test gres/gpu plus gres/mps with count
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}1
Name=mps Count=100 File=${dev}0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.2 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.4 - Test gres/mps with count and no file names
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=mps Count=210
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](105):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](105):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.4 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.6 - Explicit device specification overrules system detected
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3
Name=gpu Type=tesla File=${dev}1 Cores=0-1
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|0-1|(null)|${dev}2
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.6 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.8 - Different system devices detected; single devices merged
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=2-3
Name=gpu Type=tesla File=${dev}2 Cores=2-3
Name=gpu Type=tesla File=${dev}3 Cores=2-3
Name=gpu Type=tesla File=${dev}4 Cores=2-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}5
tesla|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.8 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.10 - Different links, different records
# ##############################################################################

# Devices 0-2 are all doubly linked to each other
# Device 5 is singly linked to 3-4
# Devices 7-8 are doubly linked to each other
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}0 Cores=0-3 Links=-1,2,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=2,-1,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=2,2,-1,0,0,0,0,0
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,0,0,-1,0,1,0,0
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,0,0,0,-1,1,0,0
Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,0,0,1,1,-1,0,0
Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,0,0,0,0,0,-1,2
Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,0,0,0,0,0,2,-1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|-1,2,2,0,0,0,0,0|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,-1,2,0,0,0,0,0|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,2,-1,0,0,0,0,0|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,-1,0,1,0,0|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,-1,1,0,0|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,1,1,-1,0,0|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,-1,2|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,2,-1|${dev}7|
"
check_configuration 2.10 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.12 - Empty everything
# ##############################################################################

set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
check_configuration 2.12 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.14 - Empty system devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.14 $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 2.16 - Test that non-GPU GRESs are excluded
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     Type=tesla File=${dev}1 Cores=0-3
Name=tmpdisk Type=tesla File=${dev}2 Cores=0-3
Name=nic     Type=tesla File=${dev}3 Cores=0-3
Name=mic     Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[tmpdisk\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[nic\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mic\](1):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.16 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.18 - Test NodeName when Name isn't specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu     Type=tesla File=${dev}1 Cores=0-3
NodeName=$nodename              Type=tesla File=${dev}2 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
"
check_configuration 2.18 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.20 - Test that empty Name does not parse and produces error
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
check_configuration 2.20 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.22 - Ensure no malloc error for large count with ignored non-GPU GRES
#             See bug 6014
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=tmpdisk Count=10g
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](10737418240):(null)|4|(null)|(null)|(null)|
"
check_configuration 2.22 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.24 - Different types, different records
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=a File=${dev}1 Cores=0-3
Name=gpu Type=a File=${dev}2 Cores=0-3
Name=gpu Type=b File=${dev}5 Cores=0-3
Name=gpu Type=b File=${dev}6 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}6|
"
check_configuration 2.24 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.26 - Same gres.conf and system devices is fine
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.26 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.28 - Test alternating device/cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla_a File=${dev}5       Cores=0-3
Name=gpu Type=tesla_b File=${dev}6       Cores=0-3
Name=gpu Type=tesla_a File=${dev}7       Cores=0-3
Name=gpu Type=tesla_b File=${dev}8       Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}8|
"
check_configuration 2.28 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.30 - Test duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.30 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.32 - Test increasing duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1       Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.32 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.34 - Test decreasing duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.34 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.36 - Test duplicate devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}1 Cores=0-1
Name=gpu Type=tesla File=${dev}1 Cores=0-2
Name=gpu Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.36 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.38 - Test overlapping cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}2 Cores=0-1
Name=gpu Type=tesla File=${dev}3 Cores=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-2|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.38 $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 2.40 - Work with null links
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=\"\"
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=null
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0|${dev}4|
"
check_configuration 2.40 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.42 - Show link ranges do NOT work - only commas
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,-1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,-1,2
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1,2|${dev}4|
"
check_configuration 2.42 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.44 - What does 0, "", and "null" Type do
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
# Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3 Type=\"\"
Name=gpu File=${dev}3 Cores=0-3 Type=null
Name=gpu File=${dev}4 Cores=0-3 Type=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):null|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):0|4|0-3|(null)|${dev}4|
"
check_configuration 2.44 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.46 - Type not specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3
Name=gpu File=${dev}3 Cores=0-3
Name=gpu File=${dev}4 Cores=0-3

"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}4|
"
check_configuration 2.46 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.48 - Cores not specified or null
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1
Name=gpu Type=tesla File=${dev}2
Name=gpu Type=tesla File=${dev}3 Cores=\"\"
Name=gpu Type=tesla File=${dev}4 Cores=null
Name=gpu Type=tesla File=${dev}5 Cores=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4||(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|null|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}5|
"
check_configuration 2.48 $gres_conf $fake_gpus_conf $expected_output


# TODO: What other tests should we add?
# Make sure current gres.conf files still work as expected
# Test to make sure CPU affinity is correct? Machine vs abstract?
# Invalid CPU counts
# Invalid cpu range

# ##############################################################################
# # Test 2.50 - Test examples in gres.conf docs
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}0
Name=mps Count=100 File=${dev}1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.50 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.52 - Test examples in gres.conf docs
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-1\] COREs=0,1
# NOTE: nvidia2 device is out of service
# Name=gpu Type=tesla File=${dev}\[2-3\] COREs=2,3
Name=gpu Type=tesla File=${dev}3 COREs=2,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|
"
check_configuration 2.52 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.54 - Test examples in gres.conf docs
# #	      Note: ${nodename_base}0 == $nodename, so only that node 0's GRES
# #	      definitions will apply
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
## Explicitly specify devices on nodes ${nodename_base}0-${nodename_base}15
# NodeName=${nodename_base}\[0-15\] Name=gpu File=${dev}\[0-3\]
# NOTE: ${nodename_base}3 ${dev}1 device is out of service
NodeName=${nodename_base}\[0-2\] Name=gpu File=${dev}\[0-3\]
NodeName=${nodename_base}3 Name=gpu File=${dev}\[0,2-3\]
NodeName=${nodename_base}\[4-15\] Name=gpu File=${dev}\[0-3\]
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}3|
"
check_configuration 2.54 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.56 - Test gres/mps with count on only some devices
# # NOTE the device numbers being out of order, one GPU without a gres/mps and
# #      a gres/mps with a device file not valid for any configured GPU
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}3 COREs=2,3
Name=mps Count=200 File=${dev}1
Name=mps Count=300 File=${dev}3
Name=mps Count=400 File=${dev}8
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](200):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](300):tesla|4|2,3|(null)|${dev}3|
"
check_configuration 2.56 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Tests for expected failures
# ##############################################################################



# ##############################################################################
# # Cleanup and finish
# ##############################################################################

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
} else {
	send_user "FAILURE: Tests $test_id.\["
	set length [llength $failed_tests]
	set i 0
	foreach test_minor $failed_tests {
		set i [expr $i+1]
		send_user "$test_minor"
		if {$i < $length} {
			send_user ","
		}
	}
	send_user "\] failed\n"
	if {$test_minor_in == ""} {
		send_user "To run an individual sub-test, do `expect test$test_id <sub-test>`\n"
	}

}

# Delete generated files
file delete $cfgpath/gres.conf
file delete $cfgpath/fake_gpus.conf
file delete $cfgpath/slurm.conf
file delete $test_prog
file delete $test_ulong_prog
file delete $dev0
file delete $dev1
file delete $dev2
file delete $dev3
file delete $dev4
file delete $dev5
file delete $dev6
file delete $dev7
file delete $dev8

# Delete generated folders
file delete $cfgpath

exit $exit_code
