#!/usr/bin/env expect
############################################################################
# Purpose:  Test gres.conf-specified and system-detected GPU device merging
#           logic
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2018 SchedMD LLC
# Written by Michael Hinton
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id		"39.18"
set exit_code		0
set test_prog		"test$test_id.prog"
set cfgdir		"test${test_id}_configs"
set cfgpath		"[pwd]/$cfgdir"
set nodename_base	"tux"
set nodename		"${nodename_base}0"
set nodename_diff	"${nodename_base}1"
set test_ulong_prog	"test$test_id.ulong"
set fast_schedule	-1
# Assume unsigned long is 64 bits unless overwritten
set ulong_bits  64
print_header $test_id

# Run the test under valgrind if any arguments are given
set use_valgrind 0
set test_minor_in ""
if {[lindex $argv 0] == "v" || [lindex $argv 0] == "valgrind"} {
	set use_valgrind 1
	send_user "Running tests under valgrind!\n"
} elseif {[lindex $argv 0] != ""} {
	set test_minor_in [lindex $argv 0]
	send_user "Running test $test_id.$test_minor_in \n"
}

proc mkdir {dir} {
	exec mkdir -p $dir
}

proc touch_file {file} {
	exec touch $file
}

proc generate_file {text filepath} {
	exec echo $text > $filepath
}

# Create or overwrite the slurm.conf file used by the test runner program
proc create_slurm_conf {} {
	global fast_schedule test_id cfgpath
	# Contain slurm.conf data to this file
	# This is all we need to trigger loading the GRES GPU plugin
	set slurm_conf "
	# This file was autogenerated by test$test_id
	ControlMachine=test_machine
	ClusterName=test_cluster
	GresTypes=gpu,mps,nic,mic,tmpdisk
	FastSchedule=$fast_schedule
	"

	generate_file $slurm_conf $cfgpath/slurm.conf
}

mkdir $cfgpath
# Set up dummy device files for testing. They just need to exist
set dev "$cfgpath/nvidia"
set dev0 "${dev}0"
set dev1 "${dev}1"
set dev2 "${dev}2"
set dev3 "${dev}3"
set dev4 "${dev}4"
set dev5 "${dev}5"
set dev6 "${dev}6"
set dev7 "${dev}7"
set dev8 "${dev}8"
touch_file $dev0
touch_file $dev1
touch_file $dev2
touch_file $dev3
touch_file $dev4
touch_file $dev5
touch_file $dev6
touch_file $dev7
touch_file $dev8


# Build test program
set build_cmd "$bin_cc ${test_prog}.c -g -pthread -export-dynamic -o ${test_prog} -I${src_dir} -I${build_dir} ${build_dir}/src/api/libslurm.o -ldl"
if {$use_valgrind} {
	set build_cmd "$build_cmd -DUSING_VALGRIND"
}
send_user "$build_cmd\n"
eval exec $build_cmd


# Build program to determine size of unsigned long in C
set build_cmd "$bin_cc ${test_ulong_prog}.c -o ${test_ulong_prog}"
send_user "$build_cmd\n"
eval exec $build_cmd

spawn ./$test_ulong_prog
expect {
	-re "($number)" {
		set ulong_bits $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "FAILURE: No response from $test_ulong_prog\n"
		exit 1
	}
	eof {
		wait
	}
}

send_user "sizeof(unsigned long) = $ulong_bits\n"


# Sees if $test_prog actual output matches expected output
#
# test_minor -		The minor test number (39.18.X). If global variable
# 			"$test_minor_in" is set, then run only that test.
# gres_conf -		The gres.conf to use.
# fake_gpus_conf -	The fake_gpus.conf to use. This file tells Slurm to
# 			pretend to have devices detected by system.
# output_expected -	The expected "GRES_PARSABLE" output.
# err_msgs (optional) -	The error message regex to look for. Use | for multiple.
# errs_expected (optional) - The number of times you expect to see "error:" or
# 			"fatal:" in the output. Defaults to 0.
# 			If errs are expected, output_expected should likely be
# 			"".
set failed_tests [list]
proc check_configuration {test_minor gres_conf fake_gpus_conf output_expected {err_msgs ""} {errs_expected 0}} {
	global cfgpath nodename test_prog alpha_numeric_special exit_code test_minor_in
	global test_id failed_tests use_valgrind

	# Skip all tests not specified by the tester; force string comparison
	if {$test_minor_in != "" && [string compare "$test_minor" "$test_minor_in"] != 0} {
		return
	}

	set failed 0
	send_user "\n====test$test_id.$test_minor====\n\n"

	generate_file $gres_conf $cfgpath/gres.conf
	generate_file $fake_gpus_conf $cfgpath/fake_gpus.conf

	if {$use_valgrind} {
		spawn valgrind --tool=memcheck --error-limit=no --leak-check=full --show-reachable=yes --max-stackframe=16777216 --num-callers=20 --child-silent-after-fork=yes --track-origins=yes ./$test_prog "$cfgpath" "$nodename"
	} else {
		spawn ./$test_prog "$cfgpath" "$nodename"
	}

	set output_actual ""
	set errs_actual 0
	set ignore_err_msgs 0
	# Doing a regex on an empty string stalls expect. So don't do it.
	if {$err_msgs == ""} {
		set err_msgs "bogus"
		set ignore_err_msgs 1
	}
	send_user "Waiting on output\n"
	expect {
		-re "(GRES_PARSABLE$alpha_numeric_special)" {
			if {$output_actual == ""} {
				set output_actual "\n"
			}
			# Append to actual output
			set output_actual "$output_actual$expect_out(1,string)\n"
			exp_continue
		}
		-re "$err_msgs" {
			if {$ignore_err_msgs == 0} {
				incr errs_actual
			}
			exp_continue
		}
		timeout {
			send_user "FAILURE: No response from $test_prog\n"
			set failed 1
		}
		eof {
			wait
		}
	}

	# Check the output with what is expected
	send_user "Actual: $output_actual\n"
	if {$output_actual != $output_expected} {
		send_user "Expected: $output_expected\n"
		log_error "Expected output does not match actual output for sub-test $test_minor"
		set failed 1
	} else {
		log_info "(This is OK) Expected output does match actual output for sub-test $test_minor"
	}

	#  Check for expected errors, if applicable
	if {$err_msgs != "bogus"} {
		set response "Errors: Expected $errs_expected, found $errs_actual for error regex '$err_msgs' in sub-test $test_minor"
		if {$errs_actual == $errs_expected} {
			set response "(This is OK) ${response}"
			log_info $response
		} else {
			log_error $response
			set failed 1
		}
	}

	if {$failed == 1} {
		set exit_code 1
		lappend failed_tests $test_minor
	}
}

################################################################################
# NOTES:
################################################################################
# fake_gpus.conf is of the following format, with each line representing one
# GPU device:
#
#     type|cpu_count|cpu_range|links|device_files
#
# Optionally, cpu_range allows special keywords starting in `~` to aid in
# testing the conversion from an NVML cpu affinity array to Slurm bitstring.
#
# The GRES_PARSABLE output is of the following format:
#
# GRES_PARSABLE[name](device_count):type|cpu_count|cpu_range|links|device_files
#
# To better decipher the tests, temporarily increase the log verbosity in
# test39.18.prog.c. E.g.:
#
#     opts.stderr_level = LOG_LEVEL_DEBUG2;
#
# Note that there is a known issue with Expect where the regex parsing fails
# when the output is somewhat large. So to avoid random errors, each test output
# should be kept concise.
#
################################################################################



# ##############################################################################
# ------------------------------------------------------------------------------
# Tests for FastSchedule = 0
# ------------------------------------------------------------------------------
# ##############################################################################
set fast_schedule 0
create_slurm_conf

# ##############################################################################
# # Test 0.2 - Empty gres.conf
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
tesla|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 0.2 $gres_conf $fake_gpus_conf $expected_output

# TODO: Figure this out with the new changes
# # ##############################################################################
# # # Test 0.4 - Test empty and null identifiers in fake_gpus.conf
# # ##############################################################################

# set gres_conf ""
# set fake_gpus_conf "
# # This file was autogenerated by test$test_id
# tesla|4|0-1|_|${dev}5
# tesla|4|(null)|(null)|${dev}7
# tesla|4|0-1|(null)|${dev}6
# tesla|4|(null)|0-1|${dev}8
# tesla|4|2-3|(null)|${dev}9
# tesla|4|(null)|(null)|(null)
# "
# set expected_output "
# GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}5|
# GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}6|
# GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}7|
# GRES_PARSABLE\[gpu\](1):tesla|4|(null)|0-1|${dev}8|
# GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}9|
# GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}5|
# GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}6|
# GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}7|
# GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}8|
# GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}9|
# "
# check_configuration 0.4 $gres_conf $fake_gpus_conf $expected_output



# ##############################################################################
# The following tests are NVML-specific
# Only execute them if HAVE_NVML = 1 in config.h
# Make sure that AutoDetect=nvml is in ad-hoc gres.conf, or test prog will fatal
# ##############################################################################

if {[have_nvml] == 1} {
	if {$test_minor_in == ""} {
		log_info "Running NVML-specific tests:"
	}

# ##############################################################################
# # Test 0.6 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 20 unsigned longs with all bits set
# ##############################################################################

set arr_len 20
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.6 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.8 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1 unsigned longs with all bits set
# ##############################################################################

set arr_len 1
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.8 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.10 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 3 unsigned longs with all bits set
# ##############################################################################

set arr_len 3
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.10 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.12 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of the maximum # of CPUs supported 2^15
# ##############################################################################

set max_cpus [expr pow(2, 15)]
set arr_len [expr int($max_cpus/$ulong_bits)]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~max|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 0.12 $gres_conf $fake_gpus_conf $expected_output

# TODO: This test fails this assertion:
# "bit_alloc: Assertion `(nbits) > 0' failed."

# # ##############################################################################
# # # Test 0.14 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# # 	      An array of all zeros
# # ##############################################################################

# set gres_conf "AutoDetect=nvml"
# set fake_gpus_conf "
# # This file was autogenerated by test$test_id
# tesla|0|~zero|(null)|${dev}1
# "
# set expected_output "
# GRES_PARSABLE\[gpu\](1):tesla|0|(null)|(null)|${dev}1|
# GRES_PARSABLE\[mps\](0):tesla|0|(null)|(null)|${dev}1|
# "
# check_configuration 0.14 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.16 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1, with only CPU 0 set out of 16 system CPUs
# ##############################################################################

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~one|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|16|0|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|16|0|(null)|${dev}1|
"
check_configuration 0.16 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.18 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1, with CPUs 0-1 set out of 4 system CPUs
# ##############################################################################

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|~three|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}1|
"
check_configuration 0.18 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 0.20 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1, with CPUs 8-15 set out of 16 system CPUs
# ##############################################################################

set gres_conf "AutoDetect=nvml"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~half|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|16|8-15|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|16|8-15|(null)|${dev}1|
"
check_configuration 0.20 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# End NVML-specific tests
# ##############################################################################

} else {
	send_user "\n====Omitting NVML-specific tests====\n\n"
}




# ##############################################################################
# ------------------------------------------------------------------------------
# Tests for FastSchedule = 1
# ------------------------------------------------------------------------------
# ##############################################################################
set fast_schedule 1
create_slurm_conf

# TODO: Test out this functionality!


# ##############################################################################
# ------------------------------------------------------------------------------
# Tests for FastSchedule = 2
# ------------------------------------------------------------------------------
# ##############################################################################
set fast_schedule 2
create_slurm_conf




# ##############################################################################
# # Test 2.2 - Test gres/gpu plus gres/mps with count
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}1
Name=mps Count=100 File=${dev}0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.2 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.4 - Test gres/mps with count and no file names
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=mps Count=210
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](105):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](105):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.4 $gres_conf $fake_gpus_conf $expected_output



# ##############################################################################
# # Test 2.6 - Explicit device specification overrules system detected
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3
Name=gpu Type=tesla File=${dev}1 Cores=0-1
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|0-1|(null)|${dev}2
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.6 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.8 - Different system devices detected; single devices merged
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=2-3
Name=gpu Type=tesla File=${dev}2 Cores=2-3
Name=gpu Type=tesla File=${dev}3 Cores=2-3
Name=gpu Type=tesla File=${dev}4 Cores=2-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}5
tesla|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.8 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.10 - Different links, different records
# ##############################################################################

# Devices 0-2 are all doubly linked to each other
# Device 5 is singly linked to 3-4
# Devices 7-8 are doubly linked to each other
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}0 Cores=0-3 Links=-1,2,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=2,-1,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=2,2,-1,0,0,0,0,0
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,0,0,-1,0,1,0,0
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,0,0,0,-1,1,0,0
Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,0,0,1,1,-1,0,0
Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,0,0,0,0,0,-1,2
Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,0,0,0,0,0,2,-1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|-1,2,2,0,0,0,0,0|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,-1,2,0,0,0,0,0|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,2,-1,0,0,0,0,0|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,-1,0,1,0,0|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,-1,1,0,0|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,1,1,-1,0,0|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,-1,2|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,2,-1|${dev}7|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}7|
"
check_configuration 2.10 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.12 - Empty everything
# ##############################################################################

set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
check_configuration 2.12 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.14 - Empty system devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 2.14 $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 2.16 - Test that non-GPU GRESs are excluded
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     Type=tesla File=${dev}1 Cores=0-3
Name=tmpdisk Type=tesla File=${dev}2 Cores=0-3
Name=nic     Type=tesla File=${dev}3 Cores=0-3
Name=mic     Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[tmpdisk\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[nic\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mic\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
"
check_configuration 2.16 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.18 - Test NodeName when Name isn't specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu     Type=tesla File=${dev}1 Cores=0-3
NodeName=$nodename              Type=tesla File=${dev}2 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
"
check_configuration 2.18 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.20 - Test that empty Name does not parse and produces error
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
check_configuration 2.20 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.22 - Ensure no malloc error for large count with ignored non-GPU GRES
#             See bug 6014
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=tmpdisk Count=10g
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](10737418240):(null)|4|(null)|(null)|(null)|
"
check_configuration 2.22 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.24 - Different types, different records
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=a File=${dev}1 Cores=0-3
Name=gpu Type=a File=${dev}2 Cores=0-3
Name=gpu Type=b File=${dev}5 Cores=0-3
Name=gpu Type=b File=${dev}6 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):b|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):b|4|0-3|(null)|${dev}6|
"
check_configuration 2.24 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.26 - Same gres.conf and system devices is fine
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.26 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.28 - Test alternating device/cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla_a File=${dev}5       Cores=0-3
Name=gpu Type=tesla_b File=${dev}6       Cores=0-3
Name=gpu Type=tesla_a File=${dev}7       Cores=0-3
Name=gpu Type=tesla_b File=${dev}8       Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}8|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}8|
"
check_configuration 2.28 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.30 - Test duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.30 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.32 - Test increasing duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1       Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.32 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.34 - Test decreasing duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.34 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.36 - Test duplicate devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}1 Cores=0-1
Name=gpu Type=tesla File=${dev}1 Cores=0-2
Name=gpu Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
set err_msgs "gpu duplicate device file name"
set expected_errs 1
check_configuration 2.36 $gres_conf $fake_gpus_conf $expected_output $err_msgs $expected_errs

# ##############################################################################
# # Test 2.38 - Test overlapping cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}2 Cores=0-1
Name=gpu Type=tesla File=${dev}3 Cores=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-2|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-2|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.38 $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 2.40 - Work with null links
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=\"\"
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=null
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.40 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.42 - Show link ranges do NOT work - only commas
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,-1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,-1,2
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1,2|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 2.42 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.44 - What does 0, "", and "null" Type do
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
# Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3 Type=\"\"
Name=gpu File=${dev}3 Cores=0-3 Type=null
Name=gpu File=${dev}4 Cores=0-3 Type=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):null|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):0|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):null|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):0|4|0-3|(null)|${dev}4|
"
check_configuration 2.44 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.46 - Type not specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3
Name=gpu File=${dev}3 Cores=0-3
Name=gpu File=${dev}4 Cores=0-3

"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}4|
"
check_configuration 2.46 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.48 - Cores not specified or null
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1
Name=gpu Type=tesla File=${dev}2
Name=gpu Type=tesla File=${dev}3 Cores=\"\"
Name=gpu Type=tesla File=${dev}4 Cores=null
Name=gpu Type=tesla File=${dev}5 Cores=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4||(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|null|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4||(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|null|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0|(null)|${dev}5|
"
check_configuration 2.48 $gres_conf $fake_gpus_conf $expected_output


# TODO: What other tests should we add?
# Make sure current gres.conf files still work as expected
# Test to make sure CPU affinity is correct? Machine vs abstract?
# Invalid CPU counts
# Invalid cpu range

# ##############################################################################
# # Test 2.50 - Test examples in gres.conf docs
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}0 COREs=0,1
Name=mps Count=100 File=${dev}1 COREs=2,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2.50 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.52 - Test examples in gres.conf docs
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-1\] COREs=0,1
# NOTE: nvidia2 device is out of service
# Name=gpu Type=tesla File=${dev}\[2-3\] COREs=2,3
Name=gpu Type=tesla File=${dev}3 COREs=2,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):tesla|4|0,1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2,3|(null)|${dev}3|
"
check_configuration 2.52 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2.54 - Test examples in gres.conf docs
# #	      Note: ${nodename_base}0 == $nodename, so only that node 0's GRES
# #	      definitions will apply
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
## Explicitly specify devices on nodes ${nodename_base}0-${nodename_base}15
# NodeName=${nodename_base}\[0-15\] Name=gpu File=${dev}\[0-3\]
# NOTE: ${nodename_base}3 ${dev}1 device is out of service
NodeName=${nodename_base}\[0-2\] Name=gpu File=${dev}\[0-3\]
NodeName=${nodename_base}3 Name=gpu File=${dev}\[0,2-3\]
NodeName=${nodename_base}\[4-15\] Name=gpu File=${dev}\[0-3\]
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}3|
"
check_configuration 2.54 $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Tests for expected failures
# ##############################################################################



# ##############################################################################
# # Cleanup and finish
# ##############################################################################

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
} else {
	send_user "FAILURE: Tests $test_id.\["
	set length [llength $failed_tests]
	set i 0
	foreach test_minor $failed_tests {
		set i [expr $i+1]
		send_user "$test_minor"
		if {$i < $length} {
			send_user ","
		}
	}
	send_user "\] failed\n"

}

# Delete generated files
file delete $cfgpath/gres.conf
file delete $cfgpath/fake_gpus.conf
file delete $cfgpath/slurm.conf
file delete $test_prog
file delete $test_ulong_prog
file delete $dev0
file delete $dev1
file delete $dev2
file delete $dev3
file delete $dev4
file delete $dev5
file delete $dev6
file delete $dev7
file delete $dev8

# Delete generated folders
file delete $cfgpath

exit $exit_code
