#!/usr/bin/env expect
############################################################################
# Purpose:  Test gres.conf-specified and system-detected GPU device merging
#           logic
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2018 SchedMD LLC
# Written by Michael Hinton
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id		"39.18"
set exit_code		0
set test_prog		"test$test_id.prog"
set cfgdir		"test${test_id}_configs"
set cfgpath		"[pwd]/$cfgdir"
set nodename_base	"tux"
set nodename		"${nodename_base}0"
set nodename_diff	"${nodename_base}1"
set test_ulong_prog	"test$test_id.ulong"
# Assume unsigned long is 64 bits unless overwritten
set ulong_bits  64
print_header $test_id

# Run the test under valgrind if any arguments are given
set use_valgrind 0
if {[lindex $argv 0] != ""} {
	set use_valgrind 1
	send_user "Running tests under valgrind!\n"
}

proc mkdir {dir} {
	exec mkdir -p $dir
}

proc touch_file {file} {
	exec touch $file
}

proc generate_file {text filepath} {
	exec echo $text > $filepath
}

# Contain slurm.conf data to this file
# This is all we need to trigger loading the GRES GPU plugin
set slurm_conf "
# This file was autogenerated by test$test_id
ControlMachine=test_machine
ClusterName=test_cluster
GresTypes=gpu,mps,nic,mic,tmpdisk
"

mkdir $cfgpath
generate_file $slurm_conf $cfgpath/slurm.conf

# Set up dummy device files for testing. They just need to exist
set dev "$cfgpath/nvidia"
set dev0 "${dev}0"
set dev1 "${dev}1"
set dev2 "${dev}2"
set dev3 "${dev}3"
set dev4 "${dev}4"
set dev5 "${dev}5"
set dev6 "${dev}6"
set dev7 "${dev}7"
set dev8 "${dev}8"
touch_file $dev0
touch_file $dev1
touch_file $dev2
touch_file $dev3
touch_file $dev4
touch_file $dev5
touch_file $dev6
touch_file $dev7
touch_file $dev8


# Build test program
set build_cmd "$bin_cc ${test_prog}.c -g -pthread -export-dynamic -o ${test_prog} -I${src_dir} -I${build_dir} ${build_dir}/src/api/libslurm.o -ldl"
if {$use_valgrind} {
	set build_cmd "$build_cmd -DUSING_VALGRIND"
}
send_user "$build_cmd\n"
eval exec $build_cmd


# Build program to determine size of unsigned long in C
set build_cmd "$bin_cc ${test_ulong_prog}.c -o ${test_ulong_prog}"
send_user "$build_cmd\n"
eval exec $build_cmd

spawn ./$test_ulong_prog
expect {
	-re "($number)" {
		set ulong_bits $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "FAILURE: No response from $test_ulong_prog\n"
		exit 1
	}
	eof {
		wait
	}
}

send_user "sizeof(unsigned long) = $ulong_bits\n"


# log_user 0
# Takes in gres.conf (user-specified devices), fake_gpus.conf (system-detected
# devices), and expected output, and then checks to see if the actual output
# matches the expected output
set failed_tests [list]
proc check_configuration {test_minor gres_conf fake_gpus_conf expected_output} {
	global cfgpath nodename test_prog alpha_numeric_special exit_code
	global test_id failed_tests use_valgrind
	set failed 0
	send_user "\n====test$test_id.$test_minor====\n\n"

	generate_file $gres_conf $cfgpath/gres.conf
	generate_file $fake_gpus_conf $cfgpath/fake_gpus.conf

	if {$use_valgrind} {
		spawn valgrind --tool=memcheck --error-limit=no --leak-check=full --show-reachable=yes --max-stackframe=16777216 --num-callers=20 --child-silent-after-fork=yes --track-origins=yes ./$test_prog "$cfgpath" "$nodename"
	} else {
		spawn ./$test_prog "$cfgpath" "$nodename"
	}

	set actual_output ""
	send_user "Waiting on output\n"
	expect {
		-re "failure" {
			log_error "Program $test_prog failed"
			set failed 1
			exp_continue
		}
		-re "(GRES_PARSABLE$alpha_numeric_special)" {
			if {$actual_output == ""} {
				set actual_output "\n"
			}
			# Append to actual output
			set actual_output "$actual_output$expect_out(1,string)\n"
			exp_continue
		}
		timeout {
			send_user "FAILURE: No response from $test_prog\n"
			set failed 1
		}
		eof {
			wait
		}
	}

	# Check the output with what is expected
	send_user "Actual: $actual_output\n"
	if {$actual_output != $expected_output} {
		send_user "Expected: $expected_output\n"
		log_error "Expected does not match actual for sub-test $test_minor"
		set failed 1
	} else {
		log_info "Expected does match actual for sub-test $test_minor"
	}
	if {$failed == 1} {
		set exit_code 1
		lappend failed_tests $test_minor
	}

}

################################################################################
# NOTES:
################################################################################
# fake_gpus.conf is of the following format, with each line representing one
# GPU device:
#
#     type|cpu_count|cpu_range|links|device_files
#
# Optionally, cpu_range allows special keywords starting in `~` to aid in
# testing the conversion from an NVML cpu affinity array to Slurm bitstring.
#
# The GRES_PARSABLE output is of the following format:
#
# GRES_PARSABLE[name](device_count):type|cpu_count|cpu_range|links|device_files
#
# To better decipher the tests, temporarily increase the log verbosity in
# test39.18.prog.c. E.g.:
#
#     opts.stderr_level = LOG_LEVEL_DEBUG2;
#
# Note that there is a known issue with Expect where the regex parsing fails
# when the output is somewhat large. So to avoid random errors, each test output
# should be kept concise.
#
################################################################################

# ##############################################################################
# # Test 2 - Test gres/gpu plus gres/mps with count
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}1
Name=mps Count=100 File=${dev}0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 2 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 4 - Test gres/mps with count and no file names
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=mps Count=210
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](105):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](105):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 4 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 6 - Empty gres.conf; merge system devices
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
tesla|4|0-1|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 6 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 8 - Single ignore line in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}\[3-4\] Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|2-3|(null)|${dev}1
tesla|4|2-3|(null)|${dev}2
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}2|
"
check_configuration 8 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 10 - Explicit device specification overrules system detected
# # NOTE the device numbers being out of order
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3
Name=gpu Type=tesla File=${dev}1 Cores=0-1
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}1
tesla|4|0-1|(null)|${dev}2
tesla|4|2-3|(null)|${dev}3
tesla|4|2-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 10 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 12 - Test that ignore lines only care about Name and File
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
# Should be ignored, even with mismatching fields other than Name and File
Name=gpu File=${dev}1 Cores=0-1 Ignore=true
Name=gpu File=${dev}2 Links=-1,0 Ignore=true
Name=gpu File=${dev}3 Ignore=true
Name=gpu File=${dev}4 Cores=0-1 Links=-1,0 Ignore=true
Name=gpu File=${dev}7 Links=-1,1 Ignore=true
# Should be ignored with only Name and File
Name=gpu File=${dev}5 Ignore=true
# Should be ignored, even with matching fields
Name=gpu File=${dev}8 Cores=2-3 Links=0,-1 Ignore=true
# Device 6 and 0 will not be ignored
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
(null)|4|2-3|(null)|${dev}1
(null)|4|2-3|(null)|${dev}2
(null)|4|2-3|(null)|${dev}3
(null)|4|2-3|(null)|${dev}4
(null)|4|2-3|(null)|${dev}5
(null)|4|2-3|(null)|${dev}6
(null)|4|2-3|-1,0|${dev}7
(null)|4|2-3|0,-1|${dev}8
(null)|4|2-3|(null)|${dev}0
"
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):(null)|4|2-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):(null)|4|2-3|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):(null)|4|2-3|(null)|${dev}6|
"
check_configuration 12 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 14 - Different system devices detected; single devices merged
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=2-3
Name=gpu Type=tesla File=${dev}2 Cores=2-3
Name=gpu Type=tesla File=${dev}3 Cores=2-3
Name=gpu Type=tesla File=${dev}4 Cores=2-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|(null)|${dev}5
tesla|4|0-1|(null)|${dev}6
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 14 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 16 - Different links, different records
# ##############################################################################

# Devices 0-2 are all doubly linked to each other
# Device 5 is singly linked to 3-4
# Devices 7-8 are doubly linked to each other
set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}0 Cores=0-3 Links=-1,2,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=2,-1,2,0,0,0,0,0
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=2,2,-1,0,0,0,0,0
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,0,0,-1,0,1,0,0
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,0,0,0,-1,1,0,0
Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,0,0,1,1,-1,0,0
Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,0,0,0,0,0,-1,2
Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,0,0,0,0,0,2,-1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|-1,2,2,0,0,0,0,0|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,-1,2,0,0,0,0,0|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|2,2,-1,0,0,0,0,0|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,-1,0,1,0,0|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,-1,1,0,0|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,1,1,-1,0,0|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,-1,2|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,0,0,0,0,0,2,-1|${dev}7|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}7|
"
check_configuration 16 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 18 - Empty everything
# ##############################################################################

set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
check_configuration 18 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 20 - Empty system devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}4|
"
check_configuration 20 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 22 - Test empty and null identifiers in fake_gpus.conf
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|_|${dev}5
tesla|4|(null)|(null)|${dev}7
tesla|4|0-1|(null)|${dev}6
tesla|4|(null)|0-1|${dev}8
tesla|4|2-3|(null)|${dev}9
tesla|4|(null)|(null)|(null)
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|0-1|${dev}8|
GRES_PARSABLE\[gpu\](1):tesla|4|2-3|(null)|${dev}9|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}7|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}8|
GRES_PARSABLE\[mps\](0):tesla|4|2-3|(null)|${dev}9|
"
check_configuration 22 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 24 - Test that non-GPU GRESs are excluded
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     Type=tesla File=${dev}1 Cores=0-3
Name=tmpdisk Type=tesla File=${dev}2 Cores=0-3
Name=nic     Type=tesla File=${dev}3 Cores=0-3
Name=mic     Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[tmpdisk\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[nic\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mic\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
"
check_configuration 24 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 26 - Test that non-GPU GRESs are excluded from ignore calculations
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     File=${dev}1 Ignore=true
Name=tmpdisk File=${dev}2 Ignore=true
Name=nic     File=${dev}3 Ignore=true
Name=mic     File=${dev}4 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[tmpdisk\](1):(null)|4|(null)|(null)|${dev}2|IGNORE
GRES_PARSABLE\[nic\](1):(null)|4|(null)|(null)|${dev}3|IGNORE
GRES_PARSABLE\[mic\](1):(null)|4|(null)|(null)|${dev}4|IGNORE
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 26 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 28 - Test NodeName when Name isn't specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu     Type=tesla File=${dev}1 Cores=0-3
NodeName=$nodename              Type=tesla File=${dev}2 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
"
check_configuration 28 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 30 - Test ignore with NodeName when Name is and isn't specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu File=${dev}1 Ignore=true
NodeName=$nodename          File=${dev}2 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
"
check_configuration 30 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 32 - Test that empty Name does not parse and produces error
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
check_configuration 32 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 34 - Ensure no malloc error for large count with ignored non-GPU GRES
#             See bug 6014
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=tmpdisk Count=10g
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[tmpdisk\](18446744071562067968):(null)|4|(null)|(null)|(null)|
"
check_configuration 34 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 36 - Different types, different records
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=a File=${dev}1 Cores=0-3
Name=gpu Type=a File=${dev}2 Cores=0-3
Name=gpu Type=b File=${dev}5 Cores=0-3
Name=gpu Type=b File=${dev}6 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):b|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):b|4|0-3|(null)|${dev}6|
"
check_configuration 36 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 38 - Ignore records for a different node should take no effect
# # NOTE the device numbers being out of order with a skipped number
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename_diff Name=gpu File=${dev}1 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}5
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}5|
"
check_configuration 38 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 40 - Ignore records for GRESs in gres.conf should take no effect
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}2                 Ignore=true
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 40 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 42 - Same gres.conf and system devices is fine
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 42 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 44 - Test alternating device/cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla_a File=${dev}5       Cores=0-3
Name=gpu Type=tesla_b File=${dev}6       Cores=0-3
Name=gpu Type=tesla_a File=${dev}7       Cores=0-3
Name=gpu Type=tesla_b File=${dev}8       Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla_a|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):tesla_b|4|0-3|(null)|${dev}8|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):tesla_a|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[mps\](0):tesla_b|4|0-3|(null)|${dev}8|
"
check_configuration 44 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 46 - Test alternating device/cpu ranges with ignores
# ##############################################################################

set gres_conf "
Name=gpu File=${dev}3 Ignore=true
Name=gpu File=${dev}5 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
tesla|4|0-3|(null)|${dev}5
tesla|4|0-3|(null)|${dev}6
tesla|4|0-3|(null)|${dev}7
tesla|4|0-3|(null)|${dev}8
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}8|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}6|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}7|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}8|
"
check_configuration 46 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 48 - Ignores for a specified node *should* take effect
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu File=${dev}2 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 48 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 50 - Test duplicate ignores handling in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}3       Ignore=true
Name=gpu File=${dev}3       Ignore=true
Name=gpu File=${dev}4       Ignore=true
Name=gpu File=${dev}4       Ignore=true
Name=gpu File=${dev}\[3-4\] Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
"
check_configuration 50 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 52 - Test duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}3       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 52 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 54 - Test increasing duplicate device merging in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1       Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 54 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 56 - Test decreasing duplicate device merging in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}4       Cores=0-3
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 56 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 58 - Test overlapping cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}2 Cores=0-1
Name=gpu Type=tesla File=${dev}3 Cores=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-2|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-2|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 58 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 60 - Ignore already-added devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0
Name=gpu Type=tesla File=${dev}1 Cores=0-1
Name=gpu Type=tesla File=${dev}1 Cores=0-2
Name=gpu Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0|(null)|${dev}1|
"
check_configuration 60 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 62 - Work with null links
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=\"\"
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=null
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 62 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 62 - Show link ranges do NOT work - only commas
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,-1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,-1,2
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|0-3|0,-1,2|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0-3|(null)|${dev}4|
"
check_configuration 62 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 64 - What does 0, "", and "null" Type do
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
# Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3 Type=\"\"
Name=gpu File=${dev}3 Cores=0-3 Type=null
Name=gpu File=${dev}4 Cores=0-3 Type=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):null|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):0|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):null|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):0|4|0-3|(null)|${dev}4|
"
check_configuration 64 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 66 - Type not specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}1 Cores=0-3
Name=gpu File=${dev}2 Cores=0-3
Name=gpu File=${dev}3 Cores=0-3
Name=gpu File=${dev}4 Cores=0-3

"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):(null)|4|0-3|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):(null)|4|0-3|(null)|${dev}4|
"
check_configuration 66 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 68 - Cores not specified or null
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1
Name=gpu Type=tesla File=${dev}2
Name=gpu Type=tesla File=${dev}3 Cores=\"\"
Name=gpu Type=tesla File=${dev}4 Cores=null
Name=gpu Type=tesla File=${dev}5 Cores=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}3|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}4|
GRES_PARSABLE\[gpu\](1):tesla|4|0|(null)|${dev}5|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}4|
GRES_PARSABLE\[mps\](0):tesla|4|0|(null)|${dev}5|
"
check_configuration 68 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 70 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 20 unsigned longs with all bits set
# ##############################################################################

set arr_len 20
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 70 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 72 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1 unsigned longs with all bits set
# ##############################################################################

set arr_len 1
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 72 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 74 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 3 unsigned longs with all bits set
# ##############################################################################

set arr_len 3
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 74 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 76 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of the maximum # of CPUs supported 2^15
# ##############################################################################

set max_cpus [expr pow(2, 15)]
set arr_len [expr int($max_cpus/$ulong_bits)]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~max|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 76 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 78 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of all zeros
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|0|~zero|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|0|(null)|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|0|(null)|(null)|${dev}1|
"
check_configuration 78 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 80 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1, with only CPU 0 set out of 16 system CPUs
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~one|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|16|0|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|16|0|(null)|${dev}1|
"
check_configuration 80 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 82 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1, with CPUs 0-1 set out of 4 system CPUs
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|~three|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0-1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|0-1|(null)|${dev}1|
"
check_configuration 82 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 84 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	      An array of 1, with CPUs 8-15 set out of 16 system CPUs
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|16|~half|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|16|8-15|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|16|8-15|(null)|${dev}1|
"
check_configuration 84 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 86 - Test bubble sorting code
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
# `b` is a special character to trigger testing bubble sort tests
b
"
set expected_output "
GRES_PARSABLE:succeeded
"
check_configuration 86 $gres_conf $fake_gpus_conf $expected_output

# TODO: What other tests should we add?
# Make sure current gres.conf files still work as expected
# Test to make sure CPU affinity is correct? Machine vs abstract?
# Invalid CPU counts
# Invalid cpu range

# log_user 1

# ##############################################################################
# # Test 88 - Test examples in gres.conf docs
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=gtx560 File=${dev}0 COREs=0,1
Name=gpu Type=tesla File=${dev}1 COREs=2,3
Name=mps Count=100 File=${dev}0 COREs=0,1
Name=mps Count=100 File=${dev}1 COREs=2,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}1|
GRES_PARSABLE\[mps\](100):gtx560|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](100):tesla|4|2,3|(null)|${dev}1|
"
check_configuration 88 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 90 - Test examples in gres.conf docs
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}0 Ignore=true
Name=gpu File=${dev}1 Ignore=true
Name=gpu File=${dev}2 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|2,3|(null)|${dev}0
tesla|4|2,3|(null)|${dev}1
tesla|4|2,3|(null)|${dev}2
tesla|4|2,3|(null)|${dev}3
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|2,3|(null)|${dev}3|
"
check_configuration 90 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 92 - Test examples in gres.conf docs
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[0-1\] COREs=0,1
# NOTE: nvidia2 device is out of service
# Name=gpu Type=tesla File=${dev}\[2-3\] COREs=2,3
Name=gpu Type=tesla File=${dev}3 COREs=2,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|0,1|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):tesla|4|2,3|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|0,1|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):tesla|4|0,1|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):tesla|4|2,3|(null)|${dev}3|
"
check_configuration 92 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 94 - Test examples in gres.conf docs
# #	      Note: ${nodename_base}0 == $nodename, so only that node 0's GRES
# #	      definitions will apply
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
## Explicitly specify devices on nodes ${nodename_base}0-${nodename_base}15
# NodeName=${nodename_base}\[0-15\] Name=gpu File=${dev}\[0-3\]
# NOTE: ${nodename_base}3 ${dev}1 device is out of service
NodeName=${nodename_base}\[0-2\] Name=gpu File=${dev}\[0-3\]
NodeName=${nodename_base}3 Name=gpu File=${dev}\[0,2-3\]
NodeName=${nodename_base}\[4-15\] Name=gpu File=${dev}\[0-3\]
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[gpu\](1):(null)|4|(null)|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}1|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}2|
GRES_PARSABLE\[mps\](0):(null)|4|(null)|(null)|${dev}3|
"
check_configuration 94 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 96 - Test examples in gres.conf docs
# #		Note: ${nodename_base}0 == $nodename, so only that node 0's GRES
# #		definitions will apply
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
## Ignore select system-detected devices on nodes ${nodename_base}0-${nodename_base}15
NodeName=${nodename_base}0 Name=gpu File=${dev}\[1-2\] Ignore=true
NodeName=${nodename_base}\[4,9,12\] Name=gpu File=${dev}3 Ignore=true
NodeName=${nodename_base}13 Name=gpu File=${dev}\[0,4\] Ignore=true
NodeName=${nodename_base}15 Name=gpu File=${dev}0 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|(null)|(null)|${dev}0
tesla|4|(null)|(null)|${dev}3
"
set expected_output "
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[gpu\](1):tesla|4|(null)|(null)|${dev}3|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}0|
GRES_PARSABLE\[mps\](0):tesla|4|(null)|(null)|${dev}3|
"
check_configuration 96 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Tests for expected failures
# ##############################################################################



# ##############################################################################
# # Cleanup and finish
# ##############################################################################

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
} else {
	send_user "FAILURE: Tests $test_id.\["
	set length [llength $failed_tests]
	set i 0
	foreach test_minor $failed_tests {
		set i [expr $i+1]
		send_user "$test_minor"
		if {$i < $length} {
			send_user ","
		}
	}
	send_user "\] failed\n"

}

# Delete generated files
file delete $cfgpath/gres.conf
file delete $cfgpath/fake_gpus.conf
file delete $cfgpath/slurm.conf
file delete $test_prog
file delete $test_ulong_prog
file delete $dev0
file delete $dev1
file delete $dev2
file delete $dev3
file delete $dev4
file delete $dev5
file delete $dev6
file delete $dev7
file delete $dev8

# Delete generated folders
file delete $cfgpath

exit $exit_code
