#!/usr/bin/env expect
############################################################################
# Purpose:  Test gres.conf-specified and system-detected GPU device merging
#           logic
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2018 SchedMD LLC
# Written by Michael Hinton
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id		"39.18"
set exit_code		0
set test_prog		"test$test_id.prog"
set cfgdir		"test${test_id}_configs"
set cfgpath		"[pwd]/$cfgdir"
set nodename		"test_node"
set nodename1		"different_node"
set test_ulong_prog	"test$test_id.ulong"
# Assume unsigned long is 64 bits unless overwritten
set ulong_bits  64
print_header $test_id

# Run the test under valgrind if any arguments are given
set use_valgrind 0
if {[lindex $argv 0] != ""} {
	set use_valgrind 1
	send_user "Running tests under valgrind!\n"
}

proc mkdir {dir} {
	exec mkdir -p $dir
}

proc touch_file {file} {
	exec touch $file
}

proc generate_file {text filepath} {
	exec echo $text > $filepath
}

# Contain slurm.conf data to this file
# This is all we need to trigger loading the GRES GPU plugin
set slurm_conf "
# This file was autogenerated by test$test_id
ControlMachine=test_machine
ClusterName=test_cluster
GresTypes=gpu,nic,mic,tmpdisk
"

mkdir $cfgpath
generate_file $slurm_conf $cfgpath/slurm.conf

# Set up dummy device files for testing. They just need to exist
set dev "$cfgpath/nvidia"
set dev1 "${dev}1"
set dev2 "${dev}2"
set dev3 "${dev}3"
set dev4 "${dev}4"
set dev5 "${dev}5"
set dev6 "${dev}6"
set dev7 "${dev}7"
set dev8 "${dev}8"
touch_file $dev1
touch_file $dev2
touch_file $dev3
touch_file $dev4
touch_file $dev5
touch_file $dev6
touch_file $dev7
touch_file $dev8


# Build test program
set build_cmd "$bin_cc ${test_prog}.c -g -pthread -export-dynamic -o ${test_prog} -I${src_dir} -I${build_dir} ${build_dir}/src/api/libslurm.o -ldl"
if {$use_valgrind} {
	set build_cmd "$build_cmd -DUSING_VALGRIND"
}
send_user "$build_cmd\n"
eval exec $build_cmd


# Build program to determine size of unsigned long in C
set build_cmd "$bin_cc ${test_ulong_prog}.c -o ${test_ulong_prog}"
send_user "$build_cmd\n"
eval exec $build_cmd

spawn ./$test_ulong_prog
expect {
	-re "($number)" {
		set ulong_bits $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "FAILURE: No response from $test_ulong_prog\n"
		exit 1
	}
	eof {
		wait
	}
}

send_user "sizeof(unsigned long) = $ulong_bits\n"


# log_user 0
# Takes in gres.conf (user-specified devices), fake_gpus.conf (system-detected
# devices), and expected output, and then checks to see if the actual output
# matches the expected output
set failed_tests [list]
proc check_configuration {test_minor gres_conf fake_gpus_conf expected_output} {
	global cfgpath nodename test_prog alpha_numeric_special exit_code
	global test_id failed_tests use_valgrind
	set failed 0
	send_user "\n====test$test_id.$test_minor====\n\n"

	generate_file $gres_conf $cfgpath/gres.conf
	generate_file $fake_gpus_conf $cfgpath/fake_gpus.conf

	if {$use_valgrind} {
		spawn valgrind --tool=memcheck --error-limit=no --leak-check=full --show-reachable=yes --max-stackframe=16777216 --num-callers=20 --child-silent-after-fork=yes --track-origins=yes ./$test_prog "$cfgpath" "$nodename"
	} else {
		spawn ./$test_prog "$cfgpath" "$nodename"
	}

	set actual_output ""
	send_user "Waiting on output\n"
	expect {
		-re "failure" {
			log_error "Program $test_prog failed"
			set failed 1
			exp_continue
		}
		-re "(GRES_PARSABLE$alpha_numeric_special)" {
			if {$actual_output == ""} {
				set actual_output "\n"
			}
			# Append to actual output
			set actual_output "$actual_output$expect_out(1,string)\n"
			exp_continue
		}
		timeout {
			send_user "FAILURE: No response from $test_prog\n"
			set failed 1
		}
		eof {
			wait
		}
	}

	# Check the output with what is expected
	send_user "Actual: $actual_output\n"
	send_user "Expected: $expected_output\n"
	if {$actual_output != $expected_output} {
		log_error "Expected does not match actual!"
		set failed 1
	}
	if {$failed == 1} {
		set exit_code 1
		lappend failed_tests $test_minor
	}

}

################################################################################
# NOTES:
################################################################################
# fake_gpus.conf is of the following format, with each line representing one
# device:
#
#     type|cpu_count|cpu_range|links|device_files
#
# Optionally, cpu_range allows special keywords starting in `~` to aid in
# testing the conversion from an NVML cpu affinity array to Slurm bitstring.
#
# The GRES_PARSABLE output is of the following format:
#
#     GRES_PARSABLE(device_count):type|cpu_count|cpu_range|links|device_files
#
# To better decipher the tests, temporarily increase the log verbosity in
# test39.18.prog.c. E.g.:
#
#     opts.stderr_level = LOG_LEVEL_DEBUG2;
#
# Note that there is a known issue with Expect where the regex parsing fails
# when the output is somewhat large. So to avoid random errors, each test output
# should be kept concise.
#
################################################################################

# ##############################################################################
# # Test 0 - Empty gres.conf; merge system devices
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|0,1|${dev}1
tesla|4|0-1|0,1|${dev}2
tesla|4|2-3|0,1|${dev}3
tesla|4|2-3|0,1|${dev}4
"
set expected_output "
GRES_PARSABLE(2):tesla|4|0-1|0,1|${dev}\[1-2\]|
GRES_PARSABLE(2):tesla|4|2-3|0,1|${dev}\[3-4\]|
"
check_configuration 0 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 1 - Single ignore file in gres.conf;
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Links=0,1 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|2-3|0,1|${dev}1
tesla|4|2-3|0,1|${dev}2
tesla|4|2-3|0,1|${dev}3
tesla|4|2-3|0,1|${dev}4
"
set expected_output "
GRES_PARSABLE(2):tesla|4|2-3|0,1|${dev}\[1-2\]|
"
check_configuration 1 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 2 - Explicit device specification overrules system detected
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-1 Links=0,1
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=2-3 Links=0,1
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|0,1|${dev}1
tesla|4|0-1|0,1|${dev}2
tesla|4|2-3|0,1|${dev}3
tesla|4|2-3|0,1|${dev}4
"
set expected_output "
GRES_PARSABLE(1):tesla|4|0-1|0,1|${dev}1|
GRES_PARSABLE(2):tesla|4|2-3|0,1|${dev}\[3-4\]|
"
check_configuration 2 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 4 - Different system devices detected; single devices merged
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=2-3 Links=0,1
Name=gpu Type=tesla File=${dev}2 Cores=2-3 Links=0,1
Name=gpu Type=tesla File=${dev}3 Cores=2-3 Links=0,1
Name=gpu Type=tesla File=${dev}4 Cores=2-3 Links=0,1
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|0,1|${dev}5
tesla|4|0-1|0,1|${dev}6
"
set expected_output "
GRES_PARSABLE(4):tesla|4|2-3|0,1|${dev}\[1-4\]|
"
check_configuration 4 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 5 - Different links, different records
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=1
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=1
Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}6 Cores=0-3 Links=0,2
Name=gpu Type=tesla File=${dev}7 Cores=0-3 Links=0,3
Name=gpu Type=tesla File=${dev}8 Cores=0-3 Links=0,3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(2):tesla|4|0-3|0|${dev}\[1-2\]|
GRES_PARSABLE(2):tesla|4|0-3|1|${dev}\[3-4\]|
GRES_PARSABLE(1):tesla|4|0-3|0,1|${dev}5|
GRES_PARSABLE(1):tesla|4|0-3|0,2|${dev}6|
GRES_PARSABLE(2):tesla|4|0-3|0,3|${dev}\[7-8\]|
"
check_configuration 5 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 100 - Empty everything
# ##############################################################################

set gres_conf ""
set fake_gpus_conf ""
set expected_output ""
check_configuration 100 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 101 - Empty system devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=2-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(4):tesla|4|2-3|0,1|${dev}\[1-4\]|
"
check_configuration 101 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 102 - Test empty and null identifiers in fake_gpus.conf
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-1|_|${dev}5
tesla|4|0-1|(null)|${dev}6
tesla|4|(null)|(null)|${dev}7
tesla|4|(null)|0-1|${dev}8
tesla|4|2-3|(null)|${dev}9
tesla|4|(null)|(null)|(null)
"
set expected_output "
GRES_PARSABLE(2):tesla|4|0-1|(null)|${dev}\[5-6\]|
GRES_PARSABLE(1):tesla|4|(null)|(null)|${dev}7|
GRES_PARSABLE(1):tesla|4|(null)|0-1|${dev}8|
GRES_PARSABLE(1):tesla|4|2-3|(null)|${dev}9|
"
check_configuration 102 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 103 - Test that non-GPU GRESs are excluded
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     Type=tesla File=${dev}1 Cores=0-3
Name=tmpdisk Type=tesla File=${dev}2 Cores=0-3
Name=nic     Type=tesla File=${dev}3 Cores=0-3
Name=mic     Type=tesla File=${dev}4 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(1):tesla|4|0-3|(null)|${dev}1|
"
check_configuration 103 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 104 - Test that non-GPU GRESs are excluded from ignore calculations
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu     Type=tesla File=${dev}1 Cores=0-3 Ignore=true
Name=tmpdisk Type=tesla File=${dev}2 Cores=0-3 Ignore=true
Name=nic     Type=tesla File=${dev}3 Cores=0-3 Ignore=true
Name=mic     Type=tesla File=${dev}4 Cores=0-3 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
tesla|4|0-3|(null)|${dev}3
tesla|4|0-3|(null)|${dev}4
"
set expected_output "
GRES_PARSABLE(3):tesla|4|0-3|(null)|${dev}\[2-4\]|
"
check_configuration 104 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 105 - Test NodeName when Name isn't specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu     Type=tesla File=${dev}1 Cores=0-3
NodeName=$nodename              Type=tesla File=${dev}2 Cores=0-3
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(1):tesla|4|0-3|(null)|${dev}1|
"
check_configuration 105 $gres_conf $fake_gpus_conf $expected_output


# ##############################################################################
# # Test 106 - Test ignore with NodeName when Name isn't specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu     Type=tesla File=${dev}1 Cores=0-3 Ignore=true
NodeName=$nodename              Type=tesla File=${dev}2 Cores=0-3 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|(null)|${dev}1
tesla|4|0-3|(null)|${dev}2
"
set expected_output "
GRES_PARSABLE(1):tesla|4|0-3|(null)|${dev}2|
"
check_configuration 106 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 107 - Test that empty Name does not parse and produces error
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Type=tesla File=${dev}1 Cores=0-3
"
set fake_gpus_conf ""
set expected_output ""
check_configuration 107 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 108 - Ensure no malloc error for large count with ignored non-GPU GRES
#              Bug 6014
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=tmpdisk Count=10g
"
set fake_gpus_conf ""
set expected_output ""
check_configuration 108 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 6 - Different types, different records
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=a File=${dev}1 Cores=0-3 Links=0,1
Name=gpu Type=a File=${dev}2 Cores=0-3 Links=0,1
Name=gpu Type=b File=${dev}5 Cores=0-3 Links=0,1
Name=gpu Type=b File=${dev}6 Cores=0-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(2):a|4|0-3|0,1|${dev}\[1-2\]|
GRES_PARSABLE(2):b|4|0-3|0,1|${dev}\[5-6\]|
"
check_configuration 6 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 7 - Ignore records for a different node should take no effect
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename1 Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|0,1|${dev}1
tesla|4|0-3|0,1|${dev}2
tesla|4|0-3|0,1|${dev}3
tesla|4|0-3|0,1|${dev}4
"
set expected_output "
GRES_PARSABLE(4):tesla|4|0-3|0,1|${dev}\[1-4\]|
"
check_configuration 7 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 8 - Ignore records for gres-specified nodes take no effect
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1 Ignore=true
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(4):tesla|4|0-3|0,1|${dev}\[1-4\]|
"
check_configuration 8 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 9 - Same gres.conf and system devices is fine
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|0,1|${dev}1
tesla|4|0-3|0,1|${dev}2
tesla|4|0-3|0,1|${dev}3
tesla|4|0-3|0,1|${dev}4
"
set expected_output "
GRES_PARSABLE(4):tesla|4|0-3|0,1|${dev}\[1-4\]|
"
check_configuration 9 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 10 - Test alternating device/cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla_a File=${dev}\[1-2\] Cores=0-3 Links=0,1
Name=gpu Type=tesla_b File=${dev}\[3-4\] Cores=0-3 Links=0,1
Name=gpu Type=tesla_a File=${dev}5 Cores=0-3 Links=0,1
Name=gpu Type=tesla_b File=${dev}6 Cores=0-3 Links=0,1
Name=gpu Type=tesla_a File=${dev}7 Cores=0-3 Links=0,1
Name=gpu Type=tesla_b File=${dev}8 Cores=0-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(4):tesla_a|4|0-3|0,1|${dev}\[1-2,5,7\]|
GRES_PARSABLE(4):tesla_b|4|0-3|0,1|${dev}\[3-4,6,8\]|
"
check_configuration 10 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 11 - Test alternating device/cpu ranges with ignores
# ##############################################################################

set gres_conf "
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
Name=gpu Type=tesla File=${dev}5 Cores=0-3 Links=0,1 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|0,1|${dev}1
tesla|4|0-3|0,1|${dev}2
tesla|4|0-3|0,1|${dev}3
tesla|4|0-3|0,1|${dev}4
tesla|4|0-3|0,1|${dev}5
tesla|4|0-3|0,1|${dev}6
tesla|4|0-3|0,1|${dev}7
tesla|4|0-3|0,1|${dev}8
"
set expected_output "
GRES_PARSABLE(6):tesla|4|0-3|0,1|${dev}\[1-2,4,6-8\]|
"
check_configuration 11 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 12 - Ignores for a specified node *should* take effect
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
NodeName=$nodename Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|0,1|${dev}1
tesla|4|0-3|0,1|${dev}2
tesla|4|0-3|0,1|${dev}3
tesla|4|0-3|0,1|${dev}4
"
set expected_output "
GRES_PARSABLE(3):tesla|4|0-3|0,1|${dev}\[1,3-4\]|
"
check_configuration 12 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 13 - Test duplicate ignores handling in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1 Ignore=true
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1 Ignore=true
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1 Ignore=true
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1 Ignore=true
"
set fake_gpus_conf "
# This file was autogenerated by test$test_id
tesla|4|0-3|0,1|${dev}1
tesla|4|0-3|0,1|${dev}2
tesla|4|0-3|0,1|${dev}3
tesla|4|0-3|0,1|${dev}4
"
set expected_output "
GRES_PARSABLE(2):tesla|4|0-3|0,1|${dev}\[1-2\]|
"
check_configuration 13 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 14 - Test duplicate devices in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(2):tesla|4|0-3|0,1|${dev}\[3-4\]|
"
check_configuration 14 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 15 - Test increasing duplicate device merging in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}\[1-2\] Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}\[1-3\] Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(4):tesla|4|0-3|0,1|${dev}\[1-4\]|
"
check_configuration 15 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 16 - Test decreasing duplicate device merging in gres.conf
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}\[3-4\] Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}\[2-4\] Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}\[1-4\] Cores=0-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(4):tesla|4|0-3|0,1|${dev}\[4,3,2,1\]|
"
check_configuration 16 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 17 - Test overlapping cpu ranges
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0 Links=0,1
Name=gpu Type=tesla File=${dev}2 Cores=0-1 Links=0,1
Name=gpu Type=tesla File=${dev}3 Cores=0-2 Links=0,1
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(1):tesla|4|0|0,1|${dev}1|
GRES_PARSABLE(1):tesla|4|0-1|0,1|${dev}2|
GRES_PARSABLE(1):tesla|4|0-2|0,1|${dev}3|
GRES_PARSABLE(1):tesla|4|0-3|0,1|${dev}4|
"
check_configuration 17 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 18 - Ignore already-added devices
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0 Links=0,1
Name=gpu Type=tesla File=${dev}1 Cores=0-1 Links=0,1
Name=gpu Type=tesla File=${dev}1 Cores=0-2 Links=0,1
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0,1
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(1):tesla|4|0|0,1|${dev}1|
"
check_configuration 18 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 19 - Work with null links
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=\"\"
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=null
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(3):tesla|4|0-3|(null)|${dev}\[1-3\]|
GRES_PARSABLE(1):tesla|4|0-3|0|${dev}4|
"
check_configuration 19 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 20 - Show link ranges do NOT work - only commas
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Cores=0-3 Links=0-1
Name=gpu Type=tesla File=${dev}2 Cores=0-3 Links=0,1
Name=gpu Type=tesla File=${dev}3 Cores=0-3 Links=0-2
Name=gpu Type=tesla File=${dev}4 Cores=0-3 Links=0,1,2
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(2):tesla|4|0-3|(null)|${dev}\[1,3\]|
GRES_PARSABLE(1):tesla|4|0-3|0,1|${dev}2|
GRES_PARSABLE(1):tesla|4|0-3|0,1,2|${dev}4|
"
check_configuration 20 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 21 - What does 0, "", and "null" Type do
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
# Name=gpu File=${dev}1 Cores=0-3 Links=0,1
Name=gpu File=${dev}2 Cores=0-3 Links=0,1 Type=\"\"
Name=gpu File=${dev}3 Cores=0-3 Links=0,1 Type=null
Name=gpu File=${dev}4 Cores=0-3 Links=0,1 Type=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(1):|4|0-3|0,1|${dev}2|
GRES_PARSABLE(1):null|4|0-3|0,1|${dev}3|
GRES_PARSABLE(1):0|4|0-3|0,1|${dev}4|
"
check_configuration 21 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 22 - Type not specified
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu File=${dev}1 Cores=0-3 Links=0,1
Name=gpu File=${dev}2 Cores=0-3 Links=0,1
Name=gpu File=${dev}3 Cores=0-3 Links=0,1
Name=gpu File=${dev}4 Cores=0-3 Links=0,1

"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(4):(null)|4|0-3|0,1|${dev}\[1-4\]|
"
check_configuration 22 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 23 - Cores not specified or null
# ##############################################################################

set gres_conf "
# This file was autogenerated by test$test_id
Name=gpu Type=tesla File=${dev}1 Links=0,1
Name=gpu Type=tesla File=${dev}2 Links=0,1
Name=gpu Type=tesla File=${dev}3 Links=0,1 Cores=\"\"
Name=gpu Type=tesla File=${dev}4 Links=0,1 Cores=null
Name=gpu Type=tesla File=${dev}5 Links=0,1 Cores=0
"
set fake_gpus_conf ""
set expected_output "
GRES_PARSABLE(3):tesla|4|(null)|0,1|${dev}\[1-3\]|
GRES_PARSABLE(1):tesla|4|null|0,1|${dev}4|
GRES_PARSABLE(1):tesla|4|0|0,1|${dev}5|
"
check_configuration 23 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 200 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of 20 unsigned longs with all bits set
# ##############################################################################

set arr_len 20
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 200 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 201 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of 1 unsigned longs with all bits set
# ##############################################################################

set arr_len 1
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 201 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 202 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of 3 unsigned longs with all bits set
# ##############################################################################

set arr_len 3
set cpu_array [string repeat "X" $arr_len]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~$cpu_array|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 202 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 203 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of the maximum # of CPUs supported 2^15
# ##############################################################################

set max_cpus [expr pow(2, 15)]
set arr_len [expr int($max_cpus/$ulong_bits)]
set cpus_count [expr $arr_len*$ulong_bits]
set cpus_count_m1 [expr $cpus_count-1]

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|$cpus_count|~max|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|$cpus_count|0-$cpus_count_m1|(null)|${dev}1|
"
check_configuration 203 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 204 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of all zeros
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|0|~zero|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|0|(null)|(null)|${dev}1|
"
check_configuration 204 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 205 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of 1, with only cpu 0 set out of 16 system cpus
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|16|~one|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|16|0|(null)|${dev}1|
"
check_configuration 205 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 206 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of 1, with cpus 0-1 set out of 4 system cpus
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|4|~three|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|4|0-1|(null)|${dev}1|
"
check_configuration 206 $gres_conf $fake_gpus_conf $expected_output

# ##############################################################################
# # Test 207 - Test conversion of NVML-style cpu affinity array to Slurm bitstr
# 	       An array of 1, with cpus 8-15 set out of 16 system cpus
# ##############################################################################

set gres_conf ""
set fake_gpus_conf "# This file was autogenerated by test$test_id
tesla|16|~half|(null)|${dev}1
"
set expected_output "
GRES_PARSABLE(1):tesla|16|8-15|(null)|${dev}1|
"
check_configuration 207 $gres_conf $fake_gpus_conf $expected_output

# TODO: What other tests should we add?
# Test more ignore scenarios
# Test more merge scenarios
# Make sure current gres.conf files still work as expected
# Test to make sure CPU affinity is correct? Machine vs abstract?
# Invalid CPU counts
# Invalid cpu range

# log_user 1

# ##############################################################################
# # Tests for expected failures
# ##############################################################################



# ##############################################################################
# # Cleanup and finish
# ##############################################################################

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
} else {
	send_user "FAILURE: Tests $test_id.\["
	set length [llength $failed_tests]
	set i 0
	foreach test_minor $failed_tests {
		set i [expr $i+1]
		send_user "$test_minor"
		if {$i < $length} {
			send_user ","
		}
	}
	send_user "\] failed\n"

}

# Delete generated files
file delete $cfgpath/gres.conf
file delete $cfgpath/fake_gpus.conf
file delete $cfgpath/slurm.conf
file delete $test_prog
file delete $test_ulong_prog
file delete $dev1
file delete $dev2
file delete $dev3
file delete $dev4
file delete $dev5
file delete $dev6
file delete $dev7
file delete $dev8

# Delete generated folders
file delete $cfgpath

exit $exit_code
