#!/usr/bin/env expect
############################################################################
# Purpose: scontrol --local, --sibling options
#          scancel --sibling=<jobid>
#
# Reqs:    1. Using slurmdbd accounting storage type and is up
#          2. fed_slurm_base is defined in globals.local - set to directory that
#          has access to each federation configure (fedc1, fedc2, fedc3).
#          Eg.
#          fedr/slurm/ (src)
#          fedr/fed1/bin
#          fedr/fed1/sbin
#          fedr/fed1/etc
#          fedr/fed1/...
#          fedr/fed2/...
#          fedr/fed3/...
#          3. controllers are up and running.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2017 SchedMD LLC.
# Written by Isaac Hartung <ihartung@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

source ./globals
source ./globals_accounting
source ./globals_federation

set test_id     "37.8"
set exit_code   0
set fed_name    "feda"
set user_name   ""
set srun_job_cnt 0
set my_scontrol "${fed_slurm_base}/$fedc1/bin/scontrol"
set my_sacctmgr "${fed_slurm_base}/$fedc1/bin/sacctmgr"
set my_sbatch "${fed_slurm_base}/$fedc1/bin/sbatch"
set my_srun "${fed_slurm_base}/$fedc1/bin/srun"
set my_salloc "${fed_slurm_base}/$fedc1/bin/salloc"
set my_squeue "${fed_slurm_base}/$fedc1/bin/squeue"
set min_job_age [expr {[get_min_job_age] + 65}]

set eol "\r\n"

print_header $test_id

#
# Check accounting config and bail if not found.
#
if { [test_account_storage] == 0 } {
	log_warn "This test can't be run without a usable AccountStorageType"
	exit 0
}

if { [string compare [check_accounting_admin_level] "Administrator"] } {
	log_warn "This test can't be run without being an Accounting administrator.\n \
	 	  Use: sacctmgr mod user \$USER set admin=admin."
	exit 0
}


proc scontrol { option job_id regex args } {
	global fedc1 fed_slurm_base bin_sleep my_scontrol

	set matches 0

	set command "$my_scontrol $option show job $job_id"
	if { [string match $option "--local"] } {
		append command " -a"
		append command " $args"
	}

	spawn {*}$command
	expect {
		-re "$regex" {
			incr matches
		}
		timeout {
			log_error "scontrol not responding"
			end_it 1
		}
		eof {
			wait
		}
	}
	if {$matches != 1} {
		log_error "didn't match regex $regex: $matches"
		end_it 1
	}

}

proc sbatch { } {
	global number bin_sleep node_count my_sbatch
	global fedc1 fedc2 fedc3


	set script "\"sleep 35\""
	set matches 0
	set job_id 0
	set command "$my_sbatch -N$node_count --exclusive --output=/dev/null \
		--error=/dev/null -t300 --requeue --wrap $script \
		-M$fedc1,$fedc2,$fedc3"
	set regex "Submitted batch job ($number)"
	set collect 1

	spawn {*}$command
	expect {
		-re "$regex" {
			incr matches
			if { $collect } {
				set job_id $expect_out(1,string)
			}

		}
		timeout {
			log_error "sbatch not responding"
			end_it 1
		}
		eof {
			wait
		}
	}
	if {$matches != 1} {
		log_error "batch submit failure"
		end_it 1
	}

	$bin_sleep 2

	return $job_id
}

proc squeue { job_id regex args } {
	global my_squeue
	global fedc1 fedc2 fedc3

	set matches 0
	set command "$my_squeue --noheader -a --local -Ostatecompact:.5i\
		-j $job_id"

	if {$args ne ""} {
		append command " -M$args"
	} else {
		append command " -M$fedc1,$fedc2,$fedc3"
	}
	spawn {*}$command
	expect {
		-re "$regex" {
			incr matches
		}
		eof {
			wait
		}
	}
	if {$matches != 1} {
		log_error "unexpected error in squeue. expected $regex"
		end_it 1
	}
}

proc verify { job_id output } {
	global squeue fedc1 fedc2 fedc3

	if {[string match $output $fedc1]} {
		squeue $job_id "CLUSTER: $fedc1\\s+R\\s+CLUSTER: $fedc2\\s+slurm_load_jobs error: Invalid job id specified\\s+CLUSTER: $fedc3\\s+slurm_load_jobs error: Invalid job id specified\\s+"
		scontrol --local $job_id "JobId.+" -M$fedc1
		scontrol --local $job_id "slurm_load_jobs error: Invalid job id specified" -M$fedc2
		scontrol --local $job_id "slurm_load_jobs error: Invalid job id specified" -M$fedc3
	}
	if {[string match $output $fedc2]} {
		squeue $job_id "CLUSTER: $fedc1\\s+RV\\s+CLUSTER: $fedc2\\s+R\\s+CLUSTER: $fedc3\\s+slurm_load_jobs error: Invalid job id specified\\s+"
		scontrol --local $job_id "JobId.+" -M$fedc1
		scontrol --local $job_id "slurm_load_jobs error: Invalid job id specified" -M$fedc3
		scontrol --local $job_id "JobId.+" -M$fedc2
	}
	if {[string match $output $fedc3]} {
		squeue $job_id "CLUSTER: $fedc1\\s+RV\\s+CLUSTER: $fedc2\\s+slurm_load_jobs error: Invalid job id specified\\s+CLUSTER: $fedc3\\s+R\\s+"
		scontrol --local $job_id "JobId.+" -M$fedc1
		scontrol --local $job_id "slurm_load_jobs error: Invalid job id specified" -M$fedc2
		scontrol --local $job_id "JobId.+" -M$fedc3
	}

}

proc scancel_mod { sibling job_id regex} {
	global fed_slurm_base fedc1

	set matches 0
	set my_scancel "${fed_slurm_base}/$fedc1/bin/scancel"

	spawn $my_scancel --sibling=$sibling $job_id
	expect {
		-re "$regex" {
			incr matches
		}
		timeout {
			log_error "scancel not responding"
			end_it 1
		}
		eof {
			wait
		}
	}
	if {$matches != 1} {
		log_error "scancel failure, expected $regex\n"
		end_it 1
	}
}

proc cancel_all_jobs { } {
	global scancel user_name fedc1 fedc2 fedc3

	spawn $scancel -M$fedc1,$fedc2,$fedc3 --user $user_name
	expect {
		eof {
			wait
		}
	}
	sleep 5
}

proc cleanup { } {
	global scancel fed_name user_name bin_rm fedc1 fedc2 fedc3
	global test_id bin_bash my_sacctmgr

	cancel_all_jobs
	exec $bin_bash -c "$bin_rm -f test$test_id*.out"

	return [delete_federations $fed_name]
}

proc end_it { exit_code } {
	global test_id
	cleanup
	if {$exit_code == 0} {
		print_success $test_id
	}
	exit $exit_code
}

#start test

if {[test_federation_setup]} {
	log_warn "WARNING: This test can't be run without fed_slurm_base,\
		fedc1, fedc2, fedc3 setup in globals.local."
	exit 0
}

if {[test_all_up]} {
	exit 0
}

set user_name [get_my_user_name]

# Remove existing setup
if {[cleanup] != 0} {
	log_error "failed to cleanup"
	end_it 1
}

# add clusters to federation
if {[setup_federation $fed_name]} {
	log_error "failed to setup federation"
	end_it 1
}

# get number of nodes per cluster
set node_count [available_nodes "" ""]

send_user "\n################################################################\n"
send_user "Setup cluster features"
send_user "\n################################################################\n"

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $fedc1 set features=fa]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fa" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr mod not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $fedc2 set features=fb]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fb" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr mod not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $fedc3 set features=fc]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fc" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sacctmgr mod not responding\n"
		slow_kill $my_pid
		set exit_code 1
	}
	eof {
		wait
	}
}
if {$exit_code || $matches != 4} {
	send_user "$matches FAILURE: unexpected error.\n"
	end_it 1
}

send_user "\n################################################################\n"
send_user "test scontrol --local and --sibling"
send_user "\n################################################################\n"

set ji0 [sbatch]
set ji1 [sbatch]
set ji2 [sbatch]

set job_id [sbatch]

squeue $job_id "CLUSTER: $fedc1\\s+PD\\s+CLUSTER: $fedc2\\s+PD\\s+CLUSTER: $fedc3\\s+PD\\s+"

scontrol --sibling $job_id "JobId.+JobId.+JobId.+"

set output [wait_for_fed_job $ji0 RUNNING $fedc1,$fedc2,$fedc3]

verify $ji0 $output

set output [wait_for_fed_job $ji1 RUNNING $fedc1,$fedc2,$fedc3]

verify $ji1 $output

set output [wait_for_fed_job $ji2 RUNNING $fedc1,$fedc2,$fedc3]

verify $ji2 $output

send_user "\n################################################################\n"
send_user "scancel --sibling=<clustername><jobid> will remove on active pending"
send_user "\n################################################################\n"

cancel_all_jobs

set ji0 [sbatch]
set ji1 [sbatch]
set ji2 [sbatch]

set output [wait_for_fed_job $ji0 RUNNING $fedc1,$fedc2,$fedc3]

verify $ji0 $output

set output [wait_for_fed_job $ji1 RUNNING $fedc1,$fedc2,$fedc3]

verify $ji1 $output

set output [wait_for_fed_job $ji2 RUNNING $fedc1,$fedc2,$fedc3]

verify $ji2 $output

scancel_mod $fedc1 $ji0 "scancel: error: Kill job error on job id $ji0: Job is\
 no longer pending execution"

scancel_mod $fedc1 $ji1 "scancel: error: Kill job error on job id $ji1: Job is\
 no longer pending execution"

scancel_mod $fedc1 $ji2 "scancel: error: Kill job error on job id $ji2: Job is\
 no longer pending execution"

set job_id [sbatch]

squeue $job_id "\\s+PD\\s+" $fedc1
squeue $job_id "\\s+PD\\s+" $fedc2
squeue $job_id "\\s+PD\\s+" $fedc3

scancel_mod $fedc1 $job_id ""

squeue $job_id "\\s+RV\\s+" $fedc1
squeue $job_id "\\s+PD\\s+" $fedc2

scancel_mod $fedc2 $job_id ""

squeue $job_id "\\s+RV\\s+" $fedc1
squeue $job_id "slurm_load_jobs error: Invalid job id specified\\s+" $fedc2
squeue $job_id "\\s+PD\\s+" $fedc3

scancel_mod $fedc3 $job_id ""

squeue $job_id "\\s+RV\\s+" $fedc1
squeue $job_id "slurm_load_jobs error: Invalid job id specified\\s+" $fedc2
squeue $job_id "slurm_load_jobs error: Invalid job id specified\\s+" $fedc3

# All Done
end_it 0
