#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Make certain that srun behaves when its controlling terminal
#          disappears.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2006 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Christopher J. Morrone <morrone2@llnl.gov>
# UCRL-CODE-217948.
# 
# This file is part of SLURM, a resource management program.
# For details, see <http://www.llnl.gov/linux/slurm/>.
#  
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
# 
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
# 
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
############################################################################
source ./globals

set test_id              "1.55"
set test_script          "test$test_id.sh"

print_header $test_id

#
# Create a test script to be launch by srun
#
exec $bin_rm -f $test_script
make_bash_script $test_script {
    echo "Running"

    sleep 5

    for ((i = 0; i < 100; i++)); do
        cat /etc/hosts
    done
}

#
# Launch the test script
#
set srun_pid [spawn $srun -u -v -n1 $test_script]

set timeout $max_job_delay
set jobid  0
set stepid 0
expect {
    -re "launching (($number)\.0)" {
	set stepid $expect_out(1,string)
	set jobid $expect_out(2,string)
	exp_continue
    }
    "Running" {}
    timeout {
	send_user "\nFAILURE: Failed to launch test program through srun\n"
	catch {exec $scancel $jobid}
	exit 1
    }
}
if {$jobid == 0} {
	send_user "\nFAILURE: Failed to launch test program through srun\n"
	exit 1
}

#
# Kill srun's stdio streams
#
send_user "Test saw stepid $stepid\n"
set file [exp_open]
close $file


#
# Lets see if the job step finishes normally
#
set running "seed"
set i 0
while {[string compare $running ""]} {
    set running [exec $squeue --noheader --states=running --steps=$stepid]
    sleep 1
    if {$i == 30} {
	send_user "\nFAILURE: job step is not completing\n"
	exec $scancel -q $jobid
	sleep 5
	exec kill -9 $srun_pid
	exit 1
    }
    incr i 1
}
send_user "Test sees step is gone, srun should have exited as well.\n"

#
# And finally check to see if srun is still hanging around (it should
# have exited by now).
#
if [catch {exec kill -0 $srun_pid}] {
    send_user "\nSUCCESS\n"
    exec $bin_rm -f $test_script
    exit 0
} else {
    send_user "\nFAILURE: srun is still running after job exits!\n"
    exec kill -9 $srun_pid
    exit 1
}
