/*****************************************************************************\
 *  bg_block_info.c - bluegene block information from the db2 database.
 *
 *  $Id$
 *****************************************************************************
 *  Copyright (C) 2004-2006 The Regents of the University of California.
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 *  Written by Danny Auble <da@llnl.gov>
 *  
 *  This file is part of SLURM, a resource management program.
 *  For details, see <https://computing.llnl.gov/linux/slurm/>.
 *  Please also read the included file: DISCLAIMER.
 *  
 *  SLURM is free software; you can redistribute it and/or modify it under
 *  the terms of the GNU General Public License as published by the Free
 *  Software Foundation; either version 2 of the License, or (at your option)
 *  any later version.
 *
 *  In addition, as a special exception, the copyright holders give permission 
 *  to link the code of portions of this program with the OpenSSL library under
 *  certain conditions as described in each individual source file, and 
 *  distribute linked combinations including the two. You must obey the GNU 
 *  General Public License in all respects for all of the code used other than 
 *  OpenSSL. If you modify file(s) with this exception, you may extend this 
 *  exception to your version of the file(s), but you are not obligated to do 
 *  so. If you do not wish to do so, delete this exception statement from your
 *  version.  If you delete this exception statement from all source files in 
 *  the program, then also delete it here.
 *  
 *  SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
 *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 *  details.
 *  
 *  You should have received a copy of the GNU General Public License along
 *  with SLURM; if not, write to the Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
\*****************************************************************************/

#ifdef HAVE_CONFIG_H
#  include "config.h"
#  if HAVE_STDINT_H
#    include <stdint.h>
#  endif
#  if HAVE_INTTYPES_H
#    include <inttypes.h>
#  endif
#  if WITH_PTHREADS
#    include <pthread.h>
#  endif
#endif

#include <signal.h>
#include <unistd.h>

#include <slurm/slurm_errno.h>

#include <pwd.h>
#include <sys/types.h>
#include "src/common/hostlist.h"
#include "src/common/list.h"
#include "src/common/macros.h"
#include "src/common/node_select.h"
#include "src/common/uid.h"
#include "src/common/xstring.h"
#include "src/slurmctld/proc_req.h"
#include "src/api/job_info.h"
#include "src/slurmctld/trigger_mgr.h"
#include "src/slurmctld/locks.h"
#include "bluegene.h"

#define _DEBUG 0
#define RETRY_BOOT_COUNT 3

#ifdef HAVE_BG_FILES

typedef struct {
	int jobid;
} kill_job_struct_t;

List kill_job_list = NULL;

static int _block_is_deallocating(bg_record_t *bg_record);
static void _destroy_kill_struct(void *object);

static int _block_is_deallocating(bg_record_t *bg_record)
{
	int jobid = bg_record->job_running;
	char *user_name = NULL;

	if(bg_record->modifying)
		return SLURM_SUCCESS;
	
	user_name = xstrdup(bg_conf->slurm_user_name);
	if(remove_all_users(bg_record->bg_block_id, NULL) 
	   == REMOVE_USER_ERR) {
		error("Something happened removing "
		      "users from block %s", 
		      bg_record->bg_block_id);
	} 	
	
	if(bg_record->target_name && bg_record->user_name) {
		if(!strcmp(bg_record->target_name, user_name)) {
			if(strcmp(bg_record->target_name, bg_record->user_name)
			   || (jobid > NO_JOB_RUNNING)) {
				kill_job_struct_t *freeit =
					xmalloc(sizeof(freeit));
				freeit->jobid = jobid;
				list_push(kill_job_list, freeit);
				
				error("Block %s was in a ready state "
				      "for user %s but is being freed. "
				      "Job %d was lost.",
				      bg_record->bg_block_id,
				      bg_record->user_name,
				      jobid);
			} else {
				debug("Block %s was in a ready state "
				      "but is being freed. No job running.",
				      bg_record->bg_block_id);
			}
		} else {
			error("State went to free on a boot "
			      "for block %s.",
			      bg_record->bg_block_id);
		}
	} else if(bg_record->user_name) {
		error("Target Name was not set "
		      "not set for block %s.",
		      bg_record->bg_block_id);
		bg_record->target_name = xstrdup(bg_record->user_name);
	} else {
		error("Target Name and User Name are "
		      "not set for block %s.",
		      bg_record->bg_block_id);
		bg_record->user_name = xstrdup(user_name);
		bg_record->target_name = xstrdup(bg_record->user_name);
	}

	if(remove_from_bg_list(bg_lists->job_running, bg_record)
	   == SLURM_SUCCESS) 
		num_unused_cpus += bg_record->cpu_cnt;			       
	remove_from_bg_list(bg_lists->booted, bg_record);

	xfree(user_name);
			
	return SLURM_SUCCESS;
}
static void _destroy_kill_struct(void *object)
{
	kill_job_struct_t *freeit = (kill_job_struct_t *)object;

	if(freeit) {
		xfree(freeit);
	}
}

#endif


/*
 * check to see if block is ready to execute.  Meaning
 * User is added to the list of users able to run, and no one 
 * else is running on the block.
 *
 * NOTE: This happens in parallel with srun and slurmd spawning
 * the job. A prolog script is expected to defer initiation of
 * the job script until the BG block is available for use.
 */
extern int block_ready(struct job_record *job_ptr)
{
	int rc = 1;
	char *block_id = NULL;
	bg_record_t *bg_record = NULL;
	
	rc = select_g_select_jobinfo_get(job_ptr->select_jobinfo,
					 SELECT_JOBDATA_BLOCK_ID, &block_id);
	if (rc == SLURM_SUCCESS) {
		slurm_mutex_lock(&block_state_mutex);
		bg_record = find_bg_record_in_list(bg_lists->main, block_id);
		
		if(bg_record) {
			if(bg_record->job_running != job_ptr->job_id) {
				rc = 0;
			} else if ((bg_record->user_uid == job_ptr->user_id)
				   && (bg_record->state 
				       == RM_PARTITION_READY)) {
				rc = 1;
			} else if (bg_record->user_uid != job_ptr->user_id)
				rc = 0;
			else
				rc = READY_JOB_ERROR;	/* try again */
		} else {
			error("block_ready: block %s not in bg_lists->main.",
			      block_id);
			rc = READY_JOB_FATAL;	/* fatal error */
		}
		slurm_mutex_unlock(&block_state_mutex);
		xfree(block_id);
	} else
		rc = READY_JOB_ERROR;
/* 	info("returning %d for job %u %d %d", */
/* 	     rc, job_ptr->job_id, READY_JOB_ERROR, READY_JOB_FATAL); */
	return rc;
}				

/* Pack all relevent information about a block */
extern void pack_block(bg_record_t *bg_record, Buf buffer)
{
	packstr(bg_record->nodes, buffer);
	packstr(bg_record->ionodes, buffer);
	packstr(bg_record->user_name, buffer);
	packstr(bg_record->bg_block_id, buffer);
	pack16((uint16_t)bg_record->state, buffer);
	pack16((uint16_t)bg_record->conn_type, buffer);
#ifdef HAVE_BGL
	pack16((uint16_t)bg_record->node_use, buffer);	
#endif
	pack32((uint32_t)bg_record->node_cnt, buffer);
	pack32((uint32_t)bg_record->job_running, buffer);
	pack_bit_fmt(bg_record->bitmap, buffer);
	pack_bit_fmt(bg_record->ionode_bitmap, buffer);
#ifdef HAVE_BGL
	packstr(bg_record->blrtsimage, buffer);
#endif
	packstr(bg_record->linuximage, buffer);
	packstr(bg_record->mloaderimage, buffer);
	packstr(bg_record->ramdiskimage, buffer);
}

extern int update_block_list()
{
	int updated = 0;
#ifdef HAVE_BG_FILES
	int rc;
	rm_partition_t *block_ptr = NULL;
#ifdef HAVE_BGL
	rm_partition_mode_t node_use;
#endif
	rm_partition_state_t state;
	char *name = NULL;
	bg_record_t *bg_record = NULL;
	time_t now;
	kill_job_struct_t *freeit = NULL;
	ListIterator itr = NULL;
	slurmctld_lock_t job_write_lock = {
		NO_LOCK, WRITE_LOCK, WRITE_LOCK, NO_LOCK };
	
	if(!kill_job_list)
		kill_job_list = list_create(_destroy_kill_struct);

	if(!bg_lists->main) 
		return updated;
	
	slurm_mutex_lock(&block_state_mutex);
	itr = list_iterator_create(bg_lists->main);
	while ((bg_record = (bg_record_t *) list_next(itr)) != NULL) {
		if(!bg_record->bg_block_id)
			continue;
		name = bg_record->bg_block_id;
		if ((rc = bridge_get_block_info(name, &block_ptr)) 
		    != STATUS_OK) {
			if(bg_conf->layout_mode == LAYOUT_DYNAMIC) {
				switch(rc) {
				case INCONSISTENT_DATA:
					debug2("got inconsistent data when "
					       "quering block %s", name);
					continue;
					break;
				case PARTITION_NOT_FOUND:
					debug("block %s not found, removing "
					      "from slurm", name);
					list_remove(itr);
					destroy_bg_record(bg_record);
					continue;
					break;
				default:
					break;
				}
			}
			error("bridge_get_block_info(%s): %s", 
			      name, 
			      bg_err_str(rc));
			continue;
		}
				
#ifdef HAVE_BGL
		if ((rc = bridge_get_data(block_ptr, RM_PartitionMode,
					  &node_use))
		    != STATUS_OK) {
			error("bridge_get_data(RM_PartitionMode): %s",
			      bg_err_str(rc));
			updated = -1;
			goto next_block;
		} else if(bg_record->node_use != node_use) {
			debug("node_use of Block %s was %d "
			      "and now is %d",
			      bg_record->bg_block_id, 
			      bg_record->node_use, 
			      node_use);
			bg_record->node_use = node_use;
			updated = 1;
		}
#else
		if((bg_record->node_cnt < bg_conf->bp_node_cnt) 
		   || (bg_conf->bp_node_cnt == bg_conf->nodecard_node_cnt)) {
			char *mode = NULL;
			uint16_t conn_type = SELECT_SMALL;
			if ((rc = bridge_get_data(block_ptr,
						  RM_PartitionOptions,
						  &mode))
			    != STATUS_OK) {
				error("bridge_get_data(RM_PartitionOptions): "
				      "%s", bg_err_str(rc));
				updated = -1;
				goto next_block;
			} else if(mode) {
				switch(mode[0]) {
				case 's':
					conn_type = SELECT_HTC_S;
					break;
				case 'd':
					conn_type = SELECT_HTC_D;
					break;
				case 'v':
					conn_type = SELECT_HTC_V;
					break;
				case 'l':
					conn_type = SELECT_HTC_L;
					break;
				default:
					conn_type = SELECT_SMALL;
					break;
				}
				free(mode);
			}
			
			if(bg_record->conn_type != conn_type) {
				debug("mode of small Block %s was %u "
				      "and now is %u",
				      bg_record->bg_block_id, 
				      bg_record->conn_type, 
				      conn_type);
				bg_record->conn_type = conn_type;
				updated = 1;
			}
		}
#endif		
		if ((rc = bridge_get_data(block_ptr, RM_PartitionState,
					  &state))
		    != STATUS_OK) {
			error("bridge_get_data(RM_PartitionState): %s",
			      bg_err_str(rc));
			updated = -1;
			goto next_block;
		} else if(bg_record->job_running != BLOCK_ERROR_STATE 
			  //plugin set error
			  && bg_record->state != state) {
			int skipped_dealloc = 0;
			debug("state of Block %s was %d and now is %d",
			      bg_record->bg_block_id, 
			      bg_record->state, 
			      state);
			/* 
			   check to make sure block went 
			   through freeing correctly 
			*/
			if((bg_record->state != RM_PARTITION_DEALLOCATING
			    && bg_record->state != RM_PARTITION_ERROR)
			   && state == RM_PARTITION_FREE)
				skipped_dealloc = 1;
			else if((bg_record->state == RM_PARTITION_READY)
				&& (state == RM_PARTITION_CONFIGURING)) {
				/* This means the user did a reboot through
				   mpirun but we missed the state
				   change */
				debug("Block %s skipped rebooting, "
				      "but it really is.  "
				      "Setting target_name back to %s",
				      bg_record->bg_block_id,
				      bg_record->user_name);
				xfree(bg_record->target_name);
				bg_record->target_name =
					xstrdup(bg_record->user_name);
			}

			bg_record->state = state;

			if(bg_record->state == RM_PARTITION_DEALLOCATING
			   || skipped_dealloc) 
				_block_is_deallocating(bg_record);
#ifndef HAVE_BGL
			else if(bg_record->state == RM_PARTITION_REBOOTING) {
				/* This means the user did a reboot through
				   mpirun */
				debug("Block %s rebooting.  "
				      "Setting target_name back to %s",
				      bg_record->bg_block_id,
				      bg_record->user_name);
				xfree(bg_record->target_name);
				bg_record->target_name =
					xstrdup(bg_record->user_name);
			}
#endif
			else if(bg_record->state == RM_PARTITION_CONFIGURING) 
				bg_record->boot_state = 1;
			else if(bg_record->state == RM_PARTITION_FREE) {
				if(remove_from_bg_list(bg_lists->job_running, 
						       bg_record) 
				   == SLURM_SUCCESS) {
					num_unused_cpus += bg_record->cpu_cnt;
				}
				remove_from_bg_list(bg_lists->booted,
						    bg_record);
			} else if(bg_record->state == RM_PARTITION_ERROR) {
				if(bg_record->boot_state == 1)
					error("Block %s in an error "
					      "state while booting.",
					      bg_record->bg_block_id);
				else					
					error("Block %s in an error state.",
					      bg_record->bg_block_id);
				remove_from_bg_list(bg_lists->booted,
						    bg_record);
				trigger_block_error();
			}
			updated = 1;
			
		}

		/* check the boot state */
		debug3("boot state for block %s is %d",
		       bg_record->bg_block_id,
		       bg_record->boot_state);
		if(bg_record->boot_state == 1) {
			switch(bg_record->state) {
			case RM_PARTITION_CONFIGURING:
				debug3("checking to make sure user %s "
				       "is the user.",
				       bg_record->target_name);
				
				if(update_block_user(bg_record, 0) == 1)
					last_bg_update = time(NULL);
				if(bg_record->job_ptr)
					bg_record->job_ptr->job_state |=
						JOB_CONFIGURING;
				break;
			case RM_PARTITION_ERROR:
				/* If we get an error on boot that
				 * means it is a transparent L3 error
				 * and should be trying to fix
				 * itself.  If this is the case we
				 * just hang out waiting for the state
				 * to go to free where we will try to
				 * boot again below.
				 */
				break;
			case RM_PARTITION_FREE:
				if(bg_record->boot_count < RETRY_BOOT_COUNT) {
					slurm_mutex_unlock(&block_state_mutex);
					if((rc = boot_block(bg_record))
					   != SLURM_SUCCESS) {
						updated = -1;
					}
					slurm_mutex_lock(&block_state_mutex);
					debug("boot count for block "
					      "%s is %d",
					      bg_record->bg_block_id,
					      bg_record->boot_count);
					bg_record->boot_count++;
				} else {
					char reason[128], time_str[32];

					error("Couldn't boot Block %s "
					      "for user %s",
					      bg_record->bg_block_id, 
					      bg_record->target_name);
					slurm_mutex_unlock(&block_state_mutex);
					
					now = time(NULL);
					slurm_make_time_str(&now, time_str,
							    sizeof(time_str));
					snprintf(reason, 
						 sizeof(reason),
						 "update_block_list: "
						 "Boot fails "
						 "[SLURM@%s]", 
						 time_str);
					drain_as_needed(bg_record, reason);
					slurm_mutex_lock(&block_state_mutex);
					bg_record->boot_state = 0;
					bg_record->boot_count = 0;
					if(remove_from_bg_list(
						   bg_lists->job_running, 
						   bg_record) 
					   == SLURM_SUCCESS) {
						num_unused_cpus += 
							bg_record->cpu_cnt;
					} 
					remove_from_bg_list(
						bg_lists->booted,
						bg_record);
				}
				break;
			case RM_PARTITION_READY:
				debug("block %s is ready.",
				      bg_record->bg_block_id);
				if(bg_record->job_ptr)
					bg_record->job_ptr->job_state &=
						(~JOB_CONFIGURING);
				if(set_block_user(bg_record) == SLURM_ERROR) {
					freeit = xmalloc(
						sizeof(kill_job_struct_t));
					freeit->jobid = bg_record->job_running;
					list_push(kill_job_list, freeit);
				}
				break;
			case RM_PARTITION_DEALLOCATING:
				debug2("Block %s is in a deallocating state "
				       "during a boot.  Doing nothing until "
				       "free state.",
				       bg_record->bg_block_id);
				break;
#ifndef HAVE_BGL
			case RM_PARTITION_REBOOTING:
				debug2("Block %s is rebooting.",
				       bg_record->bg_block_id);
				break;
#endif
			default:
				debug("Hey the state of block "
				      "%s is %d(%s) doing nothing.",
				      bg_record->bg_block_id,
				      bg_record->state,
				      bg_block_state_string(bg_record->state));
				break;
			}
		}
	next_block:
		if ((rc = bridge_free_block(block_ptr)) 
		    != STATUS_OK) {
			error("bridge_free_block(): %s", 
			      bg_err_str(rc));
		}				
	}
	list_iterator_destroy(itr);
	slurm_mutex_unlock(&block_state_mutex);
	
	/* kill all the jobs from unexpectedly freed blocks */
	while((freeit = list_pop(kill_job_list))) {
		debug2("Trying to requeue job %d", freeit->jobid);
		lock_slurmctld(job_write_lock);
		if((rc = job_requeue(0, freeit->jobid, -1))) {
			error("couldn't requeue job %u, failing it: %s",
			      freeit->jobid, 
			      slurm_strerror(rc));
			(void) job_fail(freeit->jobid);
		}
		unlock_slurmctld(job_write_lock);
		_destroy_kill_struct(freeit);
	}
		
#endif
	return updated;
}

extern int update_freeing_block_list()
{
	int updated = 0;
#ifdef HAVE_BG_FILES
	int rc;
	rm_partition_t *block_ptr = NULL;
	rm_partition_state_t state;
	char *name = NULL;
	bg_record_t *bg_record = NULL;
	ListIterator itr = NULL;
	
	if(!bg_lists->freeing) 
		return updated;
	
	slurm_mutex_lock(&block_state_mutex);
	itr = list_iterator_create(bg_lists->freeing);
	while ((bg_record = (bg_record_t *) list_next(itr)) != NULL) {
		if(!bg_record->bg_block_id)
			continue;

		name = bg_record->bg_block_id;
		if ((rc = bridge_get_block_info(name, &block_ptr)) 
		    != STATUS_OK) {
			if(bg_conf->layout_mode == LAYOUT_DYNAMIC) {
				switch(rc) {
				case INCONSISTENT_DATA:
					debug2("got inconsistent data when "
					       "quering block %s", name);
					continue;
					break;
				case PARTITION_NOT_FOUND:
					debug("block %s not found, removing "
					      "from slurm", name);
					list_remove(itr);
					destroy_bg_record(bg_record);
					continue;
					break;
				default:
					break;
				}
			}
			error("bridge_get_block_info(%s): %s", 
			      name, 
			      bg_err_str(rc));
			continue;
		}
				
		if ((rc = bridge_get_data(block_ptr, RM_PartitionState,
					  &state))
		    != STATUS_OK) {
			error("bridge_get_data(RM_PartitionState): %s",
			      bg_err_str(rc));
			updated = -1;
			goto next_block;
		} else if(bg_record->state != state) {
			debug("freeing state of Block %s was %d and now is %d",
			      bg_record->bg_block_id, 
			      bg_record->state, 
			      state);

			bg_record->state = state;
			updated = 1;
		}
	next_block:
		if ((rc = bridge_free_block(block_ptr)) 
		    != STATUS_OK) {
			error("bridge_free_block(): %s", 
			      bg_err_str(rc));
		}
	}
	list_iterator_destroy(itr);
	slurm_mutex_unlock(&block_state_mutex);
		
#endif
	return updated;
}
