stonithd.c

linux集群服务器软件代码包

/* $Id: stonithd.c,v 1.17 2005/02/17 08:21:42 sunjd Exp $ */

/* File: stonithd.c
 * Description: STONITH daemon for node fencing
 *
 * Author: Sun Jiang Dong <[e-mail]>
 * Copyright (c) 2004 International Business Machines
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
/* Todo: 
 * 1. Change to a obvious DFA?
 * 2. uuid support
 * 3. How to make a stonith object as a master object? Only depend to the 
 *    stonith plugins?
 */

#include <config.h>
#include <portability.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <unistd.h>
#include <fcntl.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/wait.h>
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif /* HAVE_GETOPT_H */
#include <errno.h>
#include <glib.h>
#include <pils/plugin.h>
#include <stonith/stonith.h>
#include <pils/generic.h>
#include <clplumbing/cl_signal.h>
#include <clplumbing/uids.h>
#include <clplumbing/cl_log.h>
#include <clplumbing/lsb_exitcodes.h>
#include <clplumbing/proctrack.h>
#include <clplumbing/GSource.h>
#include <clplumbing/cl_log.h>

#include <apphb.h>
#include <ha_msg.h>
#include <hb_api.h>
#include <lrm/raexec.h>
#include <fencing/stonithd_msg.h>
#include <fencing/stonithd_api.h>

/* For integration with heartbeat */
#define MAGIC_EC 100

typedef struct {
	char * name;
	pid_t pid;		/* client pid */
	uid_t uid;		/* client UID */
	gid_t gid;		/* client GID */
	IPC_Channel * ch;
	char * removereason;
} stonithd_client_t;

typedef enum {
	STONITH_RA_OP,	/* stonith resource operation */
	STONITH_INIT,	/* stonith operation initiated by myself */
	STONITH_REQ,	/* stonith operation required by others */
} operate_scenario_t;

typedef struct {
	operate_scenario_t  scenario;
	void *	result_receiver;	/* final result receiver -- IPC_Channel
					 * or a node name/uuid */
	union {
		stonith_ops_t	* st_op;
		stonithRA_ops_t * ra_op;
	} op_union;
	void * data;	/* private data */
} common_op_t;

typedef struct stonith_rsc
{
	char *    rsc_id;
	char *    ra_name;
	GHashTable *    params;
	Stonith *	stonith_obj;
	char **		node_list;
} stonith_rsc_t;

/* Must correspond to stonith_type_t */
/*
static const char * stonith_op_strname[] =
{
	"QUERY", "RESET", "POWERON", "POWERON"
};
*/

static GList * client_list = NULL;
static GHashTable * executing_queue = NULL;
static GList * local_started_stonith_rsc = NULL;
static int negative_callid_counter = -2;

typedef int (*stonithd_api_msg_handler)(const struct ha_msg * msg,
					gpointer data);

struct api_msg_to_handler
{
	const char *			msg_type;
	stonithd_api_msg_handler	handler;
};

typedef int (*RA_subop_handler)(stonithRA_ops_t * ra_op, gpointer data);

struct RA_operation_to_handler
{
	const char *			op_type;
	RA_subop_handler		handler;
	RA_subop_handler		post_handler;
};

/* Miscellaneous functions such as daemon routines and others. */
static void become_daemon(gboolean);
static void show_daemon_status(const char * pidfile);
static int kill_running_daemon(const char * pidfile);
static pid_t running_daemon_pid(const char * pidfile);
static int create_pidfile(const char * pidfile);
static void stonithd_log(int priority, const char * fmt, ...)G_GNUC_PRINTF(2,3);
static void stonithd_quit(int signo);

/* Dealing with the child quit/abort event when executing STONTIH RA plugins. 
 */
static void child_quit(int signo);
static gboolean on_polled_input_prepare(GSource * source, gint * timeout);
static gboolean on_polled_input_check(GSource * source);
static gboolean on_polled_input_dispatch(GSource * source, GSourceFunc callback,
					 gpointer user_data);

/* For application heartbeat related */
static const unsigned long
        DEFAULT_APPHB_INTERVAL  = 2000, /* MS */
        DEFAULT_APPHB_WARNTIME  = 6000, /* MS */
	APPHB_INTVL_DETLA 	= 30;

#define MY_APPHB_HB \
	if (SIGNONED_TO_APPHBD == TRUE) { \
		if (apphb_hb() != 0) { \
			SIGNONED_TO_APPHBD = FALSE; \
		} \
	}

/* 
 * Functions related to application heartbeat ( apphbd )
 */
static gboolean emit_apphb(gpointer data);
static int init_using_apphb(void);

/* Communication between nodes related.
 * For stonithing one node in the cluster.
 */
static void handle_msg_twhocan(const struct ha_msg* msg, void* private_data);
static void handle_msg_ticanst(const struct ha_msg* msg, void* private_data);
static void handle_msg_tstit(const struct ha_msg* msg, void* private_data);
static void handle_msg_trstit(const struct ha_msg* msg, void* private_data);
static gboolean stonithd_hb_msg_dispatch(IPC_Channel * ch, gpointer user_data);
static void stonithd_hb_msg_dispatch_destroy(gpointer user_data);
static int init_hb_msg_handler(void);

/* Local IPC communication related.
 * For implementing the functions for client APIs 
 */
static gboolean stonithd_client_dispatch(IPC_Channel * ch, gpointer user_data);
static void stonithd_IPC_destroy_notify(gpointer data);
static gboolean accept_client_dispatch(IPC_Channel * ch, gpointer data);
static gboolean stonithd_process_client_msg(struct ha_msg * msg, 
					    gpointer data);
static int init_client_API_handler(void);
static void free_client(stonithd_client_t * client);
static stonithd_client_t * get_exist_client_by_chan(GList * client_list, 
						    IPC_Channel * ch);
static int delete_client_by_chan(GList ** client_list, IPC_Channel * ch);

/* Client API functions */
static int on_stonithd_signon(const struct ha_msg * msg, gpointer data);
static int on_stonithd_signoff(const struct ha_msg * msg, gpointer data);
static int on_stonithd_node_fence(const struct ha_msg * request, gpointer data);
static int on_stonithd_virtual_stonithRA_ops(const struct ha_msg * request, 
					  gpointer data);
static int on_stonithd_list_stonith_types(const struct ha_msg * request,
					  gpointer data);
static int stonithRA_operate(	stonithRA_ops_t * op, gpointer data );
static int stonithRA_start( stonithRA_ops_t * op, gpointer data );
static int stonithRA_stop( stonithRA_ops_t * op, gpointer data );
static int stonithRA_monitor( stonithRA_ops_t * op, gpointer data );
static int stonithRA_start_post( stonithRA_ops_t * op, gpointer data );
static int stonithRA_stop_post( stonithRA_ops_t * op, gpointer data );
static int stonithRA_monitor_post( stonithRA_ops_t * op, gpointer data );
static int stonithop_result_to_local_client(stonith_ops_t * st_op, 
					    gpointer data);
static int send_stonithop_final_result( common_op_t * op );
static int stonithop_result_to_other_node( stonith_ops_t * st_op,
					   gpointer data);
static int send_stonithRAop_final_result(stonithRA_ops_t * ra_op, 
					 gpointer data);
static int post_handle_raop(stonithRA_ops_t * ra_op);
static void destory_key_of_op_htable(gpointer data);
static void free_stonithRA_ops_t(stonithRA_ops_t * ra_op);
static void free_stonith_ops_t(stonith_ops_t * st_op);
static void free_common_op_t(gpointer data);
static void free_stonith_rsc(stonith_rsc_t * srsc);
static stonith_rsc_t * get_started_stonith_resource(char * rsc_id);
static stonith_rsc_t * get_local_stonithobj_can_stonith(const char * node_name,
						const char * begin_rsc_id );
static int stonith_operate_locally(stonith_ops_t * st_op, stonith_rsc_t * srsc);
static gboolean stonithop_timeout(gpointer data);
static void my_hash_table_find( GHashTable * htable, gpointer * orig_key,
				gpointer * value, gpointer user_data);
static void has_this_callid(gpointer key, gpointer value,
				gpointer user_data);
static int require_others_to_stonith(stonith_ops_t * st_op);
static int initiate_local_stonithop(stonith_ops_t * st_op, stonith_rsc_t * srsc, 
				    IPC_Channel * ch);
static int continue_local_stonithop(int old_key);
static int initiate_remote_stonithop(stonith_ops_t * st_op, stonith_rsc_t * srsc,
				     IPC_Channel * ch);
static int changeto_remote_stonithop(int old_key);
static int require_local_stonithop(stonith_ops_t * st_op, stonith_rsc_t * srsc, 
				   const char * asker_node);

static struct api_msg_to_handler api_msg_to_handlers[] = {
	{ ST_SIGNON,	on_stonithd_signon },
	{ ST_SIGNOFF,	on_stonithd_signoff },
	{ ST_STONITH,	on_stonithd_node_fence },
	{ ST_RAOP,	on_stonithd_virtual_stonithRA_ops },
	{ ST_LTYPES,	on_stonithd_list_stonith_types },
};

static struct RA_operation_to_handler raop_handler[] = {
	{ "start",	stonithRA_start, 	stonithRA_start_post },
	{ "stop",	stonithRA_stop,		stonithRA_stop_post },
	{ "monitor",	stonithRA_monitor,	stonithRA_monitor_post },
};

#define PID_FILE        HA_VARRUNDIR"/stonithd.pid"

/* define the message type between stonith daemons on different nodes */
#define T_WHOCANST  	"whocanst"	/* who can stonith a node */
#define T_ICANST	"icanst" 	/* I can stonith a node */	
#define T_STIT		"stit" 		/* please stonith it */	
#define T_RSTIT		"rstit" 	/* result of stonithing it */	
#define T_IALIVE 	"iamalive"	/* I'm alive */
#define T_QSTCAP	"qstcap"	/* query the stonith capacity -- 
					   all the nodes who can stonith */

/* 
 * Notice log messages for other programs, such as scripts to judge the 
 * status of this stonith daemon.
 */
static const char * M_STARTUP = "start up successfully.",
		  * M_RUNNING = "is already running.",
		  * M_QUIT    = "normally quit.",
		  * M_ABORT   = "abnormally abort.";

static const char * simple_help_screen =
"Usage: stonithd [-nskdh]\n"
"	-n	Do not register to apphbd. Now donnot register to apphbd by default.\n"
"	-s	Show the status of the daemons.\n"
"	-k	Kill the daemon.\n"
"	-d	Run the stonithd in debug mode. Under debug mode more\n"
"		debug information is written to log file.\n"
"	-a	Start up alone outside of heartbeat.\n" 
"		By default suppose it be started up and monitored by heartbeat.\n"
"	-h	This help information\n";

static const char * optstr = "anskdht";
/* Will replace it with dynamical a config variable */
#define PIDFILE         "/var/run/stonithd.pid"

/* Do not need itselv's log file for real wotk, only for debugging
#define DAEMON_LOG      "/var/log/stonithd.log"
#define DAEMON_DEBUG    "/var/log/stonithd.debug"
*/

static GSourceFuncs polled_input_SourceFuncs = {
	on_polled_input_prepare,
	on_polled_input_check,
	on_polled_input_dispatch,
	NULL,
};

static const char * 	local_nodename		= NULL;
static GMainLoop *	mainloop 		= NULL;
static const char * 	stonithd_name		= "stonithd";
static gboolean		STARTUP_ALONE 		= FALSE;
static gboolean 	SIGNONED_TO_HB		= FALSE;
static gboolean 	SIGNONED_TO_APPHBD	= FALSE;
static gboolean 	NEED_SIGNON_TO_APPHBD	= FALSE;
static ll_cluster_t *	hb			= NULL;
static gboolean 	childs_quit		= FALSE;
static gboolean 	DEBUG_MODE		= FALSE;
static gboolean 	TEST 			= FALSE;

int main(int argc, char ** argv)
{
	int main_rc = LSB_EXIT_OK;
	int option_char;

        cl_log_set_entity(stonithd_name);
	cl_log_enable_stderr(TRUE);
	cl_log_set_facility(LOG_DAEMON);

	do {
		option_char = getopt(argc, argv, optstr);

		if (option_char == -1) {
			break;
		}

		switch (option_char) {
			case 'a': /* Start up alone */
				STARTUP_ALONE = TRUE;
				break;

			case 'n': /* Do not register to apphbd */
				NEED_SIGNON_TO_APPHBD = FALSE;
				break;
			
			case 's': /* Show daemon status */
				show_daemon_status(PIDFILE);
				return (STARTUP_ALONE == TRUE) ? 
					LSB_EXIT_OK : MAGIC_EC;
				break; /* Never reach here, just for uniform */

			case 'k': /* kill the running daemon */
				return(kill_running_daemon(PIDFILE));
				break; /* Never reach here, just for uniform */

			case 'd': /* Run with debug mode */
				DEBUG_MODE = TRUE;
				/* adjust the PILs' debug level */ 
				PILpisysSetDebugLevel(1);
				break;

			case 'h':
				printf("%s\n",simple_help_screen);
				return (STARTUP_ALONE == TRUE) ? 
					LSB_EXIT_OK : MAGIC_EC;

			case 't': /* test only */
				TEST = TRUE;	
				break;

			default:
				stonithd_log(LOG_ERR, "Error:getopt returned"
					" character code %c.", option_char);
				printf("%s\n", simple_help_screen);
				return (STARTUP_ALONE == TRUE) ? 
					LSB_EXIT_EINVAL : MAGIC_EC;
		}
	} while (1);

	if ( running_daemon_pid(PIDFILE) > 0 ) {
		stonithd_log(LOG_NOTICE, "%s %s", argv[0], M_RUNNING);
		return (STARTUP_ALONE == TRUE) ? LSB_EXIT_OK : MAGIC_EC;
	}

	/* Not use daemon() API, since it's not POSIX compliant */
	become_daemon(STARTUP_ALONE);

	hb = ll_cluster_new("heartbeat");
	if ( hb == NULL ) {
		stonithd_log(LOG_ERR, "ll_cluster_new failed.");
		stonithd_log(LOG_ERR, "%s %s", argv[0], M_ABORT);
		return (STARTUP_ALONE == TRUE) ? LSB_EXIT_GENERIC : MAGIC_EC;
	}

	if (hb->llc_ops->signon(hb, stonithd_name)!= HA_OK) {
		stonithd_log(LOG_ERR, "Cannot signon with heartbeat");
		stonithd_log(LOG_ERR, "REASON: %s", hb->llc_ops->errmsg(hb));
		goto delhb_quit;
	} else {
		stonithd_log(LOG_INFO, "Signing in with heartbeat.");
		SIGNONED_TO_HB = TRUE;
		local_nodename = hb->llc_ops->get_mynodeid(hb);
		if (local_nodename == NULL) {
			stonithd_log(LOG_ERR, "Cannot get local node id");
			stonithd_log(LOG_ERR, "REASON: %s", 
				     hb->llc_ops->errmsg(hb));
			main_rc = LSB_EXIT_GENERIC;
			goto signoff_quit;
		}
	}

	mainloop = g_main_new(FALSE);

	/*
	 * Initialize the handler of the child quit event. Since the stonith 
	 * plugin will be runned in a child process.
	 */
	if ( NULL == G_main_add_input(G_PRIORITY_DEFAULT, FALSE,
                                        &polled_input_SourceFuncs)) {
		stonithd_log(LOG_ERR, "G_main_add_input failed.");
		stonithd_log(LOG_ERR, "Startup aborted.");
		main_rc = LSB_EXIT_GENERIC;
		goto signoff_quit;
	}
	stonithd_log(LOG_DEBUG, "address %p", &polled_input_SourceFuncs);

	/*
	 * Initialize the handler of IPC messages from hearbteat, including
	 * the messages produced by myself as a client of heartbeat.
	 */
	if ( (main_rc = init_hb_msg_handler()) != 0) {
		stonithd_log(LOG_ERR, "An error in init_hb_msg_handler.");
		goto signoff_quit;
	}
	
	/*
	 * Initialize the handler of IPC messages from my clients.
	 */
	if ( (main_rc = init_client_API_handler()) != 0) {
		stonithd_log(LOG_ERR, "An error in init_hb_msg_handler.");
		goto delhb_quit;
	}

	if (NEED_SIGNON_TO_APPHBD == TRUE) {
		if ( (main_rc=init_using_apphb()) != 0 ) {
			stonithd_log(LOG_ERR, "An error in init_using_apphb");
			goto signoff_quit;
		} else {
			SIGNONED_TO_APPHBD = TRUE;
		}
	}

	/* Initialize some global variables */