fsa.c

在LINUX下实现HA的源代码

/* 
 * Copyright (C) 2004 Andrew Beekhof <[e-mail]>
 * 
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */
#include <crm/crm.h>
#include <crmd_fsa.h>
#include <fsa_matrix.h>
#include <fsa_proto.h>
#include <stdio.h>

#include <crm/common/xmlutils.h>
#include <crm/common/msgutils.h>
#include <crm/msg_xml.h>

#include <clplumbing/Gmain_timeout.h>

#include <crmd_messages.h>
#include <string.h>
#include <time.h>

#include <crm/dmalloc_wrapper.h>

long long
do_state_transition(long long actions,
		    enum crmd_fsa_cause cause,
		    enum crmd_fsa_state cur_state,
		    enum crmd_fsa_state next_state,
		    enum crmd_fsa_input current_input,
		    void *data);

#ifdef DOT_FSA_ACTIONS
# ifdef FSA_TRACE
#  define ELSEIF_FSA_ACTION(x,y)					\
     else if(is_set(actions,x)) {					\
	CRM_DEBUG3("Invoking action %s (%.16llx)",			\
		fsa_action2string(x), x);				\
	actions = clear_bit(actions, x);				\
	next_input = y(x, cause, cur_state, last_input, data);		\
	if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT )	\
		fprintf(dot_strm,					\
			"\t// %s:\t%s\t(data? %s)\t(result=%s)\n",	\
			fsa_input2string(cur_input),			\
			fsa_action2string(x),				\
			data==NULL?"no":"yes",				\
			fsa_input2string(next_input));			\
	fflush(dot_strm);						\
	CRM_DEBUG3("Result of action %s was %s",			\
		fsa_action2string(x), fsa_input2string(next_input));	\
     }
# else
#  define ELSEIF_FSA_ACTION(x,y)					\
     else if(is_set(actions,x)) {					\
	actions = clear_bit(actions, x);				\
	next_input = y(x, cause, cur_state, last_input, data);		\
	if( (x & O_DC_TICKLE) == 0 && next_input != I_DC_HEARTBEAT )	\
		fprintf(dot_strm,					\
			"\t// %s:\t%s\t(data? %s)\t(result=%s)\n",	\
			fsa_input2string(cur_input),			\
			fsa_action2string(x),				\
			data==NULL?"no":"yes",				\
			fsa_input2string(next_input));			\
	fflush(dot_strm);						\
     }
# endif
#else
# ifdef FSA_TRACE
#  define ELSEIF_FSA_ACTION(x,y)					\
     else if(is_set(actions,x)) {					\
	CRM_DEBUG3("Invoking action %s (%.16llx)",			\
		fsa_action2string(x), x);				\
	actions = clear_bit(actions, x);				\
	next_input = y(x, cause, cur_state, last_input, data);		\
	CRM_DEBUG3("Result of action %s was %s",			\
		fsa_action2string(x), fsa_input2string(next_input));	\
     }
# else
#  define ELSEIF_FSA_ACTION(x,y)					\
     else if(is_set(actions,x)) {					\
	actions = clear_bit(actions, x);				\
	next_input = y(x, cause, cur_state, last_input, data);		\
     }
# endif
#endif

const char *dot_intro = "digraph \"g\" {\n"
"	size = \"30,30\"\n"
"	graph [\n"
"		fontsize = \"12\"\n"
"		fontname = \"Times-Roman\"\n"
"		fontcolor = \"black\"\n"
"		bb = \"0,0,398.922306,478.927856\"\n"
"		color = \"black\"\n"
"	]\n"
"	node [\n"
"		fontsize = \"12\"\n"
"		fontname = \"Times-Roman\"\n"
"		fontcolor = \"black\"\n"
"		shape = \"ellipse\"\n"
"		color = \"black\"\n"
"	]\n"
"	edge [\n"
"		fontsize = \"12\"\n"
"		fontname = \"Times-Roman\"\n"
"		fontcolor = \"black\"\n"
"		color = \"black\"\n"
"	]\n"
"// special nodes\n"
"	\"S_PENDING\" \n"
"	[\n"
"	 color = \"blue\"\n"
"	 fontcolor = \"blue\"\n"
"	 ]\n"
"	\"S_TERMINATE\" \n"
"	[\n"
"	 color = \"red\"\n"
"	 fontcolor = \"red\"\n"
"	 ]\n"
"\n"
"// DC only nodes\n"
"	\"S_RECOVERY_DC\" [ fontcolor = \"green\" ]\n"
"	\"S_INTEGRATION\" [ fontcolor = \"green\" ]\n"
"	\"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]\n"
"	\"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]\n"
"	\"S_RELEASE_DC\" [ fontcolor = \"green\" ]\n"
"	\"S_IDLE\" [ fontcolor = \"green\" ]\n";


static FILE *dot_strm = NULL;

enum crmd_fsa_state fsa_state;
oc_node_list_t *fsa_membership_copy;
ll_cluster_t   *fsa_cluster_conn;
ll_lrm_t       *fsa_lrm_conn;
long long       fsa_input_register;
const char     *fsa_our_uname;

fsa_timer_t *election_trigger = NULL;		/*  */
fsa_timer_t *election_timeout = NULL;		/*  */
fsa_timer_t *shutdown_escalation_timmer = NULL;	/*  */
fsa_timer_t *integration_timer = NULL;
fsa_timer_t *dc_heartbeat = NULL;

long long
toggle_bit(long long action_list, long long action)
{
/*	CRM_DEBUG2("Toggling bit %.16llx", action); */
	action_list ^= action;
/*	CRM_DEBUG2("Result %.16llx", action_list & action); */
	return action_list;
}

long long
clear_bit(long long action_list, long long action)
{
/*	CRM_DEBUG2("Clearing bit\t%.16llx", action); */

	/* ensure its set */
	action_list |= action;

	/* then toggle */
	action_list = action_list ^ action;

	return action_list;
}

long long
set_bit(long long action_list, long long action)
{
/*	CRM_DEBUG2("Adding bit\t%.16llx", action); */
	action_list |= action;
	return action_list;
}

void
toggle_bit_inplace(long long *action_list, long long action)
{
	*action_list = toggle_bit(*action_list, action);
}

void
clear_bit_inplace(long long *action_list, long long action)
{
	*action_list = clear_bit(*action_list, action);
}

void
set_bit_inplace(long long *action_list, long long action)
{
	*action_list = set_bit(*action_list, action);
}



gboolean
is_set(long long action_list, long long action)
{
/*	CRM_DEBUG2("Checking bit\t%.16llx", action); */
	return ((action_list & action) == action);
}

gboolean
startTimer(fsa_timer_t *timer)
{
	if(((int)timer->source_id) < 0) {
		timer->source_id =
			Gmain_timeout_add(timer->period_ms,
					  timer->callback,
					  (void*)timer);
/*
		CRM_DEBUG3("#!!#!!# Started %s timer (%d)",
			   fsa_input2string(timer->fsa_input),
			   timer->source_id);
*/
	} else {
		cl_log(LOG_INFO, "#!!#!!# Timer %s already running (%d)",
		       fsa_input2string(timer->fsa_input),
		       timer->source_id);
		return FALSE;		
	}
	return TRUE;
}


gboolean
stopTimer(fsa_timer_t *timer)
{
	if(((int)timer->source_id) > 0) {
/*
		CRM_DEBUG3("#!!#!!# Stopping %s timer (%d)",
			   fsa_input2string(timer->fsa_input),
			   timer->source_id);
*/
		g_source_remove(timer->source_id);
		timer->source_id = -2;

	} else {
		cl_log(LOG_INFO, "#!!#!!# Timer %s already stopped (%d)",
		       fsa_input2string(timer->fsa_input),
		       timer->source_id);
		return FALSE;
	}
	return TRUE;
}

enum crmd_fsa_state
s_crmd_fsa(enum crmd_fsa_cause cause,
	   enum crmd_fsa_input initial_input,
	   void *data)
{
	long long           actions = A_NOTHING, new_actions = A_NOTHING;
	enum crmd_fsa_input last_input = initial_input;
	enum crmd_fsa_input cur_input;
	enum crmd_fsa_input next_input;
	enum crmd_fsa_state last_state, cur_state, next_state, starting_state;
	
	FNIN();

	starting_state = fsa_state;
	cur_input  = initial_input;
	next_input = initial_input;
	
	last_state = starting_state;
	cur_state  = starting_state;
	next_state = starting_state;

#ifdef FSA_TRACE
	CRM_DEBUG4("FSA invoked with Cause: %s\n\tState: %s, Input: %s",
		   fsa_cause2string(cause),
		   fsa_state2string(cur_state),
		   fsa_input2string(cur_input));
#endif

	if(dot_strm == NULL) {
		dot_strm = fopen("/tmp/live.dot", "w");
		fprintf(dot_strm, "%s", dot_intro);
	}
	
	/*
	 * Process actions in order of priority but do only one
	 * action at a time to avoid complicating the ordering.
	 *
	 * Actions may result in a new I_ event, these are added to
	 * (not replace) existing actions before the next iteration.
	 *
	 */
	while(next_input != I_NULL || actions != A_NOTHING) {

		if(next_input == I_WAIT_FOR_EVENT) {
			/* we may be waiting for an a-sync task to "happen"
			 * and until it does, we cant do anything else
			 */
			cl_log(LOG_INFO, "Wait until something else happens");
			break;
		}

		cur_input = next_input;

#ifdef FSA_TRACE
		CRM_DEBUG3("FSA while loop:\tState: %s, Input: %s",
			   fsa_state2string(cur_state),
			   fsa_input2string(cur_input));
#endif		

		new_actions = crmd_fsa_actions[cur_input][cur_state];
		next_state  = crmd_fsa_state[cur_input][cur_state];
		last_state  = cur_state;
		cur_state   = next_state;
		fsa_state   = next_state;

		if(new_actions != A_NOTHING) {
#ifdef FSA_TRACE
			CRM_DEBUG2("Adding actions %.16llx", new_actions);
#endif
			actions |= new_actions;
		}

		switch(cur_input) {
			case I_NULL:
				break;
#if 0
			case I_SOME_EVENT:
			case I_SOME_OTHER_EVENT:
				cc_transitioner(cur_input, data);
				/* flow through... */
#endif
			default:
				last_input = cur_input;
				break;
		}

		/*
		 * Hook for change of state.
		 * Allows actions to be added or removed when entering a state
		 */
		if(last_state != cur_state){
			actions = do_state_transition(actions, cause,
						      last_state, cur_state,
						      last_input, data);
		}

		/* this is always run, some inputs/states may make various
		 * actions irrelevant/invalid
		 */
		actions = clear_flags(actions, cause, cur_state, cur_input);

		/* regular action processing in order of action priority
		 *
		 * Make sure all actions that connect to required systems
		 * are performed first
		 */
		if(actions == A_NOTHING) {

			cl_log(LOG_INFO, "Nothing to do");
			next_input = I_NULL;
		
#if 0
			// check registers, see if anything is pending
			if(is_set(fsa_input_register, R_SHUTDOWN)) {
				CRM_DEBUG("(Re-)invoking shutdown");
				next_input = I_SHUTDOWN;
			} else if(is_set(fsa_input_register, R_INVOKE_PE)) {
				CRM_DEBUG("Invoke the PE somehow");
			}
#endif
	}
	
	/* logging */
	ELSEIF_FSA_ACTION(A_ERROR, do_log)
		ELSEIF_FSA_ACTION(A_WARN, do_log)
		ELSEIF_FSA_ACTION(A_LOG,  do_log)
		
		/* get out of here NOW! before anything worse happens */
		ELSEIF_FSA_ACTION(A_EXIT_1,	do_exit)
		
		ELSEIF_FSA_ACTION(A_STARTUP,	do_startup)
		
		ELSEIF_FSA_ACTION(A_CIB_START,  do_cib_control)
		ELSEIF_FSA_ACTION(A_HA_CONNECT, do_ha_control)
		ELSEIF_FSA_ACTION(A_LRM_CONNECT,do_lrm_control)
		ELSEIF_FSA_ACTION(A_CCM_CONNECT,do_ccm_control)
		ELSEIF_FSA_ACTION(A_ANNOUNCE,	do_announce)
		
		/* sub-system start */
		ELSEIF_FSA_ACTION(A_PE_START,	do_pe_control)
		ELSEIF_FSA_ACTION(A_TE_START,	do_te_control)
		
		/* sub-system restart
		 */
		ELSEIF_FSA_ACTION(O_CIB_RESTART,do_cib_control)
		ELSEIF_FSA_ACTION(O_PE_RESTART, do_pe_control)
		ELSEIF_FSA_ACTION(O_TE_RESTART, do_te_control)
		
		ELSEIF_FSA_ACTION(A_STARTED,	do_started)
		
		/* DC Timer */
		ELSEIF_FSA_ACTION(O_DC_TIMER_RESTART,	do_dc_timer_control)
		ELSEIF_FSA_ACTION(A_DC_TIMER_STOP,	do_dc_timer_control)
		ELSEIF_FSA_ACTION(A_DC_TIMER_START,	do_dc_timer_control)
		
		/*
		 * Highest priority actions
		 */
		ELSEIF_FSA_ACTION(A_SHUTDOWN_REQ,	do_shutdown_req)
		ELSEIF_FSA_ACTION(A_MSG_ROUTE,		do_msg_route)
		ELSEIF_FSA_ACTION(A_RECOVER,		do_recover)
		ELSEIF_FSA_ACTION(A_ELECTION_VOTE,	do_election_vote)
		ELSEIF_FSA_ACTION(A_ELECT_TIMER_START,	do_election_timer_ctrl)
		ELSEIF_FSA_ACTION(A_ELECT_TIMER_STOP,	do_election_timer_ctrl)
		ELSEIF_FSA_ACTION(A_ELECTION_COUNT,	do_election_count_vote)
		ELSEIF_FSA_ACTION(A_ELECTION_TIMEOUT,	do_election_timer_ctrl)
		
		/*
		 * "Get this over with" actions
		 */
		ELSEIF_FSA_ACTION(A_MSG_STORE,		do_msg_store)
		ELSEIF_FSA_ACTION(A_NODE_BLOCK,		do_node_block)
		
		/*
		 * High priority actions
		 * Update the cache first
		 */
		ELSEIF_FSA_ACTION(A_CCM_UPDATE_CACHE,	do_ccm_update_cache)
		ELSEIF_FSA_ACTION(A_CCM_EVENT,		do_ccm_event)
		
		/*
		 * Medium priority actions
		 */
		ELSEIF_FSA_ACTION(A_DC_TAKEOVER,	do_dc_takeover)
		ELSEIF_FSA_ACTION(A_DC_RELEASE,		do_dc_release)
		ELSEIF_FSA_ACTION(A_JOIN_WELCOME_ALL,	do_send_welcome)
		ELSEIF_FSA_ACTION(A_JOIN_WELCOME,	do_send_welcome)
		ELSEIF_FSA_ACTION(A_JOIN_ACK,		do_ack_welcome)
		ELSEIF_FSA_ACTION(A_JOIN_PROCESS_ACK,	do_process_welcome_ack)
		
		/*
		 * Low(er) priority actions
		 * Make sure the CIB is always updated before invoking the
		 * PE, and the PE before the TE
		 */
		ELSEIF_FSA_ACTION(A_UPDATE_NODESTATUS,	do_lrm_invoke)
		ELSEIF_FSA_ACTION(A_CIB_INVOKE_LOCAL,	do_cib_invoke)
		ELSEIF_FSA_ACTION(A_CIB_INVOKE,		do_cib_invoke)
		ELSEIF_FSA_ACTION(A_CIB_BUMPGEN,	do_cib_invoke)
		ELSEIF_FSA_ACTION(A_LRM_INVOKE,		do_lrm_invoke)
		ELSEIF_FSA_ACTION(A_LRM_EVENT,		do_lrm_event)
		ELSEIF_FSA_ACTION(A_PE_INVOKE,		do_pe_invoke)
		ELSEIF_FSA_ACTION(A_TE_INVOKE,		do_te_invoke)
		
		/* sub-system stop */
		ELSEIF_FSA_ACTION(A_PE_STOP,		do_pe_control)
		ELSEIF_FSA_ACTION(A_TE_STOP,		do_te_control)
		ELSEIF_FSA_ACTION(A_DC_RELEASED,	do_dc_release)

		ELSEIF_FSA_ACTION(A_HA_DISCONNECT,	do_ha_control)
		ELSEIF_FSA_ACTION(A_CCM_DISCONNECT,	do_ccm_control)
		ELSEIF_FSA_ACTION(A_LRM_DISCONNECT,	do_lrm_control)
		ELSEIF_FSA_ACTION(A_CIB_STOP,		do_cib_control)		
		/* time to go now... */
		
		/* Some of these can probably be consolidated */
		ELSEIF_FSA_ACTION(A_SHUTDOWN,   do_shutdown)
		ELSEIF_FSA_ACTION(A_STOP,	do_stop)
		
		/* exit gracefully */
		ELSEIF_FSA_ACTION(A_EXIT_0,	do_exit)

/*		ELSEIF_FSA_ACTION(A_, do_) */
		
		else if(is_message()) {
			xmlNodePtr stored_msg = NULL;
			
			fsa_message_queue_t msg = get_message();
			
			if(msg == NULL || msg->message == NULL) {
				cl_log(LOG_ERR,
				       "Invalid stored message");
				continue;
			}
			
			data = msg->message;

#ifdef DOT_FSA_ACTIONS
			fprintf(dot_strm,
				"\t// %s:\t%s\t(data? %s)",	
				fsa_input2string(cur_input),
				fsa_action2string(A_MSG_PROCESS),
				stored_msg==NULL?"no":"yes");
			fflush(dot_strm);
#endif
#ifdef FSA_TRACE
			CRM_DEBUG3("Invoking action %s (%.16llx)",
				   fsa_action2string(A_MSG_PROCESS),
				   A_MSG_PROCESS);
#endif

			stored_msg = (xmlNodePtr)data;

#ifdef FSA_TRACE
			xml_message_debug(stored_msg,"FSA processing message");
#endif

			next_input = handle_message(stored_msg);

#ifdef DOT_FSA_ACTIONS
			fprintf(dot_strm, "\t(result=%s)\n",
				fsa_input2string(next_input));
#endif
			CRM_DEBUG3("Result of action %s was %s",
				   fsa_action2string(A_MSG_PROCESS),
				   fsa_input2string(next_input));	
	
			/* Error checking and reporting */
		} else if(cur_input != I_NULL && is_set(actions, A_NOTHING)) {
			cl_log(LOG_WARNING,
			       "No action specified for input,state (%s,%s)",
			       fsa_input2string(cur_input),