ccm.c

linux集群服务器软件代码包

/* $Id: ccm.c,v 1.58 2005/02/18 23:21:21 gshi Exp $ */
/* 
 * ccm.c: Consensus Cluster Service Program 
 *
 * Copyright (c) International Business Machines  Corp., 2002
 * Author: Ram Pai ([e-mail])
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */
#include <ccm.h>
#include <config.h>
#include <ha_config.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include <clplumbing/cl_signal.h>
#include <clplumbing/coredumps.h>


extern int global_verbose;
extern int global_debug;




#define		CCM_SET_ACTIVEPROTO(info, val) \
					info->ccm_active_proto = val
#define		CCM_SET_MAJORTRANS(info, val) 	\
		{	\
			info->ccm_transition_major = val; \
			info->ccm_max_transition = \
				(info->ccm_max_transition < val ? \
				val: info->ccm_max_transition); \
		}
#define		CCM_SET_MINORTRANS(info, val) 	\
					info->ccm_transition_minor = val
#define		CCM_INIT_MAXTRANS(info) 	\
					info->ccm_max_transition = 0

/* 	NOTE the reason the increment for majortrans is done */
/* 	as below is to force recomputation of  ccm_max_transition  */
#define		CCM_INCREMENT_MAJORTRANS(info) 	\
				CCM_SET_MAJORTRANS(info, \
					CCM_GET_MAJORTRANS(info)+1)

#define		CCM_INCREMENT_MINORTRANS(info) 	\
					info->ccm_transition_minor++
#define		CCM_RESET_MAJORTRANS(info) 	\
					info->ccm_transition_major = 0
#define		CCM_RESET_MINORTRANS(info) 	\
					info->ccm_transition_minor = 0
#define		CCM_SET_STATE(info, istate) 	\
		{  \
			if(global_debug) \
				cl_log(LOG_DEBUG,"state=%d",(istate)); \
			info->ccm_node_state = (istate); \
			if((istate)==CCM_STATE_JOINING) \
				client_influx(); \
		}


#define 	CCM_SET_JOINED_TRANSITION(info, trans) \
					info->ccm_joined_transition = trans
#define 	CCM_SET_COOKIE(info, val) \
				strncpy(info->ccm_cookie, val, COOKIESIZE)
#define 	CCM_SET_CL(info, index)	info->ccm_cluster_leader = index
#define 	CCM_SET_JOINERHEAD(info, ptr)	info->ccm_joiner_head = ptr


#define		CCM_GET_ACTIVEPROTO(info) info->ccm_active_proto
#define		CCM_GET_MAJORTRANS(info) info->ccm_transition_major
#define		CCM_GET_MINORTRANS(info) info->ccm_transition_minor
#define 	CCM_GET_MAXTRANS(info)   info->ccm_max_transition
#define		CCM_GET_STATE(info) 	info->ccm_node_state 
#define		CCM_GET_HIPROTO(info) 	info->ccm_hiproto 
#define 	CCM_GET_LLM(info) 	(&(info->ccm_llm))
#define 	CCM_GET_UPDATETABLE(info) (&(info->ccm_update))
#define 	CCM_GET_MEMCOMP(info) (&(info->ccm_memcomp))
#define 	CCM_GET_JOINED_TRANSITION(info) info->ccm_joined_transition
#define  	CCM_GET_LLM_NODECOUNT(info) LLM_GET_NODECOUNT(CCM_GET_LLM(info))
#define  	CCM_GET_MY_HOSTNAME(info)  ccm_get_my_hostname(info)
#define 	CCM_GET_COOKIE(info) info->ccm_cookie

#define 	CCM_RESET_MEMBERSHIP(info)  info->ccm_nodeCount=0
#define 	CCM_ADD_MEMBERSHIP(info, index)  \
				info->ccm_member[info->ccm_nodeCount++] = index
#define 	CCM_GET_MEMCOUNT(info)  info->ccm_nodeCount
#define 	CCM_GET_MEMINDEX(info, i)	info->ccm_member[i]
#define 	CCM_GET_MEMTABLE(info)		info->ccm_member
#define 	CCM_GET_CL(info)  		info->ccm_cluster_leader
#define 	CCM_GET_JOINERHEAD(info)	info->ccm_joiner_head
#define		CCM_TRANS_EARLIER(trans1, trans2) (trans1 < trans2) /*TOBEDONE*/
#define 	CCM_GET_VERSION(info)	&(info->ccm_version)


#define 	CCM_TMOUT_SET_U(info,t) info->tmout.u=t
#define 	CCM_TMOUT_SET_LU(info,t) info->tmout.lu=t
#define 	CCM_TMOUT_SET_VRS(info,t) info->tmout.vrs=t
#define 	CCM_TMOUT_SET_ITF(info,t) info->tmout.itf=t
#define 	CCM_TMOUT_SET_IFF(info,t) info->tmout.iff=t
#define 	CCM_TMOUT_SET_FL(info,t) info->tmout.fl=t
#define 	CCM_TMOUT_GET_U(info) info->tmout.u
#define 	CCM_TMOUT_GET_LU(info) info->tmout.lu
#define 	CCM_TMOUT_GET_VRS(info) info->tmout.vrs
#define 	CCM_TMOUT_GET_ITF(info) info->tmout.itf
#define 	CCM_TMOUT_GET_IFF(info) info->tmout.iff
#define 	CCM_TMOUT_GET_FL(info) info->tmout.fl

/* PROTOTYPE */
static void ccm_send_join_reply(ll_cluster_t *, ccm_info_t *);
static int ccm_send_final_memlist(ll_cluster_t *, ccm_info_t *, 
		char *, char *, uint32_t );
static void report_reset(void);
static int ccm_already_joined(ccm_info_t *);
static void ccm_memcomp_reset(ccm_info_t *);

/* For enhanced membership service */
static void append_change_msg(ccm_info_t *info,const char *node);
static int received_all_change_msg(ccm_info_t *info);
static int is_expected_change_msg(ccm_info_t *info, const char *node,
		enum change_event_type);
static void add_change_msg(ccm_info_t *info, const char *node, 
		const char *orig, enum change_event_type);
static int send_node_leave_to_leader(ll_cluster_t *hb, ccm_info_t *info, 
		const char *node);
static void update_membership(ccm_info_t *info, const char *node, 
		enum change_event_type change_type);
static void reset_change_info(ccm_info_t *info); 
static int ccm_send_alive_msg(ll_cluster_t *hb, ccm_info_t *info);
static int ccm_send_newnode_to_leader(ll_cluster_t *hb, ccm_info_t *info, 
		const char *node);
static void send_mem_list_to_all(ll_cluster_t *hb, ccm_info_t *info, 
		char *cookie);
static int ccm_send_to_all(ll_cluster_t *hb, ccm_info_t *info, char *memlist, 
		char *newcookie, void *uptime_list, size_t uptime_size);
static void ccm_fill_update_table(ccm_info_t *info, 
		ccm_update_t *update_table, const void *uptime_list);

static longclock_t change_time;
static void
change_time_init(void)
{
	change_time = ccm_get_time();
}
static int
change_timeout(unsigned long timeout)
{
	return(ccm_timeout(change_time, ccm_get_time(), timeout));
}

static longclock_t mem_list_time;
static void
mem_list_time_init(void)
{
	mem_list_time = ccm_get_time();
}
static int
mem_list_timeout(unsigned long timeout)
{
	return(ccm_timeout(mem_list_time, ccm_get_time(), timeout));
}

static longclock_t  new_node_mem_list_time;
static void new_node_mem_list_time_init(void)
{
    new_node_mem_list_time = ccm_get_time();
}
static int new_node_mem_list_timeout(unsigned long timeout)
{
    return(ccm_timeout(new_node_mem_list_time, ccm_get_time(), timeout));
}

#define CCM_GET_MYNODE_ID(info) \
	info->ccm_llm.llm_nodes[info->ccm_llm.llm_mynode].NodeID
#define CCM_GET_CL_NODEID(info) \
	info->ccm_llm.llm_nodes[info->ccm_member[CCM_GET_CL(info)]].NodeID 
#define CCM_GET_RECEIVED_CHANGE_MSG(info, node) \
	CCM_GET_LLM(info)->llm_nodes[info->ccm_member[ccm_get_membership_index(info, node)]].received_change_msg
#define CCM_SET_RECEIVED_CHANGE_MSG(info, node, value) \
	CCM_GET_LLM(info)->llm_nodes[info->ccm_member[ccm_get_membership_index(info, node)]].received_change_msg = value

/*
////////////////////////////////////////////////////////////////
// BEGIN OF Functions associated with CCM token types that are
// communicated accross nodes and their values.
////////////////////////////////////////////////////////////////
*/

/* the ccm types tokens used locally, these are the integer equivalents */
/* for the F_TYPE tokens. The strings defined in ccm_type_str are */
/* communicated accross the wire. But locally they are mapped to */
/* ccm_types for easier processing. */
enum ccm_type {
	CCM_TYPE_PROTOVERSION=1,
	CCM_TYPE_PROTOVERSION_RESP,
	CCM_TYPE_JOIN,
	CCM_TYPE_REQ_MEMLIST,
	CCM_TYPE_RES_MEMLIST,
	CCM_TYPE_FINAL_MEMLIST,
	CCM_TYPE_ABORT,
	CCM_TYPE_LEAVE,
	CCM_TYPE_TIMEOUT,
	CCM_TYPE_ERROR,
	CCM_TYPE_NODE_LEAVE,
	CCM_TYPE_MEM_LIST,
	CCM_TYPE_ALIVE,
	CCM_TYPE_NEW_NODE,
	CCM_TYPE_LAST = 16
};

static void ccm_state_wait_for_mem_list(enum ccm_type ccm_msg_type, 
			struct ha_msg *reply, 
			ll_cluster_t *hb, 
			ccm_info_t *info);
static void ccm_state_new_node_wait_for_mem_list(enum ccm_type ccm_msg_type, 
	              struct ha_msg *reply, 
	              ll_cluster_t *hb, 
			ccm_info_t *info);


/* the ccm strings tokens communicated aross the wire. */
/* these are the values for the F_TYPE names. */
#define TYPESTRSIZE 20
char  ccm_type_str[CCM_TYPE_LAST][TYPESTRSIZE] = {
			"",
			"ccmpver",
			"ccmpverresp",
			"ccmjoin",
			"ccmreqmlst",
			"ccmresmlst",
			"ccmfnlmlst",
			"ccmabrt",
			"ccmlv",
			"ccmtmout",
			"ccmerror",
			"ccmnodeleave",
			"ccmmemlst",
			"ccmalive",
			"ccmnewnode",
			""
	};

/* */
/* ccm defined new type tokens used by the CCM protocol. */
/* */
#define CCM_VERSIONVAL  "ccmpverval" 	  /* version value token */
#define CCM_UPTIME      "ccmuptime"       /* Uptime for Consensus  */
#define CCM_MEMLIST     "ccmmemlist"      /* bitmap for membership */
#define CCM_PROTOCOL    "ccmproto"        /* protocol version */
#define CCM_MAJORTRANS  "ccmmajor"        /* major transition version*/
#define CCM_MINORTRANS  "ccmminor"        /* minor transition version */
#define CCM_MAXTRANS    "ccmmaxt"        /* minor transition version */
#define CCM_COOKIE      "ccmcookie"       /* communication context */
#define CCM_NEWCOOKIE   "ccmnewcookie"    /* new communication context */
#define CCM_CLSIZE   	"ccmclsize"       /* new cluster size */
#define CCM_UPTIMELIST "ccmuptimelist" /*uptime list*/

/* given a ccm_type return the string representation associated with it. */
/* NOTE: string representation is used to communicate accross node. */
/*       and ccm_type is used for easier local processing. */
static char *
ccm_type2string(enum ccm_type type)
{
	return ccm_type_str[type];
}

/* */
/* given a string representation return the string type. */
/* */
static enum ccm_type 
ccm_string2type(const char *type)
{
	enum ccm_type i;

	for ( i = CCM_TYPE_PROTOVERSION; i <= CCM_TYPE_LAST; i++ ) {
		if (strncmp(ccm_type_str[i], type, TYPESTRSIZE) == 0){
			return i;
		}
	}
	return CCM_TYPE_ERROR;
}

/* END OF TYPE_STR datastructure and associated functions */


/* */
/* timeout configuration function */
/* */
static void
ccm_configure_timeout(ll_cluster_t *hb, ccm_info_t *info)
{
	long keepalive = hb->llc_ops->get_keepalive(hb);

	if(global_debug) {
		cl_log(LOG_INFO, "ccm_configure_timeout  "
			"keepalive=%ld", keepalive);
	}

	CCM_TMOUT_SET_U(info, 9*keepalive);
	CCM_TMOUT_SET_LU(info, 30*keepalive);
	CCM_TMOUT_SET_VRS(info, 9*keepalive);
	CCM_TMOUT_SET_ITF(info, 18*keepalive);
	CCM_TMOUT_SET_IFF(info, 12*keepalive);
	CCM_TMOUT_SET_FL(info, CCM_TMOUT_GET_ITF(info)+5);
}


/* */
/* ccm_get_my_hostname: return my nodename. */
/* */
static char *
ccm_get_my_hostname(ccm_info_t *info)
{
	llm_info_t *llm = CCM_GET_LLM(info);
	return(LLM_GET_MYNODEID(llm));
}


/* */
/* timeout_msg_create:  */
/*	fake up a timeout message, which is in the */
/* 	same format as the other messages that are */
/*	communicated across the nodes. */
/* */

static struct ha_msg * timeout_msg = NULL;



static int
timeout_msg_init(ccm_info_t *info)
{
	struct ha_msg *m;
	char majortrans[15]; /* 10 is the maximum number of digits in 
				UINT_MAX , adding a buffer of 5 */
	char minortrans[15]; /*		ditto 	*/
	char *hname;

	if ((m=ha_msg_new(0)) == NULL) {
		cl_log(LOG_ERR, "Cannot send CCM version msg");
		return(HA_FAIL);
	}

	hname = ccm_get_my_hostname(info);

	snprintf(majortrans, sizeof(majortrans), "%d", 0);
	snprintf(minortrans, sizeof(majortrans), "%d", 0);
	if((ha_msg_add(m, F_TYPE, ccm_type2string(CCM_TYPE_TIMEOUT)) == HA_FAIL)
		||(ha_msg_add(m, F_ORIG, hname) == HA_FAIL) 
		||(ha_msg_add(m, CCM_COOKIE, "  ") == HA_FAIL) 
		||(ha_msg_add(m, CCM_COOKIE, "  ") == HA_FAIL) 
		||(ha_msg_add(m, CCM_MAJORTRANS, majortrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_MINORTRANS, minortrans) == HA_FAIL)){
			cl_log(LOG_ERR, "timeout_msg_create: Cannot "
				"create timeout message");
		return HA_FAIL;
	}
	timeout_msg = m;
	return 0;
}

static struct ha_msg  *
timeout_msg_mod(ccm_info_t *info)
{
	char majortrans[15]; /* 10 is the maximum number of digits in 
				UINT_MAX , adding a buffer of 5 */
	char minortrans[15]; /*		ditto 	*/

	char *cookie = CCM_GET_COOKIE(info);

	int major  = CCM_GET_MAJORTRANS(info);
	int minor  = CCM_GET_MINORTRANS(info);

	struct ha_msg *m = timeout_msg;
	assert(m);
	snprintf(majortrans, sizeof(majortrans), "%d", major);
	snprintf(minortrans, sizeof(minortrans), "%d", minor);
	if((ha_msg_mod(m, CCM_COOKIE, cookie) == HA_FAIL)
		||(ha_msg_mod(m, CCM_MAJORTRANS, majortrans) == HA_FAIL)
		||(ha_msg_mod(m, CCM_MINORTRANS, minortrans) == HA_FAIL)){
			cl_log(LOG_ERR, "timeout_msg_mod: Cannot "
				"modify timeout message");
		return NULL;
	}
	return m;
}


#ifdef TIMEOUT_MSG_FUNCTIONS_NEEDED
/* */
/* timeout_msg_done:  */
/*   done with the processing of this message. */
static void
timeout_msg_done(void)
{
	/* nothing to do. */
	return;
}


/* */
/* timeout_msg_del:  */
/*   delete the given timeout message. */
/*   nobody calls this function.  */
/*   someday somebody will call it :) */
static void
timeout_msg_del(void)
{
	ha_msg_del(timeout_msg);
	timeout_msg = NULL;
}
#endif


/* */
/* These are the function that keep track of number of time a version */
/* response message has been dropped. These function are consulted by */
/* the CCM algorithm to determine if a version response message has */
/* to be dropped or not. */
/* */
static int respdrop=0;
#define MAXDROP 3

static int
resp_can_i_drop(void)
{
	if (respdrop >= MAXDROP){
		return FALSE;
	}
	return TRUE;
}

static void
resp_dropped(void)
{
	respdrop++;
}

static void
resp_reset(void)
{
	respdrop=0;
}
/* */
/* End of response processing messages. */
/* */


/* */
/* BEGIN OF functions that track the time since a connectivity reply has */
/* been sent to the leader. */
/* */
static longclock_t finallist_time;

static void
finallist_init(void)
{
	finallist_time = ccm_get_time();
}

static void
finallist_reset(void)
{
	finallist_time = 0;
}

static int
finallist_timeout(unsigned long timeout)
{
	return(ccm_timeout(finallist_time, ccm_get_time(), timeout));
}
/* */
/* END OF functions that track the time since a connectivity reply has */
/* been sent to the leader. */
/* */


/* BEGINE of the functions that track asynchronous leave */
/* */
/* When ccm running on a  node leaves the cluster voluntarily it  */