ccm.c

在LINUX下实现HA的源代码

/* $Id: ccm.c,v 1.36.2.8 2005/01/12 02:25:47 horms Exp $ */
/* 
 * ccm.c: Consensus Cluster Service Program 
 *
 * Copyright (c) International Business Machines  Corp., 2002
 * Author: Ram Pai ([e-mail])
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 */
#include <ccm.h>
#include <config.h>
#include <ha_config.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include <clplumbing/cl_signal.h>
#include <clplumbing/coredumps.h>


extern int global_verbose;
extern int global_debug;

/* */
/* the various states of the CCM state machine. */
/* */
enum ccm_state  {
	CCM_STATE_NONE=0,		/* is in NULL state  */
	CCM_STATE_VERSION_REQUEST=10,	/* sent a request for protocol version */
	CCM_STATE_JOINING=20,  		/* has initiated a join protocol  */
	CCM_STATE_RCVD_UPDATE=30,	/* has recevied the updates from other nodes */
	CCM_STATE_SENT_MEMLISTREQ=40,	/* CL has sent a request for member list  */
					/* this state is applicable only on CL */
	CCM_STATE_REQ_MEMLIST=50,	/* CL has requested member list */
				  	/* this state is applicable only on non-CL */
	CCM_STATE_MEMLIST_RES=60,	/* Responded member list to the Cluster  */
				 	/* Leader */
	CCM_STATE_JOINED=70,    /* PART of the CCM cluster membership! */
	CCM_STATE_WAIT_FOR_MEM_LIST=80,
	CCM_STATE_WAIT_FOR_CHANGE=90,
	CCM_STATE_NEW_NODE_WAIT_FOR_MEM_LIST=100,

	CCM_STATE_END
};

/* the times for repeating sending message */
#define REPEAT_TIMES 10

/* add new enums to this structure as and when new protocols are added */
enum ccm_protocol {
	CCM_VER_NONE = 0,
	CCM_VER_1,
	CCM_VER_LAST
};

typedef struct ccm_proto_s {
	enum ccm_protocol  com_hiproto;/* highest protocol version that  */
				/* this node can handle */
	int	com_active_proto;/* protocol version */
} ccm_proto_t;


typedef struct memcomp_s {
	graph_t		*mem_graph;  /* memlist calculation graph */

	GSList 		*mem_maxt; 	    /* the maxtrans of each node */
				    /* participating in the computation . */
				    /* NOTE: the transition number of the */
				    /* next transition is always 1 higher */
				    /* than that of all transitions seen  */
				    /* by each node participating in the  */
				    /* membership */
	longclock_t  	mem_inittime; /* the time got intialized */
} memcomp_t;
#define 	MEMCOMP_GET_GRAPH(memc)  	memc->mem_graph
#define 	MEMCOMP_GET_MAXT(memc)  	memc->mem_maxt
#define 	MEMCOMP_GET_INITTIME(memc)  	memc->mem_inittime
#define 	MEMCOMP_SET_GRAPH(memc, gr)  	memc->mem_graph=gr
#define 	MEMCOMP_SET_MAXT(memc, list)  	memc->mem_maxt=list
#define 	MEMCOMP_SET_INITTIME(memc,time)	memc->mem_inittime=time


typedef struct ccm_tmout_s {
	long	iff;  /* membership_Info_From_Followers_timeout */
	long	itf;  /* membership_Info_To_Followers_timeout */
	long	fl;  /* membership_Final_List_timeout */
	long	u;  /* update timeout */
	long	lu;  /* long update timeout */
	long	vrs;  /* version timeout */
} ccm_tmout_t;

enum change_event_type{
    TYPE_NONE,	
    NODE_LEAVE,
    NEW_NODE
};

#define COOKIESIZE 15
typedef struct ccm_info_s {
	llm_info_t 	ccm_llm;	/*  low level membership info */

	int		ccm_nodeCount;	/*  number of nodes in the ccm cluster */
	int		ccm_member[MAXNODE];/* members of the ccm cluster */
	memcomp_t	ccm_memcomp;	/* the datastructure to compute the  */
					/* final membership for each membership */
	 				/* computation instance of the ccm protocol. */
	 				/* used by the leader only. */

	ccm_proto_t  	ccm_proto;	/* protocol version information */
#define ccm_active_proto ccm_proto.com_active_proto
#define ccm_hiproto	  ccm_proto.com_hiproto

	char		ccm_cookie[COOKIESIZE];/* context identification string. */
	uint32_t	ccm_transition_major;/* transition number of the cluster */
	int		ccm_cluster_leader; /* cluster leader of the last major */
				/* transition. index of cl in ccm_member table */
	int		ccm_joined_transition;
					/* this indicates the major transition  */
					/* number during which this node became */
					/* a member of the cluster. */
					/* A sideeffect of this is it also */
					/* is used to figure out if this node */
					/* was ever a part of the cluster. */
					/* Should be intially set to 0 */
	uint32_t	ccm_max_transition;	/* the maximum transition number seen */
					/* by this node ever since it was born. */
	enum ccm_state 	ccm_node_state;	/* cluster state of this node  */
	uint32_t	ccm_transition_minor;/* minor transition number of the  */
					/* cluster */

	ccm_update_t   ccm_update; 	/* structure that keeps track */
					/* of uptime of each member */
	GSList		*ccm_joiner_head;/* keeps track of new-bees version */
					/* request.  */
	ccm_version_t  ccm_version;     /* keeps track of version request  */
					/* related info */
	ccm_tmout_t	tmout;
	uint32_t change_event_remaining_count; 		
	enum change_event_type change_type;
	char change_node_id[NODEIDSIZE];

} ccm_info_t;




#define		CCM_SET_ACTIVEPROTO(info, val) \
					info->ccm_active_proto = val
#define		CCM_SET_MAJORTRANS(info, val) 	\
		{	\
			info->ccm_transition_major = val; \
			info->ccm_max_transition = \
				(info->ccm_max_transition < val ? \
				val: info->ccm_max_transition); \
		}
#define		CCM_SET_MINORTRANS(info, val) 	\
					info->ccm_transition_minor = val
#define		CCM_INIT_MAXTRANS(info) 	\
					info->ccm_max_transition = 0

/* 	NOTE the reason the increment for majortrans is done */
/* 	as below is to force recomputation of  ccm_max_transition  */
#define		CCM_INCREMENT_MAJORTRANS(info) 	\
				CCM_SET_MAJORTRANS(info, \
					CCM_GET_MAJORTRANS(info)+1)

#define		CCM_INCREMENT_MINORTRANS(info) 	\
					info->ccm_transition_minor++
#define		CCM_RESET_MAJORTRANS(info) 	\
					info->ccm_transition_major = 0
#define		CCM_RESET_MINORTRANS(info) 	\
					info->ccm_transition_minor = 0
#define		CCM_SET_STATE(info, istate) 	\
		{  \
			if(global_debug) \
				cl_log(LOG_DEBUG,"state=%d",(istate)); \
			info->ccm_node_state = (istate); \
			if((istate)==CCM_STATE_JOINING) \
				client_influx(); \
		}


#define 	CCM_SET_JOINED_TRANSITION(info, trans) \
					info->ccm_joined_transition = trans
#define 	CCM_SET_COOKIE(info, val) \
				strncpy(info->ccm_cookie, val, COOKIESIZE)
#define 	CCM_SET_CL(info, index)	info->ccm_cluster_leader = index
#define 	CCM_SET_JOINERHEAD(info, ptr)	info->ccm_joiner_head = ptr


#define		CCM_GET_ACTIVEPROTO(info) info->ccm_active_proto
#define		CCM_GET_MAJORTRANS(info) info->ccm_transition_major
#define		CCM_GET_MINORTRANS(info) info->ccm_transition_minor
#define 	CCM_GET_MAXTRANS(info)   info->ccm_max_transition
#define		CCM_GET_STATE(info) 	info->ccm_node_state 
#define		CCM_GET_HIPROTO(info) 	info->ccm_hiproto 
#define 	CCM_GET_LLM(info) 	(&(info->ccm_llm))
#define 	CCM_GET_UPDATETABLE(info) (&(info->ccm_update))
#define 	CCM_GET_MEMCOMP(info) (&(info->ccm_memcomp))
#define 	CCM_GET_JOINED_TRANSITION(info) info->ccm_joined_transition
#define  	CCM_GET_LLM_NODECOUNT(info) LLM_GET_NODECOUNT(CCM_GET_LLM(info))
#define  	CCM_GET_MY_HOSTNAME(info)  ccm_get_my_hostname(info)
#define 	CCM_GET_COOKIE(info) info->ccm_cookie

#define 	CCM_RESET_MEMBERSHIP(info)  info->ccm_nodeCount=0
#define 	CCM_ADD_MEMBERSHIP(info, index)  \
				info->ccm_member[info->ccm_nodeCount++] = index
#define 	CCM_GET_MEMCOUNT(info)  info->ccm_nodeCount
#define 	CCM_GET_MEMINDEX(info, i)	info->ccm_member[i]
#define 	CCM_GET_MEMTABLE(info)		info->ccm_member
#define 	CCM_GET_CL(info)  		info->ccm_cluster_leader
#define 	CCM_GET_JOINERHEAD(info)	info->ccm_joiner_head
#define		CCM_TRANS_EARLIER(trans1, trans2) (trans1 < trans2) /*TOBEDONE*/
#define 	CCM_GET_VERSION(info)	&(info->ccm_version)


#define 	CCM_TMOUT_SET_U(info,t) info->tmout.u=t
#define 	CCM_TMOUT_SET_LU(info,t) info->tmout.lu=t
#define 	CCM_TMOUT_SET_VRS(info,t) info->tmout.vrs=t
#define 	CCM_TMOUT_SET_ITF(info,t) info->tmout.itf=t
#define 	CCM_TMOUT_SET_IFF(info,t) info->tmout.iff=t
#define 	CCM_TMOUT_SET_FL(info,t) info->tmout.fl=t
#define 	CCM_TMOUT_GET_U(info) info->tmout.u
#define 	CCM_TMOUT_GET_LU(info) info->tmout.lu
#define 	CCM_TMOUT_GET_VRS(info) info->tmout.vrs
#define 	CCM_TMOUT_GET_ITF(info) info->tmout.itf
#define 	CCM_TMOUT_GET_IFF(info) info->tmout.iff
#define 	CCM_TMOUT_GET_FL(info) info->tmout.fl

/* PROTOTYPE */
static void ccm_send_join_reply(ll_cluster_t *, ccm_info_t *);
static int ccm_send_final_memlist(ll_cluster_t *, ccm_info_t *, 
		char *, char *, uint32_t );
static void report_reset(void);
static int ccm_already_joined(ccm_info_t *);
static void ccm_memcomp_reset(ccm_info_t *);

/* For enhanced membership service */
static void append_change_msg(ccm_info_t *info,const char *node);
static int received_all_change_msg(ccm_info_t *info);
static int is_expected_change_msg(ccm_info_t *info, const char *node,
		enum change_event_type);
static void add_change_msg(ccm_info_t *info, const char *node, 
		const char *orig, enum change_event_type);
static int send_node_leave_to_leader(ll_cluster_t *hb, ccm_info_t *info, 
		const char *node);
static void update_membership(ccm_info_t *info, const char *node, 
		enum change_event_type change_type);
static void reset_change_info(ccm_info_t *info); 
static int ccm_send_alive_msg(ll_cluster_t *hb, ccm_info_t *info);
static int ccm_send_newnode_to_leader(ll_cluster_t *hb, ccm_info_t *info, 
		const char *node);
static void send_mem_list_to_all(ll_cluster_t *hb, ccm_info_t *info, 
		char *cookie);
static int ccm_send_to_all(ll_cluster_t *hb, ccm_info_t *info, char *memlist, 
		char *newcookie, void *uptime_list, size_t uptime_size);
static void ccm_fill_update_table(ccm_info_t *info, 
		ccm_update_t *update_table, const void *uptime_list);

static longclock_t change_time;
static void
change_time_init(void)
{
	change_time = ccm_get_time();
}
static int
change_timeout(unsigned long timeout)
{
	return(ccm_timeout(change_time, ccm_get_time(), timeout));
}

static longclock_t mem_list_time;
static void
mem_list_time_init(void)
{
	mem_list_time = ccm_get_time();
}
static int
mem_list_timeout(unsigned long timeout)
{
	return(ccm_timeout(mem_list_time, ccm_get_time(), timeout));
}

static longclock_t  new_node_mem_list_time;
static void new_node_mem_list_time_init(void)
{
    new_node_mem_list_time = ccm_get_time();
}
static int new_node_mem_list_timeout(unsigned long timeout)
{
    return(ccm_timeout(new_node_mem_list_time, ccm_get_time(), timeout));
}

#define CCM_GET_MYNODE_ID(info) \
	info->ccm_llm.llm_nodes[info->ccm_llm.llm_mynode].NodeID
#define CCM_GET_CL_NODEID(info) \
	info->ccm_llm.llm_nodes[info->ccm_member[CCM_GET_CL(info)]].NodeID 
#define CCM_GET_RECEIVED_CHANGE_MSG(info, node) \
	CCM_GET_LLM(info)->llm_nodes[info->ccm_member[ccm_get_membership_index(info, node)]].received_change_msg
#define CCM_SET_RECEIVED_CHANGE_MSG(info, node, value) \
	CCM_GET_LLM(info)->llm_nodes[info->ccm_member[ccm_get_membership_index(info, node)]].received_change_msg = value

/*
////////////////////////////////////////////////////////////////
// BEGIN OF Functions associated with CCM token types that are
// communicated accross nodes and their values.
////////////////////////////////////////////////////////////////
*/

/* the ccm types tokens used locally, these are the integer equivalents */
/* for the F_TYPE tokens. The strings defined in ccm_type_str are */
/* communicated accross the wire. But locally they are mapped to */
/* ccm_types for easier processing. */
enum ccm_type {
	CCM_TYPE_PROTOVERSION=1,
	CCM_TYPE_PROTOVERSION_RESP,
	CCM_TYPE_JOIN,
	CCM_TYPE_REQ_MEMLIST,
	CCM_TYPE_RES_MEMLIST,
	CCM_TYPE_FINAL_MEMLIST,
	CCM_TYPE_ABORT,
	CCM_TYPE_LEAVE,
	CCM_TYPE_TIMEOUT,
	CCM_TYPE_ERROR,
	CCM_TYPE_NODE_LEAVE,
	CCM_TYPE_MEM_LIST,
	CCM_TYPE_ALIVE,
	CCM_TYPE_NEW_NODE,
	CCM_TYPE_LAST = 16
};

static void ccm_state_wait_for_mem_list(enum ccm_type ccm_msg_type, 
			struct ha_msg *reply, 
			ll_cluster_t *hb, 
			ccm_info_t *info);
static void ccm_state_new_node_wait_for_mem_list(enum ccm_type ccm_msg_type, 
	              struct ha_msg *reply, 
	              ll_cluster_t *hb, 
			ccm_info_t *info);


/* the ccm strings tokens communicated aross the wire. */
/* these are the values for the F_TYPE names. */
#define TYPESTRSIZE 20
char  ccm_type_str[CCM_TYPE_LAST][TYPESTRSIZE] = {
			"",
			"ccmpver",
			"ccmpverresp",
			"ccmjoin",
			"ccmreqmlst",
			"ccmresmlst",
			"ccmfnlmlst",
			"ccmabrt",
			"ccmlv",
			"ccmtmout",
			"ccmerror",
			"ccmnodeleave",
			"ccmmemlst",
			"ccmalive",
			"ccmnewnode",
			""
	};

/* */
/* ccm defined new type tokens used by the CCM protocol. */
/* */
#define CCM_VERSIONVAL  "ccmpverval" 	  /* version value token */
#define CCM_UPTIME      "ccmuptime"       /* Uptime for Consensus  */
#define CCM_MEMLIST     "ccmmemlist"      /* bitmap for membership */
#define CCM_PROTOCOL    "ccmproto"        /* protocol version */
#define CCM_MAJORTRANS  "ccmmajor"        /* major transition version*/
#define CCM_MINORTRANS  "ccmminor"        /* minor transition version */
#define CCM_MAXTRANS    "ccmmaxt"        /* minor transition version */
#define CCM_COOKIE      "ccmcookie"       /* communication context */
#define CCM_NEWCOOKIE   "ccmnewcookie"    /* new communication context */
#define CCM_CLSIZE   	"ccmclsize"       /* new cluster size */
#define CCM_UPTIMELIST "ccmuptimelist" /*uptime list*/

/* given a ccm_type return the string representation associated with it. */
/* NOTE: string representation is used to communicate accross node. */
/*       and ccm_type is used for easier local processing. */
static char *
ccm_type2string(enum ccm_type type)
{
	return ccm_type_str[type];
}

/* */
/* given a string representation return the string type. */
/* */
static enum ccm_type 
ccm_string2type(const char *type)
{
	enum ccm_type i;

	for ( i = CCM_TYPE_PROTOVERSION; i <= CCM_TYPE_LAST; i++ ) {
		if (strncmp(ccm_type_str[i], type, TYPESTRSIZE) == 0)
			return i;
	}
	return CCM_TYPE_ERROR;
}

/* END OF TYPE_STR datastructure and associated functions */


/* */
/* timeout configuration function */
/* */
static void
ccm_configure_timeout(ll_cluster_t *hb, ccm_info_t *info)
{
	long keepalive = hb->llc_ops->get_keepalive(hb);

	if(global_debug) {
		cl_log(LOG_INFO, "ccm_configure_timeout  "
			"keepalive=%ld", keepalive);
	}

	CCM_TMOUT_SET_U(info, 9*keepalive);
	CCM_TMOUT_SET_LU(info, 30*keepalive);
	CCM_TMOUT_SET_VRS(info, 9*keepalive);
	CCM_TMOUT_SET_ITF(info, 18*keepalive);
	CCM_TMOUT_SET_IFF(info, 12*keepalive);
	CCM_TMOUT_SET_FL(info, CCM_TMOUT_GET_ITF(info)+5);
}


/* */
/* ccm_get_my_hostname: return my nodename. */
/* */
static char *
ccm_get_my_hostname(ccm_info_t *info)
{
	llm_info_t *llm = CCM_GET_LLM(info);
	return(LLM_GET_MYNODEID(llm));
}


/* */
/* timeout_msg_create:  */
/*	fake up a timeout message, which is in the */
/* 	same format as the other messages that are */
/*	communicated across the nodes. */
/* */

static struct ha_msg * timeout_msg = NULL;



static int
timeout_msg_init(ccm_info_t *info)
{
	struct ha_msg *m;
	char majortrans[15]; /* 10 is the maximum number of digits in 
				UINT_MAX , adding a buffer of 5 */
	char minortrans[15]; /*		ditto 	*/
	char *hname;

	if ((m=ha_msg_new(0)) == NULL) {
		cl_log(LOG_ERR, "Cannot send CCM version msg");
		return(HA_FAIL);
	}

	hname = ccm_get_my_hostname(info);

	snprintf(majortrans, sizeof(majortrans), "%d", 0);
	snprintf(minortrans, sizeof(majortrans), "%d", 0);
	if((ha_msg_add(m, F_TYPE, ccm_type2string(CCM_TYPE_TIMEOUT)) == HA_FAIL)
		||(ha_msg_add(m, F_ORIG, hname) == HA_FAIL) 
		||(ha_msg_add(m, CCM_COOKIE, "  ") == HA_FAIL) 
		||(ha_msg_add(m, CCM_COOKIE, "  ") == HA_FAIL)