ccm.c

linux集群服务器软件代码包

ccm_send_join(ll_cluster_t *hb, ccm_info_t *info)
{
	struct ha_msg *m;
	char majortrans[15]; /* 10 is the maximum number of digits in 
				UINT_MAX , adding a buffer of 5 */
	char minortrans[15]; /*		ditto 	*/
	char joinedtrans[15]; /*		ditto 	*/
	int  joinedtrans_val;
	char *cookie;
	int  rc;

	if ((m=ha_msg_new(0)) == NULL) {
		cl_log(LOG_ERR, "Cannot send CCM version msg");
		return(HA_FAIL);
	}
	
	snprintf(majortrans, sizeof(majortrans), "%d", 
				CCM_GET_MAJORTRANS(info));
	snprintf(minortrans, sizeof(minortrans), "%d", 
				CCM_GET_MINORTRANS(info));
	/* uptime is based on the transition during which a given node
	 * officially joined the cluster 
	 */
	cookie = CCM_GET_COOKIE(info);
	assert(cookie && *cookie);

	joinedtrans_val = CCM_GET_JOINED_TRANSITION(info);
	joinedtrans_val = (joinedtrans_val == -1)? 0: joinedtrans_val;
	snprintf(joinedtrans, sizeof(joinedtrans_val), "%d", joinedtrans_val);

	if ((ha_msg_add(m, F_TYPE, ccm_type2string(CCM_TYPE_JOIN)) == HA_FAIL)
		||(ha_msg_add(m, CCM_COOKIE, cookie) == HA_FAIL)
		||(ha_msg_add(m, CCM_MAJORTRANS, majortrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_MINORTRANS, minortrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_UPTIME, joinedtrans) == HA_FAIL)) {
			cl_log(LOG_ERR, "ccm_send_join: Cannot create JOIN "
						    "message");
		rc = HA_FAIL;
	} else {
		/*delay by microseconds to avoid message collision */
		ccm_delay_random_interval();
		rc = hb->llc_ops->sendclustermsg(hb, m);
	}
	ha_msg_del(m);
	return(rc);
}


/* */
/* send out the connectivity information to the cluster leader. */
/* */
static int
ccm_send_memlist_res(ll_cluster_t *hb, 
			ccm_info_t *info,
			const char *nodename, 
			char *memlist)
{
	struct ha_msg *m;
	char majortrans[15]; /* 10 is the maximum number of digits in 
				UINT_MAX , adding a buffer of 5 */
	char minortrans[15]; /*		ditto 	*/
	char maxtrans[15]; /*		ditto 	*/
	char *cookie;
	int  rc;
	unsigned char *bitmap;
	gboolean del_flag=FALSE;
	
	if ((m=ha_msg_new(0)) == NULL) {
		cl_log(LOG_ERR, "ccm_send_memlist_res: Cannot allocate "
				"memory ");
		return(HA_FAIL);
	}
	
	snprintf(majortrans, sizeof(majortrans), "%d", 
					CCM_GET_MAJORTRANS(info));
	snprintf(minortrans, sizeof(minortrans), "%d", 
					CCM_GET_MINORTRANS(info));
	snprintf(maxtrans, sizeof(maxtrans), "%d", 
					CCM_GET_MAXTRANS(info));

	cookie = CCM_GET_COOKIE(info);
	assert(cookie && *cookie);

	if (!memlist) {
		int numBytes = bitmap_create(&bitmap, MAXNODE);
		(void) ccm_bitmap2str(bitmap, numBytes, &memlist);
		bitmap_delete(bitmap);
		del_flag = TRUE;
	} 

	if ((ha_msg_add(m, F_TYPE, ccm_type2string(CCM_TYPE_RES_MEMLIST)) 
							== HA_FAIL)
		||(ha_msg_add(m, CCM_COOKIE, cookie) == HA_FAIL)
		||(ha_msg_add(m, CCM_MAJORTRANS, majortrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_MINORTRANS, minortrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_MAXTRANS, maxtrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_MEMLIST, memlist) == HA_FAIL)) {
		cl_log(LOG_ERR, "ccm_send_memlist_res: Cannot create "
						"RES_MEMLIST message");
		rc = HA_FAIL;
	} else {
		/*delay by microseconds to avoid message collision */
		ccm_delay_random_interval();
		rc = hb->llc_ops->sendnodemsg(hb, m, nodename);
	}

	if(del_flag) {
		g_free(memlist);
	}
	
	ha_msg_del(m);
	return(rc);
}

/* */
/* send out a message to all the members of the cluster, asking for */
/* their connectivity information. */
/* */
/* NOTE: called by the cluster leader only. */
/* */
static int
ccm_send_memlist_request(ll_cluster_t *hb, ccm_info_t *info)
{
	struct ha_msg *m;
	char majortrans[15]; /* 10 is the maximum number of digits in 
					UINT_MAX , adding a buffer of 5 */
	char minortrans[15]; /*		ditto 	*/
	char *cookie;
	int  rc;

	if ((m=ha_msg_new(0)) == NULL) {
		cl_log(LOG_ERR, "ccm_send_memlist_request: Cannot allocate "
				"memory");
		return(HA_FAIL);
	}
	
	snprintf(majortrans, sizeof(majortrans), "%d", 
					CCM_GET_MAJORTRANS(info));
	snprintf(minortrans, sizeof(minortrans), "%d", 
					CCM_GET_MINORTRANS(info));
	cookie = CCM_GET_COOKIE(info);
	assert(cookie && *cookie);

	if ((ha_msg_add(m, F_TYPE, ccm_type2string(CCM_TYPE_REQ_MEMLIST)) 
						== HA_FAIL)
		||(ha_msg_add(m, CCM_COOKIE, cookie) == HA_FAIL)
		||(ha_msg_add(m, CCM_MAJORTRANS, majortrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_MINORTRANS, minortrans) == HA_FAIL)) {
			cl_log(LOG_ERR, "ccm_send_memlist_request: Cannot "
				"create REQ_MEMLIST message");
		rc = HA_FAIL;
	} else {
		rc = hb->llc_ops->sendclustermsg(hb, m);
	}
	ha_msg_del(m);
	return(rc);
}




/* */
/* Browse through the list of all the connectivity request messages */
/* from cluster leaders. Send out the connectivity information only */
/* to the node which we believe is the cluster leader. To everybody  */
/* else send out a null message. */
/* */
static int
ccm_send_cl_reply(ll_cluster_t *hb, ccm_info_t *info)
{
	int ret=FALSE, bitmap_strlen;
	char *memlist, *cl, *cl_tmp;
	void *cltrack;
	uint  trans;
	int repeat;
	/*
        * Get the name of the cluster leader
	*/
	cl = update_get_cl_name(CCM_GET_UPDATETABLE(info), 
				CCM_GET_LLM(info));

	/* search through the update list and find if any Cluster
	 * leader has sent a memlist request. For each, check if
	 * that node is the one which we believe is the leader.
	 * if it is the leader, send it our membership list.
	 * if not send it an NULL membership reply.
	 */
	cltrack = update_initlink(CCM_GET_UPDATETABLE(info));
	while((cl_tmp = update_next_link(CCM_GET_UPDATETABLE(info), 
				CCM_GET_LLM(info), cltrack, &trans)) != NULL) {
		if(strncmp(cl, cl_tmp, 
			LLM_GET_NODEIDSIZE(CCM_GET_LLM(info))) == 0) {

			if(ccm_already_joined(info) && 
				CCM_GET_MAJORTRANS(info) != trans){
				cl_log(LOG_INFO, "evicted");
				ccm_reset(info);
				return FALSE;
			}
			ret = TRUE;
			bitmap_strlen = update_strcreate(
				CCM_GET_UPDATETABLE(info), 
				&memlist, CCM_GET_LLM(info));

			/* send Cluster Leader our memlist only if we are 
			 * operating in the same transition as that of 
			 * the leader, provided we have been a cluster member 
			 * in the past 
			 */
			repeat = 0;
			while (ccm_send_memlist_res(hb, info, cl, memlist)
						!=HA_OK) {
				if(repeat < REPEAT_TIMES){
					cl_log(LOG_ERR,
						"ccm_state_version_request: "
						"failure to send join");
					cl_shortsleep();
					repeat++;
				}else{
					break;
				}
			}
			update_strdelete(memlist);
		} else {
			/* I dont trust this Cluster Leader.
			Send NULL memlist message */
			repeat = 0;
			while (ccm_send_memlist_res(hb, info, cl_tmp, NULL)
					!= HA_OK) {
				if(repeat < REPEAT_TIMES){
					cl_log(LOG_ERR, 
					"ccm_state_version_request: failure "
						"to send join");
					cl_shortsleep();
					repeat++;
				}else{
					break;
				}
			}
		}
	}
	update_freelink(CCM_GET_UPDATETABLE(info), cltrack);
	update_free_memlist_request(CCM_GET_UPDATETABLE(info)); 
	return ret;
}
/*
/////////////////////////////////////////////////////////////////////
//
// END OF FUNCTIONS THAT SEND OUT messages to nodes of the cluster
//
/////////////////////////////////////////////////////////////////////
*/


/* */
/* Fake up a leave message. */
/* This is generally done when heartbeat informs ccm of the crash of */
/* a cluster member. */
/* */
static struct ha_msg *
ccm_create_leave_msg(ccm_info_t *info, int uuid)
{
	struct ha_msg *m;
	char majortrans[15]; /* 10 is the maximum number of digits in 
				UINT_MAX , adding a buffer of 5 */
	char minortrans[15]; /*		ditto 	*/
	llm_info_t *llm;
	char *nodename, *cookie;
	


	/* find the name of the node at index */
	llm = CCM_GET_LLM(info);
	nodename = llm_get_nodeid_from_uuid(llm, uuid);

    	if ((m=ha_msg_new(0)) == NULL) {
		cl_log(LOG_ERR, "ccm_send_memlist_request: "
				"Cannot allocate memory");
		return(HA_FAIL);
	}
	
	snprintf(majortrans, sizeof(majortrans), "%d", 
				CCM_GET_MAJORTRANS(info));
	snprintf(minortrans, sizeof(minortrans), "%d", 
				CCM_GET_MINORTRANS(info));
	cookie = CCM_GET_COOKIE(info);
	assert(cookie && *cookie);

	if ((ha_msg_add(m, F_TYPE, ccm_type2string(CCM_TYPE_LEAVE)) 
							== HA_FAIL)
		||(ha_msg_add(m, F_ORIG, nodename) == HA_FAIL) 
		||(ha_msg_add(m, CCM_MAJORTRANS, majortrans) == HA_FAIL)
		||(ha_msg_add(m, CCM_COOKIE, cookie) == HA_FAIL)
		||(ha_msg_add(m, CCM_MINORTRANS, minortrans) == HA_FAIL)) {
		cl_log(LOG_ERR, "ccm_create_leave_msg: Cannot create REQ_LEAVE "
						    "message");
		return NULL;
	} 
	return(m);
}


/* */
/* Watch out for new messages. As and when they arrive, return the */
/* message. */
/* */
static struct ha_msg *
ccm_readmsg(ccm_info_t *info, ll_cluster_t *hb)
{
	int 	uuid;

	assert(hb);

	/* check if there are any leave events to be delivered */
	if ((uuid=leave_get_next()) != -1) {
		/* create a leave message and return it */
		return ccm_create_leave_msg(info, uuid);
	}
	
	return hb->llc_ops->readmsg(hb, 0);
}



/* */
/* Move the state of this ccm node, from joining state directly to */
/* the joined state. */
/* */
/* NOTE: this is generally called when a joining nodes determines */
/* that it is the only node in the cluster, and everybody else are */
/* dead. */
/* */
static void
ccm_joining_to_joined(ll_cluster_t *hb, ccm_info_t *info)
{
	unsigned char *bitmap;
	char *cookie = NULL;

	/* create a bitmap with the membership information */
	(void) bitmap_create(&bitmap, MAXNODE);
	bitmap_mark(LLM_GET_MYUUID(CCM_GET_LLM(info)), bitmap, MAXNODE);

	/* 
	 * I am the only around! Lets discard any cookie that we
	 * got from others, and create a new cookie.
	 * This bug was noticed: when testing with partitioned
	 * clusters.
	 */
	cookie = ccm_generate_random_cookie();

	/* fill my new memlist and update the new cookie if any */
	ccm_fill_memlist_from_bitmap(info, bitmap);
	bitmap_delete(bitmap);

	/* increment the major transition number and reset the
	 * minor transition number
	 */
	CCM_INCREMENT_MAJORTRANS(info); 
	CCM_RESET_MINORTRANS(info);

	/* if cookie has changed update it.
	 */
	if (cookie) {
		cl_log(LOG_INFO, "ccm_joining_to_joined: "
				"cookie changed ");
		CCM_SET_COOKIE(info, cookie); 
		ccm_free_random_cookie(cookie);
	}

	/* check if any joiner is waiting for a response from us. 
	 * If so respond 
	 */
	ccm_send_join_reply(hb, info);
	g_slist_free(CCM_GET_JOINERHEAD(info));
	CCM_SET_JOINERHEAD(info, NULL);
	
	CCM_SET_CL(info, ccm_get_my_membership_index(info));
	update_reset(CCM_GET_UPDATETABLE(info));
	CCM_SET_STATE(info, CCM_STATE_JOINED);
	report_mbrs(info);
	if(!ccm_already_joined(info)) {
		CCM_SET_JOINED_TRANSITION(info, 1);
	}
	return;
}

/* */
/* Move the state of this ccm node, from init state directly to */
/* the joined state. */
/* */
/* NOTE: this is generally called when a node when it  determines */
/* that it is all alone in the cluster. */
/* */
static void
ccm_init_to_joined(ccm_info_t *info)
{
	int numBytes;
	unsigned char *bitlist;
	char *cookie;

	numBytes = bitmap_create(&bitlist, MAXNODE);
	bitmap_mark(LLM_GET_MYUUID(CCM_GET_LLM(info)), bitlist,MAXNODE);
	ccm_fill_memlist_from_bitmap(info, bitlist);
	bitmap_delete(bitlist);
	CCM_SET_MAJORTRANS(info, 1);
	CCM_SET_MINORTRANS(info, 0);
	cookie = ccm_generate_random_cookie();
	CCM_SET_COOKIE(info, cookie);
	ccm_free_random_cookie(cookie);
	CCM_SET_CL(info, ccm_get_my_membership_index(info));
	CCM_SET_STATE(info, CCM_STATE_JOINED);
	CCM_SET_JOINED_TRANSITION(info, 1);
	report_mbrs(info);
	return;
}



/* */
/* The state machine that processes message when it is */
/*	the CCM_STATE_VERSION_REQUEST state */
/* */
static void
ccm_state_version_request(enum ccm_type ccm_msg_type,
			struct ha_msg *reply,
			ll_cluster_t *hb, 
			ccm_info_t *info)
{
	const char *orig, *proto, *cookie, *trans, *clsize;
	uint trans_val;
	int  proto_val, clsize_val;
	int try;
	int repeat;
	
	/* who sent this message */
	if ((orig = ha_msg_value(reply, F_ORIG)) == NULL) {
		cl_log(LOG_WARNING, "ccm_state_version_request: "
			"received message from unknown");
		return;
	}

	if(!llm_is_valid_node(CCM_GET_LLM(info), orig)) { 
		cl_log(LOG_WARNING, "ccm_state_version_request: "
			"received message from unknown host %s", orig);
		return;
	}

	switch (ccm_msg_type)  {

	case CCM_TYPE_PROTOVERSION_RESP:

		/* get the protocol version */
		if ((proto = ha_msg_value(reply, CCM_PROTOCOL)) == NULL) {
			cl_log(LOG_WARNING, "ccm_state_version_request: "
					"no protocol information");
			return;
		}

		proto_val = atoi(proto); /*TOBEDONE*/
		if (proto_val >= CCM_VER_LAST) {
			cl_log(LOG_WARNING, "ccm_state_version_request: "
					"unknown protocol value");
			ccm_reset(info);
			return;
		}


		/* if this reply has come from a node which is a member
		 * of a larger cluster, we will try to join that cluster
		 * else we will wait for some time, by dropping this
		 * response.
		 */
		if(resp_can_i_drop()) {
			if ((clsize = ha_msg_value(reply, CCM_CLSIZE)) == NULL){
				cl_log(LOG_WARNING, "ccm_state_version_request: "
						" no cookie information");
				return;
			}
			clsize_val = atoi(clsize);
			if((clsize_val+1) <=
			   ((llm_get_active_nodecount(CCM_GET_LLM(info))+1)/2)) {
				/* drop the response. We will wait for 
			  	 * a response from a bigger group 
				 */
				resp_dropped();
				cl_shortsleep(); /* sleep for a while */
				/* send a fresh version request message */
				version_reset(CCM_GET_VERSION(info));
				CCM_SET_STATE(info, CCM_STATE_NONE);
				/* free all the joiners that we accumulated */
				g_slist_free(CCM_GET_JOINERHEAD(info));
				CCM_SET_JOINERHEAD(info, NULL);
				break;