ccm.c

linux集群服务器软件代码包

/* sends  a  leave  message  to  the  other nodes in the cluster.  */
/* Similarly  whenever  ccm  running on some node of the cluster, */
/* dies  the  local  heartbeat   delivers a leave message to ccm. */
/* And  whenever  some node in the cluster dies, local heartbeat  */
/* informs  the  death  through  a  callback.  */
/* In all these cases, ccm is informed about the loss of the node, */
/* asynchronously, in  some context where immidiate processing of  */
/* the message is not possible.  */
/* The  following  set of routines act as a cache that keep track  */
/* of  message  leaves  and  facilitates  the  delivery  of these  */
/* messages at a convinient time. */
/*  */
/* */
static unsigned char *leave_bitmap=NULL;

static void
leave_init(void)
{
	int numBytes;

	assert(!leave_bitmap);
	numBytes = bitmap_create(&leave_bitmap, MAXNODE);
	memset(leave_bitmap, 0, numBytes);
}

static void
leave_reset(void)
{
	int numBytes = bitmap_size(MAXNODE);
	if(!leave_bitmap) {
		return;
	}
	memset(leave_bitmap, 0, numBytes);
	return;
}

static void
leave_cache(int i)
{
	assert(leave_bitmap);
	bitmap_mark(i, leave_bitmap, MAXNODE);
}

static int
leave_get_next(void)
{
	int i;

	assert(leave_bitmap);
	for ( i = 0 ; i < MAXNODE; i++ ) {
		if(bitmap_test(i,leave_bitmap,MAXNODE)) {
			bitmap_clear(i,leave_bitmap,MAXNODE);
			return i;
		}
	}
	return -1;
}

static int
leave_any(void)
{
	if(bitmap_count(leave_bitmap,MAXNODE)){
		return TRUE;
	}
	return FALSE;
}
/* leave bitmap relate routines end */




/* Reset all the datastructures. Go to a state which is equivalent */
/* to a state when the node is just about to join a cluster. */
static void 
ccm_reset(ccm_info_t *info)
{

	if(ccm_already_joined(info)){
		client_evicted();
	}

	CCM_RESET_MEMBERSHIP(info);
	ccm_memcomp_reset(info);
	CCM_SET_ACTIVEPROTO(info, CCM_VER_NONE);
	CCM_SET_COOKIE(info,"");
	CCM_SET_MAJORTRANS(info,0);
	CCM_SET_MINORTRANS(info,0);
	CCM_SET_CL(info,-1);
	CCM_SET_JOINED_TRANSITION(info, 0);
	CCM_SET_STATE(info, CCM_STATE_NONE);
	update_reset(CCM_GET_UPDATETABLE(info));
	g_slist_free(CCM_GET_JOINERHEAD(info));
	CCM_SET_JOINERHEAD(info, NULL);
	version_reset(CCM_GET_VERSION(info));
	finallist_reset();
	leave_reset();
	report_reset();
}

static void 
ccm_init(ccm_info_t *info)
{
	update_init(CCM_GET_UPDATETABLE(info));
	CCM_SET_JOINERHEAD(info, NULL);
	CCM_INIT_MAXTRANS(info);
        leave_init();
        (void)timeout_msg_init(info);
	ccm_reset(info);
}



/*
 * BEGIN OF ROUTINES THAT REPORT THE MEMBERSHIP TO CLIENTS.
 */
static void
report_reset(void)
{
	return;
}

/* */
/* print and report the cluster membership to clients. */
/* */
static void
report_mbrs(ccm_info_t *info)
{
	int i;
	char *nodename;

	static struct born_s  {
		int index;
		int bornon;
	}  bornon[MAXNODE];/*avoid making it a 
				stack variable*/
	

	if(CCM_GET_MEMCOUNT(info)==1){
		bornon[0].index  = CCM_GET_MEMINDEX(info,0);
		bornon[0].bornon = CCM_GET_MAJORTRANS(info);
	} else for(i=0; i < CCM_GET_MEMCOUNT(info); i++){
		bornon[i].index = CCM_GET_MEMINDEX(info,i);
		bornon[i].bornon = update_get_uptime(CCM_GET_UPDATETABLE(info), 
				CCM_GET_LLM(info),
				CCM_GET_MEMINDEX(info,i));
		if(bornon[i].bornon==0) 
			bornon[i].bornon=CCM_GET_MAJORTRANS(info);
		assert(bornon[i].bornon!=-1);
	}

	if(global_verbose) {
		cl_log(LOG_DEBUG,"\t\t the following are the members " 
			"of the group of transition=%d",
			CCM_GET_MAJORTRANS(info));

		for (i=0 ;  i < CCM_GET_MEMCOUNT(info); i++) {
			nodename = LLM_GET_NODEID(CCM_GET_LLM(info), 
					CCM_GET_MEMINDEX(info,i));
			cl_log(LOG_DEBUG,"\t\tnodename=%s bornon=%d", nodename, 
					bornon[i].bornon);
		}
	}


	/* 
	 * report to clients, the new membership 
	 */
	client_new_mbrship(info,
			   bornon);
	return;
}
/*
 * END OF ROUTINES THAT REPORT THE MEMBERSHIP TO CLIENTS.
 */




/* */
/* generate a random cookie. */
/* NOTE: cookie  is  a  mechanism  of  seperating out the contexts */
/* of  messages  of  partially  partitioned  clusters. */
/* For example, consider  a  case  where   node  A  is  physically */
/* in  the  partition  X  and  partition  Y,  and  but  has joined  */
/* membership  in partition X. It will end up getting ccm protocol */
/* messages  sent  by  members in both the partitions. In order to  */
/* seperate  out  messages  belonging  to  individual partition, a  */
/* random  string  is  used  as  a identifier by each partition to  */
/* identify  its  messages.  In  the above case A will get message  */
/* from  both  the  partitions  but  only listens to messages from  */
/* partition X and drops messages from partition Y. */
/* */
static char *
ccm_generate_random_cookie(void)
{
	char *cookie;
	int i;
	struct timeval tmp;

	cookie = g_malloc(COOKIESIZE*sizeof(char));
	/* g_malloc never returns NULL: assert(cookie); */

	/* seed the random with a random value */
	gettimeofday(&tmp, NULL);
	srandom((unsigned int)tmp.tv_usec); 

	for ( i = 0 ; i < COOKIESIZE-1; i++ ) {
		cookie[i] = random()%(127-'!')+'!';
	}
	cookie[i] = '\0';
	return cookie;
}


static void
ccm_free_random_cookie(char *cookie)
{
	assert(cookie && *cookie);
	g_free(cookie);
}



/* BEGIN OF FUNCTIONS that keep track of connectivity  information  */
/* conveyed by individual members of the cluster. These  functions  */
/* are used by only the cluster leader. Ultimately these connectivity */
/* information is used by the cluster to extract out the members */
/* of the cluster that have total connectivity. */
static int
ccm_memcomp_cmpr(gconstpointer a, gconstpointer b)
{
	return(*((const uint32_t *)a)-*((const uint32_t *)b));
}
static void
ccm_memcomp_free(gpointer data, gpointer userdata)
{
	if(data) {
		g_free(data);
	}
	return;
}

static void
ccm_memcomp_note(ccm_info_t *info, const char *orig, 
		uint32_t maxtrans, const char *memlist)
{
	int uuid, numbytes;
	unsigned char *bitlist;
	uint32_t *ptr;
	memcomp_t *mem_comp = CCM_GET_MEMCOMP(info);

	/* find the uuid of the originator */
	uuid = llm_get_uuid(CCM_GET_LLM(info), orig);

	/* convert the memlist into a bit map and feed it to the graph */
	numbytes = ccm_str2bitmap(memlist, &bitlist);
	
	graph_update_membership(MEMCOMP_GET_GRAPH(mem_comp), 
			uuid, bitlist);
	/*NOTE DO NOT DELETE bitlist, because it is 
	 * being handled by graph*/

	ptr = (uint32_t *)g_malloc(2*sizeof(uint32_t));
	ptr[0] = maxtrans;
	ptr[1] = uuid;
	MEMCOMP_SET_MAXT(mem_comp, 
		(g_slist_insert_sorted(MEMCOMP_GET_MAXT(mem_comp), 
			ptr, ccm_memcomp_cmpr)));
	return;
}

/* called by the cluster leader only  */
static void
ccm_memcomp_note_my_membership(ccm_info_t *info)
{
	char *memlist;
	int str_len;

	/* find the uuid of the originator */
	str_len = update_strcreate(CCM_GET_UPDATETABLE(info), 
			&memlist, CCM_GET_LLM(info));
	ccm_memcomp_note(info, ccm_get_my_hostname(info), 
			CCM_GET_MAXTRANS(info), memlist);
	update_strdelete(memlist);
	return;
}

/* add a new member to the membership list */
static void
ccm_memcomp_add(ccm_info_t *info, const char *orig)
{
	int uuid, myuuid;
	memcomp_t *mem_comp = CCM_GET_MEMCOMP(info);

	uuid = llm_get_uuid(CCM_GET_LLM(info), orig);
	myuuid = llm_get_uuid(CCM_GET_LLM(info), 
			ccm_get_my_hostname(info));
	graph_add_uuid(MEMCOMP_GET_GRAPH(mem_comp), uuid);
	graph_add_to_membership(MEMCOMP_GET_GRAPH(mem_comp), 
			myuuid, uuid);
	/* ccm_memcomp_note(info, orig, maxtrans, memlist); */
	return;
}

static void 
ccm_memcomp_init(ccm_info_t *info)
{
	int track=-1;
	int uuid;
	
	memcomp_t *mem_comp = CCM_GET_MEMCOMP(info);

	MEMCOMP_SET_GRAPH(mem_comp, graph_init());

	/* go through the update list and note down all the members who
	 * had participated in the join messages. We should be expecting
	 * reply memlist bitmaps atleast from these nodes.
	 */
	while((uuid = update_get_next_uuid(CCM_GET_UPDATETABLE(info), 
				CCM_GET_LLM(info), &track)) != -1) {
		graph_add_uuid(MEMCOMP_GET_GRAPH(mem_comp),uuid); 
	}
	MEMCOMP_SET_MAXT(mem_comp,  NULL);
	MEMCOMP_SET_INITTIME(mem_comp, ccm_get_time());
}


static void 
ccm_memcomp_reset(ccm_info_t *info)
{
	GSList *head;
	memcomp_t *mem_comp = CCM_GET_MEMCOMP(info);

	graph_free(MEMCOMP_GET_GRAPH(mem_comp));
	MEMCOMP_SET_GRAPH(mem_comp,NULL);
	head = MEMCOMP_GET_MAXT(mem_comp);
	g_slist_foreach(MEMCOMP_GET_MAXT(mem_comp), 
			ccm_memcomp_free, NULL);
	g_slist_free(MEMCOMP_GET_MAXT(mem_comp));
	MEMCOMP_SET_MAXT(mem_comp,  NULL);
	return;
}


static int
ccm_memcomp_rcvd_all(ccm_info_t *info)
{
	return graph_filled_all(MEMCOMP_GET_GRAPH(CCM_GET_MEMCOMP(info)));
}

static int
ccm_memcomp_timeout(ccm_info_t *info, long timeout)
{
	memcomp_t *mem_comp = CCM_GET_MEMCOMP(info);

	return(ccm_timeout(MEMCOMP_GET_INITTIME(mem_comp), 
				ccm_get_time(), timeout));
}

static int
ccm_memcomp_get_maxmembership(ccm_info_t *info, unsigned char **bitmap)
{
	GSList *head;
	uint32_t *ptr;
	int 	uuid;

	memcomp_t *mem_comp = CCM_GET_MEMCOMP(info);

	(void)graph_get_maxclique(MEMCOMP_GET_GRAPH(mem_comp), 
			bitmap);

	head = MEMCOMP_GET_MAXT(mem_comp);

	while (head) {
		ptr = (uint32_t *)g_slist_nth_data(head, 0);
		uuid = ptr[1];
		if(bitmap_test(uuid, *bitmap, MAXNODE)) {
			return ptr[0];
		}
		head = g_slist_next(head);
	}
	return 0;
}


/* */
/* END OF the membership tracking functions. */
/* */


static int 
ccm_am_i_member(ccm_info_t *info, const char *memlist)
{
	unsigned char *bitmap;

	int numBytes = ccm_str2bitmap(memlist, &bitmap);

	/* what is my node Uuid */
	llm_info_t *llm = CCM_GET_LLM(info);

	int my_uuid = LLM_GET_MYUUID(llm);

	if (bitmap_test(my_uuid, bitmap, numBytes)){
		bitmap_delete(bitmap);
		return TRUE;
	}

	bitmap_delete(bitmap);
	return FALSE;
}


/* */
/* BEGIN  OF  FUNCTIONS  that  keep track of stablized membership list */
/*  */
/* These  function  keep track of consensus membership once a instance */
/* of the  ccm algorithm terminates and decided on the final consensus  */
/* members of the cluster. */
/* */
static int 
ccm_memlist_changed(ccm_info_t *info, 
		  char *bitmap /* the bitmap string containing bits */)
{
	int nodeCount, i;
	llm_info_t *llm;
	uint indx, uuid;
		
		
	/* go through the membership list */
	nodeCount = CCM_GET_MEMCOUNT(info);
	llm = CCM_GET_LLM(info);
	for ( i = 0 ; i < nodeCount; i++ ) {
		indx = CCM_GET_MEMINDEX(info, i);
		assert(indx >=0 && indx < LLM_GET_NODECOUNT(llm));
		uuid = LLM_GET_UUID(llm,indx);
		assert(uuid>=0 && uuid < MAXNODE);
		if (!bitmap_test(uuid, bitmap, MAXNODE)){
			return TRUE;
		}
	}
	return FALSE;
} 

static int 
ccm_fill_memlist(ccm_info_t *info, 
	const unsigned char *bitmap)
{
	llm_info_t *llm;
	uint i, uuid;

	llm = CCM_GET_LLM(info);
	CCM_RESET_MEMBERSHIP(info);
	for ( i = 0 ; i < LLM_GET_NODECOUNT(llm); i++ ) {
		uuid = LLM_GET_UUID(llm,i);
		if(bitmap_test(uuid, bitmap, MAXNODE)){
			/*update the membership list with this member*/
			CCM_ADD_MEMBERSHIP(info, i);
		}
	}

	return FALSE;
}

static int 
ccm_fill_memlist_from_str(ccm_info_t *info, 
	const unsigned char *memlist)
{
	unsigned char *bitmap;
	int ret;

	(void)ccm_str2bitmap(memlist, &bitmap);
	ret = ccm_fill_memlist(info, bitmap);
	bitmap_delete(bitmap);
	return ret;
}

									
static int 
ccm_fill_memlist_from_bitmap(ccm_info_t *info, 
	const unsigned char *bitmap)
{
	return ccm_fill_memlist(info, bitmap);
}






static int
ccm_get_membership_index(ccm_info_t *info, const char *node)
{
	int i,indx;