ccm.c

linux集群服务器软件代码包

/* $Id: ccm.c,v 1.56 2005/02/20 14:32:00 andrew Exp $ */
/* 
 * Copyright (C) 2004 Andrew Beekhof <[e-mail]>
 * 
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * 
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

/* put these first so that uuid_t is defined without conflicts */
#include <portability.h>
#include <ocf/oc_event.h>
#include <ocf/oc_membership.h>

#include <clplumbing/GSource.h>
#include <string.h>

#include <crm/crm.h>
#include <crm/cib.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crmd_messages.h>
#include <crmd_fsa.h>
#include <fsa_proto.h>
#include <crmd_callbacks.h>

#include <crm/dmalloc_wrapper.h>

void oc_ev_special(const oc_ev_t *, oc_ev_class_t , int );

int register_with_ccm(ll_cluster_t *hb_cluster);

void msg_ccm_join(const HA_Message *msg, void *foo);

void crmd_ccm_msg_callback(oc_ed_t event,
			     void *cookie,
			     size_t size,
			     const void *data);

gboolean ghash_node_clfree(gpointer key, gpointer value, gpointer user_data);
void ghash_update_cib_node(gpointer key, gpointer value, gpointer user_data);

#define CCM_EVENT_DETAIL 1
oc_ev_t *fsa_ev_token;
int num_ccm_register_fails = 0;
int max_ccm_register_fails = 30;

/*	 A_CCM_CONNECT	*/
enum crmd_fsa_input
do_ccm_control(long long action,
		enum crmd_fsa_cause cause,
		enum crmd_fsa_state cur_state,
		enum crmd_fsa_input current_input,
		fsa_data_t *msg_data)
{
	int      ret;
 	int	 fsa_ev_fd; 
	gboolean did_fail = FALSE;
	
	if(action & A_CCM_DISCONNECT){
		oc_ev_unregister(fsa_ev_token);
	}

	if(action & A_CCM_CONNECT) {
		crm_info("Registering with CCM");
		ret = oc_ev_register(&fsa_ev_token);
		if (ret != 0) {
			crm_warn("CCM registration failed");
			did_fail = TRUE;
		}

		if(did_fail == FALSE) {
			crm_info("Setting up CCM callbacks");
			ret = oc_ev_set_callback(fsa_ev_token, OC_EV_MEMB_CLASS,
						 crmd_ccm_msg_callback, NULL);
			if (ret != 0) {
				crm_warn("CCM callback not set");
				did_fail = TRUE;
			}
		}
		if(did_fail == FALSE) {
			oc_ev_special(fsa_ev_token, OC_EV_MEMB_CLASS, 0/*don't care*/);
			
			crm_info("Activating CCM token");
			ret = oc_ev_activate(fsa_ev_token, &fsa_ev_fd);
			if (ret != 0){
				crm_warn("CCM Activation failed");
				did_fail = TRUE;
			}
		}

		if(did_fail) {
			num_ccm_register_fails++;
			oc_ev_unregister(fsa_ev_token);
			
			if(num_ccm_register_fails < max_ccm_register_fails) {
				crm_warn("CCM Connection failed"
					 " %d times (%d max)",
					 num_ccm_register_fails,
					 max_ccm_register_fails);
				
				startTimer(wait_timer);
				crmd_fsa_stall();
				return I_NULL;
				
			} else {
				crm_err("CCM Activation failed %d (max) times",
					num_ccm_register_fails);
				register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
				return I_NULL;
			}
		}
		

		crm_info("CCM Activation passed... all set to go!");

/* GFDSource* */
		G_main_add_fd(G_PRIORITY_LOW, fsa_ev_fd, FALSE, ccm_dispatch,
			      fsa_ev_token,
			      default_ipc_connection_destroy);
		
	}

	if(action & ~(A_CCM_CONNECT|A_CCM_DISCONNECT)) {
		crm_err("Unexpected action %s in %s",
		       fsa_action2string(action), __FUNCTION__);
	}
	
	return I_NULL;
}


/*	 A_CCM_EVENT	*/
enum crmd_fsa_input
do_ccm_event(long long action,
	     enum crmd_fsa_cause cause,
	     enum crmd_fsa_state cur_state,
	     enum crmd_fsa_input current_input,
	     fsa_data_t *msg_data)
{
	enum crmd_fsa_input return_input = I_NULL;
	oc_ed_t *event = NULL;
	const oc_ev_membership_t *oc = NULL;
	struct crmd_ccm_data_s *ccm_data = fsa_typed_data(fsa_dt_ccm);
	
	if(ccm_data == NULL) {
		crm_err("No data provided to FSA function");
		register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
		return I_NULL;

	} else if(msg_data->fsa_cause != C_CCM_CALLBACK) {
		crm_err("FSA function called in response to incorect input");
		register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
		return I_NULL;
	}

	event = ccm_data->event;
	oc = ccm_data->oc;
	
	crm_info("event=%s", 
	       *event==OC_EV_MS_NEW_MEMBERSHIP?"NEW MEMBERSHIP":
	       *event==OC_EV_MS_NOT_PRIMARY?"NOT PRIMARY":
	       *event==OC_EV_MS_PRIMARY_RESTORED?"PRIMARY RESTORED":
	       *event==OC_EV_MS_EVICTED?"EVICTED":
	       "NO QUORUM MEMBERSHIP");
	
	if(CCM_EVENT_DETAIL) {
		ccm_event_detail(oc, *event);
	}

	if (OC_EV_MS_EVICTED == *event) {
		/* todo: drop back to S_PENDING instead */
		/* get out... NOW!
		 *
		 * go via the error recovery process so that HA will
		 *    restart us if required
		 */
		register_fsa_error(cause, I_ERROR, msg_data->data);
		return I_NULL;
	}
	
	
	/* My understanding is that we will never get both
	 * node leaving *and* node joining callbacks at the
	 * same time.
	 *
	 * This logic would need to change if this is not
	 * the case
	 */
	
	if(oc->m_n_out != 0) {
		int lpc = 0;
		int offset = oc->m_out_idx;
		for(lpc=0; lpc < oc->m_n_out; lpc++) {
			crm_data_t *node_state = NULL;
			
			const char *uname = oc->m_array[offset+lpc].node_uname;
			
			crm_info("Node %s has left the cluster,"
				 " updating the CIB.", uname);

			if(uname == NULL) {
				crm_err("CCM node had no name");
				continue;
				
			} else if(safe_str_eq(uname, fsa_our_dc)) {
				/* did our DC leave us */
				register_fsa_input(cause, I_ELECTION, NULL);

			} else if(AM_I_DC) {
				node_state = create_node_state(
					NULL, uname, NULL,
					XML_BOOLEAN_NO, NULL, NULL, NULL);
				
				update_local_cib(
					create_cib_fragment(node_state, NULL));
				free_xml(node_state);
			}
		}
		
	} else if(oc->m_n_in !=0) {
		/* delay the I_NODE_JOIN until they acknowledge our
		 * DC status and send us their CIB
		 */
	} else {
		CRM_DEV_ASSERT(oc->m_n_in != 0 || oc->m_n_out != 0);
		crm_warn("So why are we here?  What CCM event happened?");
	}

	return return_input;
}

/*	 A_CCM_UPDATE_CACHE	*/
/*
 * Take the opportunity to update the node status in the CIB as well
 *  (but only if we are the DC)
 */
enum crmd_fsa_input
do_ccm_update_cache(long long action,
		    enum crmd_fsa_cause cause,
		    enum crmd_fsa_state cur_state,
		    enum crmd_fsa_input current_input,
		    fsa_data_t *msg_data)
{
	enum crmd_fsa_input next_input = I_NULL;
	int lpc, offset;
	GHashTable *members = NULL;
	oc_ed_t *event = NULL;
	const oc_ev_membership_t *oc = NULL;
	oc_node_list_t *tmp = NULL, *membership_copy = NULL;
	struct crmd_ccm_data_s *ccm_data = fsa_typed_data(fsa_dt_ccm);

	if(ccm_data == NULL) {
		crm_err("No data provided to FSA function");
		register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
		return I_NULL;
	}

	event = ccm_data->event;
	oc = ccm_data->oc;

	crm_info("Updating CCM cache after a \"%s\" event.", 
	       *event==OC_EV_MS_NEW_MEMBERSHIP?"NEW MEMBERSHIP":
	       *event==OC_EV_MS_NOT_PRIMARY?"NOT PRIMARY":
	       *event==OC_EV_MS_PRIMARY_RESTORED?"PRIMARY RESTORED":
	       *event==OC_EV_MS_EVICTED?"EVICTED":
	       "NO QUORUM MEMBERSHIP");

	crm_debug("instace=%d, nodes=%d, new=%d, lost=%d n_idx=%d, "
		  "new_idx=%d, old_idx=%d",
		  oc->m_instance,
		  oc->m_n_member,
		  oc->m_n_in,
		  oc->m_n_out,
		  oc->m_memb_idx,
		  oc->m_in_idx,
		  oc->m_out_idx);

	crm_malloc(membership_copy, sizeof(oc_node_list_t));

	if(membership_copy == NULL) {
		crm_crit("Couldnt create membership copy - out of memory");
		register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
		return I_NULL;
	}

	crm_devel("Copying members");
	
	/*--*-- All Member Nodes --*--*/
	offset = oc->m_memb_idx;
	membership_copy->members_size = oc->m_n_member;

	if(membership_copy->members_size > 0) {
		membership_copy->members =
			g_hash_table_new(g_str_hash, g_str_equal);
		members = membership_copy->members;
		
		for(lpc=0; lpc < membership_copy->members_size; lpc++) {
			oc_node_t *member = NULL;
			crm_devel("Copying member %d", lpc);
			crm_malloc(member, sizeof(oc_node_t));
			
			if(member == NULL) {
				continue;
			}

			member->node_id =
				oc->m_array[offset+lpc].node_id;
			
			member->node_born_on =
				oc->m_array[offset+lpc].node_born_on;
			
			member->node_uname = NULL;
			if(oc->m_array[offset+lpc].node_uname != NULL) {