vote.c

linux 内核源代码

/* -*- mode: c; c-basic-offset: 8; -*-
 * vim: noexpandtab sw=8 ts=8 sts=0:
 *
 * vote.c
 *
 * description here
 *
 * Copyright (C) 2003, 2004 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */

#include <linux/types.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/kthread.h>

#include <cluster/heartbeat.h>
#include <cluster/nodemanager.h>
#include <cluster/tcp.h>

#include <dlm/dlmapi.h>

#define MLOG_MASK_PREFIX ML_VOTE
#include <cluster/masklog.h>

#include "ocfs2.h"

#include "alloc.h"
#include "dlmglue.h"
#include "extent_map.h"
#include "heartbeat.h"
#include "inode.h"
#include "journal.h"
#include "slot_map.h"
#include "vote.h"

#include "buffer_head_io.h"

#define OCFS2_MESSAGE_TYPE_VOTE     (0x1)
#define OCFS2_MESSAGE_TYPE_RESPONSE (0x2)
struct ocfs2_msg_hdr
{
	__be32 h_response_id; /* used to lookup message handle on sending
			    * node. */
	__be32 h_request;
	__be64 h_blkno;
	__be32 h_generation;
	__be32 h_node_num;    /* node sending this particular message. */
};

struct ocfs2_vote_msg
{
	struct ocfs2_msg_hdr v_hdr;
	__be32 v_reserved1;
} __attribute__ ((packed));

/* Responses are given these values to maintain backwards
 * compatibility with older ocfs2 versions */
#define OCFS2_RESPONSE_OK		(0)
#define OCFS2_RESPONSE_BUSY		(-16)
#define OCFS2_RESPONSE_BAD_MSG		(-22)

struct ocfs2_response_msg
{
	struct ocfs2_msg_hdr r_hdr;
	__be32 r_response;
} __attribute__ ((packed));

struct ocfs2_vote_work {
	struct list_head   w_list;
	struct ocfs2_vote_msg w_msg;
};

enum ocfs2_vote_request {
	OCFS2_VOTE_REQ_INVALID = 0,
	OCFS2_VOTE_REQ_MOUNT,
	OCFS2_VOTE_REQ_UMOUNT,
	OCFS2_VOTE_REQ_LAST
};

static inline int ocfs2_is_valid_vote_request(int request)
{
	return OCFS2_VOTE_REQ_INVALID < request &&
		request < OCFS2_VOTE_REQ_LAST;
}

typedef void (*ocfs2_net_response_callback)(void *priv,
					    struct ocfs2_response_msg *resp);
struct ocfs2_net_response_cb {
	ocfs2_net_response_callback	rc_cb;
	void				*rc_priv;
};

struct ocfs2_net_wait_ctxt {
	struct list_head        n_list;
	u32                     n_response_id;
	wait_queue_head_t       n_event;
	struct ocfs2_node_map   n_node_map;
	int                     n_response; /* an agreggate response. 0 if
					     * all nodes are go, < 0 on any
					     * negative response from any
					     * node or network error. */
	struct ocfs2_net_response_cb *n_callback;
};

static void ocfs2_process_mount_request(struct ocfs2_super *osb,
					unsigned int node_num)
{
	mlog(0, "MOUNT vote from node %u\n", node_num);
	/* The other node only sends us this message when he has an EX
	 * on the superblock, so our recovery threads (if having been
	 * launched) are waiting on it.*/
	ocfs2_recovery_map_clear(osb, node_num);
	ocfs2_node_map_set_bit(osb, &osb->mounted_map, node_num);

	/* We clear the umount map here because a node may have been
	 * previously mounted, safely unmounted but never stopped
	 * heartbeating - in which case we'd have a stale entry. */
	ocfs2_node_map_clear_bit(osb, &osb->umount_map, node_num);
}

static void ocfs2_process_umount_request(struct ocfs2_super *osb,
					 unsigned int node_num)
{
	mlog(0, "UMOUNT vote from node %u\n", node_num);
	ocfs2_node_map_clear_bit(osb, &osb->mounted_map, node_num);
	ocfs2_node_map_set_bit(osb, &osb->umount_map, node_num);
}

static void ocfs2_process_vote(struct ocfs2_super *osb,
			       struct ocfs2_vote_msg *msg)
{
	int net_status, vote_response;
	unsigned int node_num;
	u64 blkno;
	enum ocfs2_vote_request request;
	struct ocfs2_msg_hdr *hdr = &msg->v_hdr;
	struct ocfs2_response_msg response;

	/* decode the network mumbo jumbo into local variables. */
	request = be32_to_cpu(hdr->h_request);
	blkno = be64_to_cpu(hdr->h_blkno);
	node_num = be32_to_cpu(hdr->h_node_num);

	mlog(0, "processing vote: request = %u, blkno = %llu, node_num = %u\n",
	     request, (unsigned long long)blkno, node_num);

	if (!ocfs2_is_valid_vote_request(request)) {
		mlog(ML_ERROR, "Invalid vote request %d from node %u\n",
		     request, node_num);
		vote_response = OCFS2_RESPONSE_BAD_MSG;
		goto respond;
	}

	vote_response = OCFS2_RESPONSE_OK;

	switch (request) {
	case OCFS2_VOTE_REQ_UMOUNT:
		ocfs2_process_umount_request(osb, node_num);
		goto respond;
	case OCFS2_VOTE_REQ_MOUNT:
		ocfs2_process_mount_request(osb, node_num);
		goto respond;
	default:
		/* avoids a gcc warning */
		break;
	}

respond:
	/* Response struture is small so we just put it on the stack
	 * and stuff it inline. */
	memset(&response, 0, sizeof(struct ocfs2_response_msg));
	response.r_hdr.h_response_id = hdr->h_response_id;
	response.r_hdr.h_blkno = hdr->h_blkno;
	response.r_hdr.h_generation = hdr->h_generation;
	response.r_hdr.h_node_num = cpu_to_be32(osb->node_num);
	response.r_response = cpu_to_be32(vote_response);

	net_status = o2net_send_message(OCFS2_MESSAGE_TYPE_RESPONSE,
					osb->net_key,
					&response,
					sizeof(struct ocfs2_response_msg),
					node_num,
					NULL);
	/* We still want to error print for ENOPROTOOPT here. The
	 * sending node shouldn't have unregistered his net handler
	 * without sending an unmount vote 1st */
	if (net_status < 0
	    && net_status != -ETIMEDOUT
	    && net_status != -ENOTCONN)
		mlog(ML_ERROR, "message to node %u fails with error %d!\n",
		     node_num, net_status);
}

static void ocfs2_vote_thread_do_work(struct ocfs2_super *osb)
{
	unsigned long processed;
	struct ocfs2_lock_res *lockres;
	struct ocfs2_vote_work *work;

	mlog_entry_void();

	spin_lock(&osb->vote_task_lock);
	/* grab this early so we know to try again if a state change and
	 * wake happens part-way through our work  */
	osb->vote_work_sequence = osb->vote_wake_sequence;

	processed = osb->blocked_lock_count;
	while (processed) {
		BUG_ON(list_empty(&osb->blocked_lock_list));

		lockres = list_entry(osb->blocked_lock_list.next,
				     struct ocfs2_lock_res, l_blocked_list);
		list_del_init(&lockres->l_blocked_list);
		osb->blocked_lock_count--;
		spin_unlock(&osb->vote_task_lock);

		BUG_ON(!processed);
		processed--;

		ocfs2_process_blocked_lock(osb, lockres);

		spin_lock(&osb->vote_task_lock);
	}

	while (osb->vote_count) {
		BUG_ON(list_empty(&osb->vote_list));
		work = list_entry(osb->vote_list.next,
				  struct ocfs2_vote_work, w_list);
		list_del(&work->w_list);
		osb->vote_count--;
		spin_unlock(&osb->vote_task_lock);

		ocfs2_process_vote(osb, &work->w_msg);
		kfree(work);

		spin_lock(&osb->vote_task_lock);
	}
	spin_unlock(&osb->vote_task_lock);

	mlog_exit_void();
}

static int ocfs2_vote_thread_lists_empty(struct ocfs2_super *osb)
{
	int empty = 0;

	spin_lock(&osb->vote_task_lock);
	if (list_empty(&osb->blocked_lock_list) &&
	    list_empty(&osb->vote_list))
		empty = 1;

	spin_unlock(&osb->vote_task_lock);
	return empty;
}

static int ocfs2_vote_thread_should_wake(struct ocfs2_super *osb)
{
	int should_wake = 0;

	spin_lock(&osb->vote_task_lock);
	if (osb->vote_work_sequence != osb->vote_wake_sequence)
		should_wake = 1;
	spin_unlock(&osb->vote_task_lock);

	return should_wake;
}

int ocfs2_vote_thread(void *arg)
{
	int status = 0;
	struct ocfs2_super *osb = arg;

	/* only quit once we've been asked to stop and there is no more
	 * work available */
	while (!(kthread_should_stop() &&
		 ocfs2_vote_thread_lists_empty(osb))) {

		wait_event_interruptible(osb->vote_event,
					 ocfs2_vote_thread_should_wake(osb) ||
					 kthread_should_stop());

		mlog(0, "vote_thread: awoken\n");

		ocfs2_vote_thread_do_work(osb);
	}

	osb->vote_task = NULL;
	return status;
}

static struct ocfs2_net_wait_ctxt *ocfs2_new_net_wait_ctxt(unsigned int response_id)
{
	struct ocfs2_net_wait_ctxt *w;

	w = kzalloc(sizeof(*w), GFP_NOFS);
	if (!w) {
		mlog_errno(-ENOMEM);
		goto bail;
	}

	INIT_LIST_HEAD(&w->n_list);
	init_waitqueue_head(&w->n_event);
	ocfs2_node_map_init(&w->n_node_map);
	w->n_response_id = response_id;
	w->n_callback = NULL;
bail:
	return w;
}

static unsigned int ocfs2_new_response_id(struct ocfs2_super *osb)
{
	unsigned int ret;

	spin_lock(&osb->net_response_lock);
	ret = ++osb->net_response_ids;
	spin_unlock(&osb->net_response_lock);

	return ret;
}

static void ocfs2_dequeue_net_wait_ctxt(struct ocfs2_super *osb,
					struct ocfs2_net_wait_ctxt *w)
{
	spin_lock(&osb->net_response_lock);
	list_del(&w->n_list);
	spin_unlock(&osb->net_response_lock);
}

static void ocfs2_queue_net_wait_ctxt(struct ocfs2_super *osb,
				      struct ocfs2_net_wait_ctxt *w)
{
	spin_lock(&osb->net_response_lock);
	list_add_tail(&w->n_list,
		      &osb->net_response_list);
	spin_unlock(&osb->net_response_lock);
}

static void __ocfs2_mark_node_responded(struct ocfs2_super *osb,
					struct ocfs2_net_wait_ctxt *w,
					int node_num)
{
	assert_spin_locked(&osb->net_response_lock);

	ocfs2_node_map_clear_bit(osb, &w->n_node_map, node_num);
	if (ocfs2_node_map_is_empty(osb, &w->n_node_map))
		wake_up(&w->n_event);
}

/* Intended to be called from the node down callback, we fake remove
 * the node from all our response contexts */
void ocfs2_remove_node_from_vote_queues(struct ocfs2_super *osb,
					int node_num)
{
	struct list_head *p;
	struct ocfs2_net_wait_ctxt *w = NULL;

	spin_lock(&osb->net_response_lock);

	list_for_each(p, &osb->net_response_list) {
		w = list_entry(p, struct ocfs2_net_wait_ctxt, n_list);

		__ocfs2_mark_node_responded(osb, w, node_num);
	}