unify.c

分布式文件系统

/*
   Copyright (c) 2006, 2007, 2008 Z RESEARCH, Inc. <http://www.zresearch.com>
   This file is part of GlusterFS.

   GlusterFS is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published
   by the Free Software Foundation; either version 3 of the License,
   or (at your option) any later version.

   GlusterFS is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see
   <http://www.gnu.org/licenses/>.
*/

/**
 * xlators/cluster/unify:
 *     - This xlator is one of the main translator in GlusterFS, which
 *   actually does the clustering work of the file system. One need to 
 *   understand that, unify assumes file to be existing in only one of 
 *   the child node, and directories to be present on all the nodes. 
 *
 * NOTE:
 *   Now, unify has support for global namespace, which is used to keep a 
 * global view of fs's namespace tree. The stat for directories are taken
 * just from the namespace, where as for files, just 'st_ino' is taken from
 * Namespace node, and other stat info is taken from the actual storage node.
 * Also Namespace node helps to keep consistant inode for files across 
 * glusterfs (re-)mounts.
 */

#ifndef _CONFIG_H
#define _CONFIG_H
#include "config.h"
#endif

#include "glusterfs.h"
#include "unify.h"
#include "dict.h"
#include "xlator.h"
#include "hashfn.h"
#include "logging.h"
#include "stack.h"
#include "defaults.h"
#include "common-utils.h"

#define CHILDDOWN ENOTCONN

#define UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR(_loc) do { \
  if (!(_loc && _loc->inode && _loc->inode->ctx &&         \
	dict_get (_loc->inode->ctx, this->name))) {        \
    TRAP_ON (!(_loc && _loc->inode && _loc->inode->ctx &&  \
               dict_get (_loc->inode->ctx, this->name)));  \
    STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);    \
    return 0;                                              \
  }                                                        \
} while(0)


#define UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(_fd) do { \
  if (!(_fd && _fd->ctx &&                             \
	dict_get (_fd->ctx, this->name))) {            \
    TRAP_ON (!(_fd && _fd->ctx &&                      \
	          dict_get (_fd->ctx, this->name)));   \
    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);      \
    return 0;                                          \
  }                                                    \
} while(0)

#define UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(_fd) do { \
  if (!(_fd && _fd->ctx)) {                        \
    TRAP_ON (!(_fd && _fd->ctx));                  \
    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);  \
    return 0;                                      \
  }                                                \
} while(0)

/**
 * unify_local_wipe - free all the extra allocation of local->* here.
 */
static void 
unify_local_wipe (unify_local_t *local)
{
  /* Free the strdup'd variables in the local structure */
  if (local->path) {
    freee (local->path);
  }
  if (local->name) {
    freee (local->name);
  }
}

/**
 * unify_buf_cbk - 
 */
int32_t
unify_buf_cbk (call_frame_t *frame,
	       void *cookie,
	       xlator_t *this,
	       int32_t op_ret,
	       int32_t op_errno,
	       struct stat *buf)
{
  int32_t callcnt = 0;
  unify_local_t *local = frame->local;
  call_frame_t *prev_frame = cookie;

  LOCK (&frame->lock);
  {
    callcnt = --local->call_count;
    
    if (op_ret == -1) {
      gf_log (this->name, GF_LOG_ERROR,
	      "%s returned %d", prev_frame->this->name, op_errno);
      local->op_errno = op_errno;
    }

    if (op_ret >= 0) {
      local->op_ret = op_ret;

      if (NS (this) == prev_frame->this) {
	local->st_ino = buf->st_ino;
	/* If the entry is directory, get the stat from NS node */
	if (S_ISDIR (buf->st_mode) || !local->stbuf.st_blksize) {
	  local->stbuf = *buf;
	}
      }

      if ((!S_ISDIR (buf->st_mode)) && 
	  (NS (this) != prev_frame->this)) {
	/* If file, take the stat info from Storage node. */
	local->stbuf = *buf;
      }
    }
  }
  UNLOCK (&frame->lock);
    
  if (!callcnt) {
    local->stbuf.st_ino = local->st_ino;
    unify_local_wipe (local);
    STACK_UNWIND (frame, local->op_ret, local->op_errno, &local->stbuf);
  }

  return 0;
}


/**
 * unify_lookup_cbk - 
 */
int32_t 
unify_lookup_cbk (call_frame_t *frame,
		  void *cookie,
		  xlator_t *this,
		  int32_t op_ret,
		  int32_t op_errno,
		  inode_t *inode,
		  struct stat *buf,
		  dict_t *dict)
{
  int32_t callcnt = 0;
  unify_private_t *priv = this->private;
  unify_local_t *local = frame->local;

  LOCK (&frame->lock);
  {
    callcnt = --local->call_count;
 
    if (op_ret == -1) {
      if ((!local->revalidate) && 
	  op_errno != CHILDDOWN && op_errno != ENOENT) {
	gf_log (this->name, GF_LOG_ERROR,
		"%s returned %d", priv->xl_array[(long)cookie]->name, op_errno);
	local->op_errno = op_errno;
	local->failed = 1;
      } else if (local->revalidate) {
	gf_log (this->name, GF_LOG_ERROR,
		"%s returned %d", priv->xl_array[(long)cookie]->name, op_errno);
	local->op_errno = op_errno;
	local->failed = 1;
      }
    }

    if (op_ret == 0) {
      local->op_ret = 0; 
      if (!local->revalidate) {
	/* This is the first time lookup */
	if (!local->list) {
	  /* list is not allocated, allocate  the max possible range */
	  local->list = calloc (1, sizeof (int16_t) * (priv->child_count + 2));
	  if (!local->list) {
	    gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
	    STACK_UNWIND (frame, -1, ENOMEM, local->inode, NULL, NULL);
	    return 0;
	  }
	}
	/* update the index of the list */
	if (local->dict && dict)
	  local->dict = dict_ref (dict);

	local->list [local->index++] = (int16_t)(long)cookie;
      }

      /* index of NS node is == total child count */
      if (priv->child_count == (int16_t)(long)cookie) {
	/* Take the inode number from namespace */
	local->st_ino = buf->st_ino;
	local->inode = inode;
	inode->st_mode = buf->st_mode;
	if (S_ISDIR (buf->st_mode) || !(local->stbuf.st_blksize))
	  local->stbuf = *buf;
      } else if (!S_ISDIR (buf->st_mode)) {
	/* If file, then get the stat from storage node */
	local->stbuf = *buf;
      }
      if (local->st_nlink < buf->st_nlink)
	local->st_nlink = buf->st_nlink;
    }
  }
  UNLOCK (&frame->lock);

  if (!callcnt) {
    if (!local->stbuf.st_blksize) {
      /* Inode not present */
      local->op_ret = -1;
    } else {
      if (!local->revalidate) { 
	int16_t *list = NULL;
	if (!S_ISDIR (local->inode->st_mode)) {
	  /* If its a file, big array is useless, allocate the smaller one */
	  list = calloc (1, sizeof (int16_t) * (local->index + 1));
	  memcpy (list, local->list, sizeof (int16_t) * local->index);
	  /* Make the end of the list as -1 */
	  freee (local->list);
	  local->list = list;
	}
	local->list [local->index] = -1;
	/* Update the inode->ctx with the proper array */
	dict_set (local->inode->ctx, this->name, data_from_ptr (local->list));
      }
      if (S_ISDIR(local->inode->st_mode)) {
	/* lookup is done for directory */
	if (local->failed && priv->self_heal) {
	  local->inode->generation = 0; /*means, self-heal required for inode*/
	  priv->inode_generation++;
	}
      } else {
	local->stbuf.st_ino = local->st_ino;
      }

      local->stbuf.st_nlink = local->st_nlink;
    }
    if (local->op_ret == -1) {
      if (!local->revalidate && local->list)
	freee (local->list);
    }

    if ((local->op_ret >= 0) && local->failed && local->revalidate) {
      /* Done revalidate, but it failed */
      gf_log (this->name, GF_LOG_ERROR, 
	      "Revalidate failed for %s", local->path);
      local->op_ret = -1;
    }
    if ((priv->self_heal) && 
	((local->op_ret == 0) && S_ISDIR(local->inode->st_mode))) {
      /* Let the self heal be done here */
      gf_unify_self_heal (frame, this, local);
    } else {
      /* either no self heal, or op_ret == -1 (failure) */
      local->inode->generation = priv->inode_generation;
      unify_local_wipe (local);
      STACK_UNWIND (frame, local->op_ret, local->op_errno, 
		    local->inode, &local->stbuf, local->dict);
    }
  }

  return 0;
}

/**
 * unify_lookup - 
 */
int32_t 
unify_lookup (call_frame_t *frame,
	      xlator_t *this,
	      loc_t *loc,
	      int32_t need_xattr)
{
  unify_local_t *local = NULL;
  unify_private_t *priv = this->private;
  int16_t *list = NULL;
  int16_t index = 0;

  if (!(loc && loc->inode && loc->inode->ctx)) {
    gf_log (this->name, GF_LOG_ERROR, 
	    "%s: Argument not right", loc?loc->path:"(null)");
    STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
    return 0;
  }

  /* Initialization */
  INIT_LOCAL (frame, local);
  local->inode = loc->inode;
  local->path = strdup (loc->path);
  if (!local->path) {
    gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
    STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
    return 0;
  }

  if (dict_get (loc->inode->ctx, this->name))
    /* check if revalidate or fresh lookup */
    local->list = data_to_ptr (dict_get (loc->inode->ctx, this->name));
  
  if (local->list) {
    if (S_ISDIR (loc->inode->st_mode) && 	 
	(priv->self_heal && 
	 (priv->inode_generation > loc->inode->generation))) {
      gf_log (this->name, GF_LOG_ERROR,
	      "returning ESTALE for %s [translator generation (%d) inode generation (%d)]", 
	      loc->path, priv->inode_generation, loc->inode->generation);
      unify_local_wipe (local);
      STACK_UNWIND (frame, -1, ESTALE, NULL, NULL);
      return 0;
    } 
    if (!S_ISDIR (loc->inode->st_mode)) {
      for (index = 0; local->list[index] != -1; index++);
      if (index != 2) {
	gf_log (this->name, GF_LOG_ERROR,
		"returning ESTALE for %s: file count is %d", 
		loc->path, index);
	/* Print where all the file is present */
	for (index = 0; local->list[index] != -1; index++)
	  gf_log (this->name, GF_LOG_ERROR, "%s: found on %s",
		  loc->path, priv->xl_array[local->list[index]]->name);
	unify_local_wipe (local);
	STACK_UNWIND (frame, -1, ESTALE, NULL, NULL);
	return 0;
      }
    }

    /* is revalidate */
    list = local->list;
    local->revalidate = 1;

    for (index = 0; list[index] != -1; index++)
      local->call_count++;
    
    for (index = 0; list[index] != -1; index++) {
      char need_break = list[index+1] == -1;
      STACK_WIND_COOKIE (frame,
		   unify_lookup_cbk,
		   (void *)(long)list [index], //cookie
		   priv->xl_array [list [index]],
		   priv->xl_array [list [index]]->fops->lookup,
		   loc,
		   need_xattr);
      if (need_break)
	break;
    }
  } else {
    /* This is first call, there is no list */
    /* call count should be all child + 1 namespace */
    local->call_count = priv->child_count + 1;

    for (index = 0; index <= priv->child_count; index++) {
      STACK_WIND_COOKIE (frame,
		   unify_lookup_cbk,
		   (void *)(long)index, //cookie
		   priv->xl_array[index],
		   priv->xl_array[index]->fops->lookup,
		   loc,
		   need_xattr);
    }
  }

  return 0;
}


/**
 * unify_forget - call inode_forget which removes it from cache 
 */
int32_t 
unify_forget (call_frame_t *frame,
	      xlator_t *this,
	      inode_t *inode)
{
  /* in dictionary list is stored as pointer, so will be freed, when dictionary
   * is destroyed 
   */

  return 0;
}

/**
 * unify_stat - if directory, get the stat directly from NameSpace child.
 *     if file, check for a hint and send it only there (also to NS).
 *     if its a fresh stat, then do it on all the nodes.
 *
 * NOTE: for all the call, sending cookie as xlator pointer, which will be 
 *       used in cbk.
 */
int32_t
unify_stat (call_frame_t *frame,
	    xlator_t *this,
	    loc_t *loc)
{
  unify_local_t *local = NULL;
  unify_private_t *priv = this->private;
  int16_t index = 0;
  int16_t *list = NULL;

  UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);

  /* Initialization */
  INIT_LOCAL (frame, local);
  local->inode = loc->inode;
  local->path = strdup (loc->path);
  if (!local->path) {
    gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
    STACK_UNWIND (frame, -1, ENOMEM, NULL);
    return 0;
  }

  if (S_ISDIR (loc->inode->st_mode)) {
    /* Directory */
    local->call_count = 1;
    STACK_WIND (frame, unify_buf_cbk, NS(this),
		NS(this)->fops->stat, loc);
  } else {
    /* File */
    list = data_to_ptr (dict_get (loc->inode->ctx, this->name));
    
    for (index = 0; list[index] != -1; index++)
      local->call_count++;
    
    for (index = 0; list[index] != -1; index++) {
      char need_break = list[index+1] == -1;
      STACK_WIND (frame,
		  unify_buf_cbk,
		  priv->xl_array[list[index]],
		  priv->xl_array[list[index]]->fops->stat,
		  loc);
      if (need_break)
	break;
    }
  }

  return 0;
}

/**
 * unify_access_cbk -
 */
int32_t
unify_access_cbk (call_frame_t *frame,
		  void *cookie,
		  xlator_t *this,
		  int32_t op_ret,
		  int32_t op_errno)
{
  STACK_UNWIND (frame, op_ret, op_errno);
  return 0;
}


/**
 * unify_access - Send request to only namespace, which has all the 
 *      attributes set for the file.
 */
int32_t
unify_access (call_frame_t *frame,
	      xlator_t *this,
	      loc_t *loc,
	      int32_t mask)
{
  UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);

  STACK_WIND (frame,
	      unify_access_cbk,
	      NS(this),
	      NS(this)->fops->access,
	      loc,
	      mask);

  return 0;
}

int32_t
unify_mkdir_cbk (call_frame_t *frame,
		 void *cookie,
		 xlator_t *this,
		 int32_t op_ret,
		 int32_t op_errno,
		 inode_t *inode,
		 struct stat *buf)
{
  int32_t callcnt = 0;
  unify_private_t *priv = this->private;
  unify_local_t *local = frame->local;

  LOCK (&frame->lock);
  {
    callcnt = --local->call_count;
  
    if (op_ret == -1) {
      /* TODO: Decrement the inode_generation of this->inode's parent inode, hence 
       * the missing directory is created properly by self-heal. Currently, there is 
       * no way to get the parent inode directly.
       */
      gf_log (this->name, GF_LOG_ERROR,
	      "%s returned %d", priv->xl_array[(long)cookie]->name, op_errno);
      local->failed = 1;
    }
  
    if (op_ret >= 0) {
      local->op_ret = 0;
      /* This is to be used as hint from the inode and also mapping */
      local->list[local->index++] = (int16_t)(long)cookie;
    }
  }
  UNLOCK (&frame->lock);
  
  if (!callcnt) {
    unify_local_wipe (local);
    if (!local->failed)
      local->inode->generation = priv->inode_generation;
    if (local->op_ret >= 0) {
      local->list[local->index] = -1;
    }
    STACK_UNWIND (frame, local->op_ret, local->op_errno, 
		  local->inode, &local->stbuf);
  }

  return 0;
}

/**
 * unify_ns_mkdir_cbk -
 */
int32_t
unify_ns_mkdir_cbk (call_frame_t *frame,
		    void *cookie,
		    xlator_t *this,
		    int32_t op_ret,
		    int32_t op_errno,
		    inode_t *inode,
		    struct stat *buf)
{
  unify_private_t *priv = this->private;
  unify_local_t *local = frame->local;
  int16_t index = 0;

  if (op_ret == -1) {
    /* No need to send mkdir request to other servers, 
     * as namespace action failed 
     */
    gf_log (this->name, GF_LOG_ERROR,
	    "mkdir on namespace failed (%d)", op_errno);
    unify_local_wipe (local);
    STACK_UNWIND (frame, op_ret, op_errno, inode, NULL);
    return 0;
  }
  
  /* Create one inode for this entry */
  local->op_ret = 0;
  local->stbuf = *buf;

  local->inode = inode;

  local->list = calloc (1, sizeof (int16_t) * (priv->child_count + 2));
  if (!local->list) {
    gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
    unify_local_wipe (local);
    STACK_UNWIND (frame, -1, ENOMEM, inode, NULL);