posix.c

分布式文件系统

  op_errno = errno;
  if (op_ret == -1) {
    gf_log (this->name, GF_LOG_WARNING, "fchmod: %s", strerror (op_errno));
  }
  if (op_ret == 0)
    fstat (_fd, &buf);
  
  SET_TO_OLD_FS_UID ();

  frame->root->rsp_refs = NULL;  
  STACK_UNWIND (frame, op_ret, op_errno, &buf);

  return 0;
}

int32_t 
posix_setdents (call_frame_t *frame,
		xlator_t *this,
		fd_t *fd,
		int32_t flags,
		dir_entry_t *entries,
		int32_t count)
{
  char *real_path;
  char *entry_path;
  int32_t real_path_len;
  int32_t entry_path_len;
  int32_t ret = 0;
  struct posix_fd *pfd;
  data_t *pfd_data = NULL;
  struct timeval tv[2];

  tv[0].tv_sec = tv[0].tv_usec = 0;
  tv[1].tv_sec = tv[1].tv_usec = 0;

  frame->root->rsp_refs = NULL;

  pfd_data = dict_get (fd->ctx, this->name);
  if (!pfd_data) {
    gf_log (this->name, GF_LOG_ERROR, "fd->ctx not found on fd=%p for %s",
	    fd, this->name);
    STACK_UNWIND (frame, -1, EBADFD);
    return 0;
  }


  pfd = data_to_ptr (pfd_data);

  if (!pfd) {
    gf_log (this->name, GF_LOG_ERROR, "pfd is NULL on fd=%p", fd);
    STACK_UNWIND (frame, -1, EBADFD);
    return 0;
  }

  real_path = pfd->path;

  if (!real_path) {
    gf_log (this->name, GF_LOG_ERROR,
	    "path is NULL on pfd=%p fd=%p", pfd, fd);
    STACK_UNWIND (frame, -1, EBADFD);
    return 0;
  }

  real_path_len = strlen (real_path);
  entry_path_len = real_path_len + 256;
  entry_path = calloc (1, entry_path_len);

  if (!entry_path) {
    STACK_UNWIND (frame, -1, ENOMEM);
    return 0;
  }

  strcpy (entry_path, real_path);
  entry_path[real_path_len] = '/';

  /* fd exists, and everything looks fine */
  {
    /**
     * create an entry for each one present in '@entries' 
     *  - if flag is set (ie, if its namespace), create both directories and 
     *    files 
     *  - if not set, create only directories.
     *
     *  after the entry is created, change the mode and ownership of the entry
     *  according to the stat present in entries->buf.  
     */
    dir_entry_t *trav = entries->next;
    while (trav) {
      char pathname[4096] = {0,};
      strcpy (pathname, entry_path);
      strcat (pathname, trav->name);

      if (S_ISDIR(trav->buf.st_mode)) {
	/* If the entry is directory, create it by calling 'mkdir'. If 
	 * directory is not present, it will be created, if its present, 
	 * no worries even if it fails.
	 */
	ret = mkdir (pathname, trav->buf.st_mode);
	if (!ret) {
	  gf_log (this->name, 
		  GF_LOG_DEBUG, 
		  "Creating directory %s with mode (0%o)", 
		  pathname,
		  trav->buf.st_mode);
	}
      } else if (flags & GF_SET_IF_NOT_PRESENT || !(flags & GF_SET_DIR_ONLY)) {
	/* Create a 0byte file here */
	if (S_ISREG (trav->buf.st_mode)) {
	  ret = open (pathname, O_CREAT|O_EXCL, trav->buf.st_mode);
	  if (ret == -1) {
	    gf_log (this->name,
		    GF_LOG_ERROR,
		    "Error creating file %s with mode (0%o)",
		    pathname, 
		    trav->buf.st_mode);
	  } else {
	    close (ret);
	  }
	} else if (S_ISLNK(trav->buf.st_mode)) {
	  ret = symlink (trav->link, pathname);
	  if (ret == -1) {
	    gf_log (this->name, GF_LOG_ERROR,
		    "error creating symlink %s", pathname);
	  }
	} else if (S_ISBLK (trav->buf.st_mode) || 
		   S_ISCHR (trav->buf.st_mode) || 
		   S_ISFIFO (trav->buf.st_mode)) {
	  ret = mknod (pathname, trav->buf.st_mode, trav->buf.st_dev);
	  if (ret == -1) {
	    gf_log (this->name,
		    GF_LOG_ERROR,
		    "error creating device file %s",
		    pathname);
	  }
	}
      }
      /* TODO: handle another flag, GF_SET_OVERWRITE */

      /* Change the mode */
      chmod (pathname, trav->buf.st_mode);
      /* change the ownership */
      chown (pathname, trav->buf.st_uid, trav->buf.st_gid);
      if (flags & GF_SET_EPOCH_TIME)
	utimes (pathname, tv); /* FIXME check return value */

      /* consider the next entry */
      trav = trav->next;
    }
  }
  //  op_errno = errno;
  
  /* Return success all the time */
  frame->root->rsp_refs = NULL;
  STACK_UNWIND (frame, 0, 0);
  
  freee (entry_path);
  return 0;
}

int32_t 
posix_fstat (call_frame_t *frame,
	     xlator_t *this,
	     fd_t *fd)
{
  int32_t _fd;
  int32_t op_ret;
  int32_t op_errno;
  struct stat buf;
  data_t *pfd_data = dict_get (fd->ctx, this->name);
  struct posix_fd *pfd;
  DECLARE_OLD_FS_UID_VAR;

  frame->root->rsp_refs = NULL;

  if (pfd_data == NULL) {
    gf_log (this->name, GF_LOG_ERROR,
	    "fd=%p has no context", fd);
    STACK_UNWIND (frame, -1, EBADF);
    return 0;
  }

  pfd = data_to_ptr (pfd_data);

  if (!pfd) {
    gf_log (this->name, GF_LOG_ERROR,
	    "pfd is NULL fd=%p", fd);
    STACK_UNWIND (frame, -1, EBADF);
    return 0;
  }

  _fd = pfd->fd;

  SET_FS_UID (frame->root->uid, frame->root->gid);

  op_ret = fstat (_fd, &buf);
  op_errno = errno;
  if (op_ret == -1) {
    gf_log (this->name, GF_LOG_WARNING, "fstat: %s", strerror (op_errno));
  }

  SET_TO_OLD_FS_UID ();

  STACK_UNWIND (frame, op_ret, op_errno, &buf);
  return 0;
}


int32_t 
posix_lk (call_frame_t *frame,
	  xlator_t *this,
	  fd_t *fd,
	  int32_t cmd,
	  struct flock *lock)
{
  struct flock nullock = {0, };
  frame->root->rsp_refs = NULL;
  STACK_UNWIND (frame, -1, ENOSYS, &nullock);
  return 0;
}


#define ALIGN(x) (((x) + sizeof (uint64_t) - 1) & ~(sizeof (uint64_t) - 1))

static int32_t
dirent_size (struct dirent *entry)
{
#ifdef GF_DARWIN_HOST_OS
  return ALIGN (24 /* FIX MEEEE!!! */ + entry->d_namlen);
#else
  return ALIGN (24 /* FIX MEEEE!!! */ + entry->d_reclen);
#endif
}

int32_t
posix_readdir (call_frame_t *frame,
	       xlator_t *this,
	       fd_t *fd,
	       size_t size,
	       off_t off)
{
  data_t *pfd_data = dict_get (fd->ctx, this->name);
  struct posix_fd *pfd;
  DIR *dir = NULL;

  frame->root->rsp_refs = NULL;

  if (pfd_data == NULL) {
    gf_log (this->name, GF_LOG_ERROR,
	    "pfd_data is NULL from fd=%p", fd);
    STACK_UNWIND (frame, -1, EBADF, NULL);
    return 0;
  }

  pfd = data_to_ptr (pfd_data);

  if (!pfd) {
    gf_log (this->name, GF_LOG_ERROR,
	    "pfd is NULL for fd=%p", fd);
    STACK_UNWIND (frame, -1, EBADF,NULL);
    return 0;
  }

  dir = pfd->dir;

  if (!dir) {
    gf_log (this->name, GF_LOG_ERROR,
	    "dir is NULL for fd=%p", fd);
    STACK_UNWIND (frame, -1, EBADF, NULL);
    return 0;
  }

  {
    char *buf = calloc (size, 1); /* readdir buffer needs 0 padding */
    size_t filled = 0;

    if (!buf) {
      gf_log (this->name, GF_LOG_ERROR,
	      "malloc (%d) returned NULL", size);
      STACK_UNWIND (frame, -1, ENOMEM, NULL);
      return 0;
    }

    /* TODO: verify if offset is where fd is parked at */
    if (!off) {
      rewinddir (dir);
    } else {
      seekdir (dir, off);
    }

    while (filled <= size) {
      gf_dirent_t *this_entry;
      struct dirent *entry;
      off_t in_case;
      int32_t this_size;

      in_case = telldir (dir);
      entry = readdir (dir);
      if (!entry)
	break;

      this_size = dirent_size (entry);

      if (this_size + filled > size) {
	seekdir (dir, in_case);
	break;
      }

      /* TODO - consider endianness here */
      this_entry = (void *)(buf + filled);
      this_entry->d_ino = entry->d_ino;
      this_entry->d_len = entry->d_reclen;

      this_entry->d_off = telldir(dir);

#ifndef GF_SOLARIS_HOST_OS
      this_entry->d_type = entry->d_type;
#endif

#ifdef GF_DARWIN_HOST_OS
      /* d_reclen in Linux == d_namlen in Darwin */
      this_entry->d_len = entry->d_namlen; 
#endif

      strncpy (this_entry->d_name, entry->d_name, this_entry->d_len);

      filled += this_size;
    }

    STACK_UNWIND (frame, filled, 0, buf);
    free (buf);
  }

  return 0;
}


int32_t 
posix_stats (call_frame_t *frame,
	     xlator_t *this,
	     int32_t flags)

{
  int32_t op_ret = 0;
  int32_t op_errno = 0;

  struct xlator_stats xlstats = {0, }, *stats = &xlstats;
  struct statvfs buf;
  struct timeval tv;
  struct posix_private *priv = (struct posix_private *)this->private;
  int64_t avg_read = 0;
  int64_t avg_write = 0;
  int64_t _time_ms = 0; 
  DECLARE_OLD_FS_UID_VAR ;

  SET_FS_UID (frame->root->uid, frame->root->gid);
    
  op_ret = statvfs (priv->base_path, &buf);
  op_errno = errno;
    
  SET_TO_OLD_FS_UID ();
  
  stats->nr_files = priv->stats.nr_files;
  stats->nr_clients = priv->stats.nr_clients; /* client info is maintained at FSd */
  stats->free_disk = buf.f_bfree * buf.f_bsize; /* Number of Free block in the filesystem. */
  stats->total_disk_size = buf.f_blocks * buf.f_bsize; /* */
  stats->disk_usage = (buf.f_blocks - buf.f_bavail) * buf.f_bsize;

  /* Calculate read and write usage */
  gettimeofday (&tv, NULL);
  
  /* Read */
  _time_ms = (tv.tv_sec - priv->init_time.tv_sec) * 1000 +
             ((tv.tv_usec - priv->init_time.tv_usec) / 1000);

  avg_read = (_time_ms) ? (priv->read_value / _time_ms) : 0; /* KBps */
  avg_write = (_time_ms) ? (priv->write_value / _time_ms) : 0; /* KBps */
  
  _time_ms = (tv.tv_sec - priv->prev_fetch_time.tv_sec) * 1000 +
             ((tv.tv_usec - priv->prev_fetch_time.tv_usec) / 1000);
  if (_time_ms && ((priv->interval_read / _time_ms) > priv->max_read)) {
    priv->max_read = (priv->interval_read / _time_ms);
  }
  if (_time_ms && ((priv->interval_write / _time_ms) > priv->max_write)) {
    priv->max_write = priv->interval_write / _time_ms;
  }

  stats->read_usage = avg_read / priv->max_read;
  stats->write_usage = avg_write / priv->max_write;

  gettimeofday (&(priv->prev_fetch_time), NULL);
  priv->interval_read = 0;
  priv->interval_write = 0;

  frame->root->rsp_refs = NULL;
  STACK_UNWIND (frame, op_ret, op_errno, stats);
  return 0;
}

int32_t 
posix_checksum (call_frame_t *frame,
		xlator_t *this,
		loc_t *loc,
		int32_t flag)
{
  char *real_path;
  DIR *dir;
  struct dirent *dirent;
  uint8_t file_checksum[4096] = {0,};
  uint8_t dir_checksum[4096] = {0,};
  int32_t op_ret = -1;
  int32_t op_errno = 2;
  int32_t i, length = 0;
  MAKE_REAL_PATH (real_path, this, loc->path);

  dir = opendir (real_path);
  
  if (!dir){
    gf_log (this->name, GF_LOG_DEBUG, 
	    "checksum: opendir() failed for `%s'", real_path);
    frame->root->rsp_refs = NULL;
    STACK_UNWIND (frame, -1, errno, NULL, NULL);
    return 0;
  } else {
    op_ret = 0;
    op_errno = 0;
  }
  
  while ((dirent = readdir (dir))) {
    struct stat buf;
    char tmp_real_path[4096];
    int ret;

    if (!dirent)
      break;

    length = strlen (dirent->d_name);

    strcpy(tmp_real_path, real_path);
    strcat (tmp_real_path, "/");
    strcat(tmp_real_path, dirent->d_name);
    ret = lstat (tmp_real_path, &buf);

    if (ret == -1)
      continue;

    if (S_ISDIR (buf.st_mode)) {
      for (i = 0; i < length; i++)
	dir_checksum[i] ^= dirent->d_name[i];
    } else {
      for (i = 0; i < length; i++)
	file_checksum[i] ^= dirent->d_name[i];
    }
  }
  closedir (dir);

  frame->root->rsp_refs = NULL;
  STACK_UNWIND (frame, op_ret, op_errno, file_checksum, dir_checksum);

  return 0;
}

/**
 * notify - when parent sends PARENT_UP, send CHILD_UP event from here
 */
int32_t
notify (xlator_t *this,
        int32_t event,
        void *data,
        ...)
{
  switch (event)
    {
    case GF_EVENT_PARENT_UP:
      {
	/* Tell the parent that posix xlator is up */
	default_notify (this, GF_EVENT_CHILD_UP, data);
      }
      break;
    default:
      /* */
      break;
    }
  return 0;
}

/**
 * init - 
 */
int32_t 
init (xlator_t *this)
{
  int32_t ret;
  struct stat buf;
  struct rlimit lim;
  struct posix_private *_private = calloc (1, sizeof (*_private));
  data_t *data = dict_get (this->options, "directory");

  if (this->children) {
    gf_log (this->name,
	    GF_LOG_ERROR,
	    "FATAL: storage/posix cannot have subvolumes");
    return -1;
  }

  if (!data) {
    gf_log (this->name, GF_LOG_ERROR,
	    "export directory not specified in spec file");
    return -1;
  }
  umask (000); // umask `masking' is done at the client side
  if (mkdir (data->data, 0777) == 0) {
    gf_log (this->name, GF_LOG_WARNING,
	    "directory specified not exists, created");
  }
  /* Check whether the specified directory exists, if not create it. */
  ret = stat (data->data, &buf);
  if (ret != 0 && !S_ISDIR (buf.st_mode)) {
    gf_log (this->name, GF_LOG_ERROR, 
	    "Specified directory doesn't exists, Exiting");
    return -1;
  }
  _private->base_path = strdup (data->data);
  _private->base_path_length = strlen (_private->base_path);

  {
    /* Stats related variables */
    gettimeofday (&_private->init_time, NULL);
    gettimeofday (&_private->prev_fetch_time, NULL);
    _private->max_read = 1;
    _private->max_write = 1;
  }

  _private->export_statfs = 1;
  data = dict_get (this->options, "export-statfs-size");
  if (data) {
    if (!strcasecmp ("no", data->data)) {
      gf_log (this->name, GF_LOG_DEBUG, "'statfs()' returns dummy size");
      _private->export_statfs = 0;
    }
  }

  lim.rlim_cur = 1048576;
  lim.rlim_max = 1048576;

#ifndef GF_DARWIN_HOST_OS
  if (setrlimit (RLIMIT_NOFILE, &lim) == -1) {
    gf_log (this->name, GF_LOG_WARNING, "WARNING: Failed to set 'ulimit -n 1048576': %s",
	    strerror(errno));
    lim.rlim_cur = 65536;
    lim.rlim_max = 65536;
  
    if (setrlimit (RLIMIT_NOFILE, &lim) == -1) {
      gf_log (this->name, GF_LOG_ERROR, "Failed to set max open fd to 64k: %s", strerror(errno));
    } else {
      gf_log (this->name, GF_LOG_ERROR, "max open fd set to 64k");
    }
  }
#endif

  this->private = (void *)_private;
  return 0;
}

void
fini (xlator_t *this)
{
  struct posix_private *priv = this->private;
  freee (priv);
  return;
}

struct xlator_mops mops = {
  .stats    = posix_stats,
  .lock     = mop_lock_impl,
  .unlock   = mop_unlock_impl,
  .checksum = posix_checksum,
};

struct xlator_fops fops = {
  .lookup      = posix_lookup,
  .forget      = posix_forget,
  .stat        = posix_stat,
  .opendir     = posix_opendir,
  .readdir     = posix_readdir,
  .closedir    = posix_closedir,
  .readlink    = posix_readlink,
  .mknod       = posix_mknod,
  .mkdir       = posix_mkdir,
  .unlink      = posix_unlink,
  .rmelem      = posix_rmelem,
  .rmdir       = posix_rmdir,
  .symlink     = posix_symlink,
  .rename      = posix_rename,
  .link        = posix_link,
  .chmod       = posix_chmod,
  .chown       = posix_chown,
  .truncate    = posix_truncate,
  .utimens     = posix_utimens,
  .create      = posix_create,
  .open        = posix_open,
  .readv       = posix_readv,
  .writev      = posix_writev,
  .statfs      = posix_statfs,
  .flush       = posix_flush,
  .close       = posix_close,
  .fsync       = posix_fsync,
  .incver      = posix_incver,
  .setxattr    = posix_setxattr,
  .getxattr    = posix_getxattr,
  .removexattr = posix_removexattr,
  .fsyncdir    = posix_fsyncdir,
  .access      = posix_access,
  .ftruncate   = posix_ftruncate,
  .fstat       = posix_fstat,
  .lk          = posix_lk,
  .fchown      = posix_fchown,
  .fchmod      = posix_fchmod,
  .setdents    = posix_setdents,
  .getdents    = posix_getdents,
};