sfsrodb.c

chord 源码 http://pdos.csail.mit.edu/chord/

  inode->reg->size = inode->reg->used = calllen;
  inode->reg->direct.setsize (1);
  inode->reg->direct[0] = *fh;

  //  strbuf sb;
  //rpc_print (sb, *inode, 5, NULL, " ");
  //warn << "storedirectory " << sb << "\n";


  return;
}


static int
compare_name (const void *file1, const void *file2)
{
  return (strcmp (*(static_cast < char * const*>(file1)),
		  *(static_cast < char * const*>(file2))));
}


void
sort_dir (const str path, vec < char *>&file_list)
{
  DIR *dirp;
  struct dirent *de = NULL;
  char *filename;

  if ((dirp = opendir (path)) == 0) {
    warn << path << " is not a directory\n";
    return;
  }

  while ((de = readdir (dirp))) {
    // XXX memory leak!!!!
    filename = New char[strlen (de->d_name) + 1];
    memcpy (filename, de->d_name, strlen (de->d_name) + 1);
    file_list.push_back (filename);
  }

  //
  // XXX this might be broken.... --josh
  //
  qsort (file_list.base (), file_list.size (),
	 sizeof (char *), compare_name);

  if (closedir (dirp) < 0) {
    warn << "closedir failed: " << path << ": " << strerror (errno) << "\n";
    exit (0);
  }

}

/* pseudo-destructor */
void
twiddle_sort_dir (vec < char *>&file_list)
{
  for (unsigned int i = 0; i < file_list.size (); i++) {
    delete file_list[i];
  }
}

// XXX keep in sync with same function in chordcd/server.C
static uint64
path2fileid(const str path)
{
  char buf[sha1::hashsize];
  bzero(buf, sha1::hashsize);
  sha1_hash (buf, path.cstr (), path.len ());
  return gethyper (buf); // XXX not using all 160 bits of the hash
}


static void
splitpath (vec<str> &out, str in)
{
  const char *p = in.cstr ();
  const char *e = p + in.len ();
  const char *n;

  for (;;) {
    while (*p == '/')
      p++;
    for (n = p; n < e && *n != '/'; n++)
      ;
    if (n == p)
      return;
    out.push_back (str (p, n - p));
    p = n;
  }
    
}

/* Given a path, split it into two pieces:
   the parent directory path and the filename.

   Examples:

   path      parent   filename
   "/"       "/"      ""
   "/a"      "/"      "a"
   "/a/"     "/"      "a"
   "/a/b"    "/a"     "b"
   "/a/b/c"  "/a/b"   "c"
*/
static void
parentpath (str &parent, str &filename, const str inpath)
{
  vec<str> ppv;
  parent = str ("/");
  filename = str ("");

  splitpath (ppv, inpath);

  if (ppv.size () == 0)
    return;

  filename = ppv.pop_back ();
  if (ppv.size () == 0)
    return;

  // What a non-intuitive way to do concatenation!
  parent = strbuf () << "/" << join (str("/"), ppv);
}


static str
parentpath (const str fullpath)
{
  str p;
  str c;
  parentpath (p, c, fullpath);
  return p;
}



/*
   Given: Path to file (any kind), an allocated fh, the inode
   number of the path's parent

   Path is like '/','/a','/a/b' (ie only the root ends in slash)

   Return: The hash and IV in the fh.

   Effects: Recursively hash everything beneath a directory
   It computes the cryptographic file handle for the
   given path, inserts the mapping from the fh to its data

 */

str timestamp = "";

int
recurse_path (const str path, sfs_hash * fh)
{
  str fspath = strbuf () << "/" << timestamp << substr (path, relpathlen);

  struct stat st;
  if (lstat (path, &st) < 0)
    fatal << path << ": " << strerror(errno) << "\n";
  
  sfsro_inode inode;

  if (S_ISLNK (st.st_mode)) {
    
    char *buf = New char[st.st_size + 1];
    int nchars = readlink (path, buf, st.st_size);

    if (nchars > MAXPATHLEN) {
      warn << "symlink target too large "
	<< path << " " << nchars << "\n";
      return -1;
    }

    sfsrodb_setinode (&st, fspath, &inode);
    inode.lnk->dest = nfspath3 (buf, nchars);

    delete[] buf;

  } else if (S_ISREG (st.st_mode)) {
    sfsrodb_setinode (&st, fspath, &inode);
    store_file (&inode, path);
  }
  else if (S_ISDIR (st.st_mode)) {
    sfsrodb_setinode (&st, fspath, &inode);
    vec < char *>file_list;
    sort_dir (path, file_list);
    sfsro_data directory (SFSRO_DIRBLK);
    directory.dir->eof = true;
    rpc_ptr < sfsro_dirent > *direntp = &directory.dir->entries;

    for (unsigned int i = 0; i < file_list.size (); i++) {
      (*direntp).alloc ();
      (*direntp)->name = file_list[i];

      if ((*direntp)->name.cmp (".") == 0) 
	(*direntp)->fileid = path2fileid(path);
      else if ((*direntp)->name.cmp ("..") == 0)
	(*direntp)->fileid = path2fileid(parentpath(path));
      else {
	str childpath = path << "/" << (*direntp)->name;
	(*direntp)->fileid = path2fileid(childpath);
	recurse_path (childpath, &(*direntp)->fh);
      }
      direntp = &(*direntp)->nextentry;
    }
 
    twiddle_sort_dir (file_list);

    store_directory (&inode, fh, &directory);
  }
  else {
    warn << "Not symlink, reg file, or directory " << path << "\n";
    return -1;
  }

  store_inode (&inode, fh);

  return 0;

}



bool
get_root_inode_and_dir (bigint rootkey, sfsro_inode *rootino, sfsro_data *rootdir)
{
  // fetch the filesystem root.
  //  
  ptr<sfsro_data> root_dat = sfsrodb_get (rootkey, DHASH_KEYHASH);
  if (!root_dat) {
    warn << "root block was null\n";
    return false;
  } else if (root_dat->type != CFS_FSINFO) {
    warn << "root block has wrong type: " << root_dat->type << "\n";
    return false;
  } 


  // fetch the root inode
  //
  ptr<sfsro_data> rootino_dat = sfsrodb_get (root_dat->fsinfo->rootfh, DHASH_CONTENTHASH);

  if (!rootino_dat) {
    warn << "root inode block was null\n";
    return false;
  } else if (rootino_dat->type != SFSRO_INODE) {
    warn << "(1) bad type on inode block: " << rootino_dat->type << "\n";
    return false;
  } else if (rootino_dat->inode->type != SFSRODIR) {
    warn << "(2) bad type on inode block: " << rootino_dat->inode->type << "\n";
    return false;
  } else if (rootino_dat->inode->reg->direct.size () != 1) {
    warn << "root inode, expected only one direct block: " <<
      rootino_dat->inode->reg->direct.size () << "\n";
    return false;
  }


  // fetch the root directory
  //
  bigint d0;
  sfs_hash *tmp = &rootino_dat->inode->reg->direct[0];
  mpz_set_rawmag_be(&d0, tmp->base (), tmp->size ());  // For big endian
  ptr<sfsro_data> rootdir_dat = sfsrodb_get (d0, DHASH_CONTENTHASH);
  if (!rootdir_dat) {
    warn << "root directory was null\n";
    return false;
  } else if (rootdir_dat->type != SFSRO_DIRBLK) {
    warn << "root directory bad type: " << rootdir_dat->type << "\n";
    return false;
  } 


  *rootino = *rootino_dat->inode;
  *rootdir = *rootdir_dat;

  return true;
}




int
sfsrodb_main (const str root, const str keyfile)
{
  time_t start = time (NULL);
  str stime (ctime (&start));
  if (x_flag) {
    timestamp = substr (stime, 0, stime.len () - 1); // trim newline
    //timestamp = "SHIT";
  }

  dhash_cli = New refcounted <dhashclient> (lsd_socket);

  ptr <sfspriv> sk;
  if (!keyfile) {
    warn << "cannot locate default file sfs_host_key\n";
    fatal ("errors!\n");
  }
  else {
    str key = file2wstr (keyfile);
    if (!key) {
      warn << keyfile << ": " << strerror (errno) << "\n";
      fatal ("errors!\n");
    }
    else if (!(sk = sfscrypt.alloc_priv (key, SFS_DECRYPT | SFS_SIGN))) {
      warn << "could not decode " << keyfile << "\n";
      warn << key << "\n";
      fatal ("errors!\n");
    }
  }

  strbuf b;
  if (!sk->export_pubkey (b, false)) {
    warn << "could not set public key: " << keyfile << "\n";
    fatal ("errors!\n");
  }
  str pk_raw = b;
  bigint pk_hash_bi = compute_hash (pk_raw.cstr (), pk_raw.len ());

  // store file system in db
  sfs_hash root_fh;

  relpathlen = root.len ();
  recurse_path (root, &root_fh);

  // for the sake of debugging output, wait for all puts to finish...
  while (out) acheck (); 

  if (x_flag) {
    // -------------------------------------------------
    sfsro_inode inode (SFSRODIR);
    sfsro_data directory (SFSRO_DIRBLK);
    
    rpc_ptr < sfsro_dirent > *direntp = &directory.dir->entries;

    bool found = false;
    if (!wipe_flag)
      found = get_root_inode_and_dir (pk_hash_bi, &inode, &directory);
    // XXX is something wrong with pk_hash_bi???
    if (wipe_flag || !found) {
      // fake up an empty directory
      directory.dir->eof = true;
      
      (*direntp).alloc ();
      (*direntp)->name = ".";
      (*direntp)->fileid = path2fileid ("/");
      direntp = &(*direntp)->nextentry;
      
      (*direntp).alloc ();
      (*direntp)->name = "..";
      (*direntp)->fileid = path2fileid ("/");
      direntp = &(*direntp)->nextentry;
    }
    
    // skip to the end
    while (*direntp)
      direntp = &(*direntp)->nextentry;
    
    // append new entry
    (*direntp).alloc ();
    (*direntp)->name = timestamp;
    (*direntp)->fileid = path2fileid (strbuf() << "/" << timestamp);
    (*direntp)->fh = root_fh;
    direntp = &(*direntp)->nextentry;

    sfs_hash junk;
    store_directory (&inode, &junk, &directory);
    store_inode (&inode, &root_fh);

#if 0
    rpc_ptr < sfsro_dirent > *dp = &directory.dir->entries;
    while (*dp) {
      warn << ">>> " << (*dp)->name << "\n";
      dp = &(*dp)->nextentry;
    }
#endif
  }


  // -------------------------------------------------

  sfsro_data dat (CFS_FSINFO);
  dat.fsinfo->start = start;
  dat.fsinfo->duration = sfsro_duration;
  dat.fsinfo->rootfh = fh2mpz (root_fh.base (), root_fh.size ());
  dat.fsinfo->blocksize = blocksize;
  time_t end = dat.fsinfo->start + dat.fsinfo->duration;

  // XX Should make sure timezone is correct
  str etime (ctime (&end));
  warn << "Database good from: \n " << stime << "until:\n " << etime;

  xdrsuio x (XDR_ENCODE);
  if (xdr_sfsro_data (x.xdrp (), &dat)) {
    void *v = suio_flatten (x.uio ());
    int l =  x.uio ()->resid ();
    chordID ID = sfsrodb_put (sk, v, l);
    warn << "exported file system under " << ID << "\n";
    xfree (v);
  }

  if (verbose_mode) {
    warn << "identical blocks:   " << identical_block << "\n";
    warn << "identical sindirs:  " << identical_sindir << "\n";
    warn << "identical dindirs:  " << identical_dindir << "\n";
    warn << "identical tindirs:  " << identical_tindir << "\n";
    warn << "identical dirs:     " << identical_dir << "\n";
    warn << "identical inodes:   " << identical_inode << "\n";
    warn << "identical symlinks: " << identical_sym << "\n";

    warn << "Database contents:\n";
    warn << "Regular inodes:      " << reginode_cnt << "\n";
    warn << "Symlink inodes:      " << lnkinode_cnt << "\n";
    warn << "Directory blocks     " << directory_cnt << "\n";
    warn << "File data blocks:    " << filedatablk_cnt << "\n";
    warn << "Single indir blocks: " << sindir_cnt << "\n";
    warn << "Double indir blocks: " << dindir_cnt << "\n";
    warn << "Triple indir blocks: " << tindir_cnt << "\n";

    warn << "identical fh's overall : " << identical_fh << "\n";
    warn << "unique fh's overall    : " << fh_cnt << "\n\n\n";
  }
  
  while (out) acheck ();
  return 0;
}


static void
usage ()
{
  warnx << "usage: " << progname
	<< " -d <export directory> -s <SK keyfile> -o <dbfile>\n";
  warnx << "              [-i] [-h <hostname for db>] [-v] [-b <blocksize>]\n";
  warnx << "-d <export directory> : The directory hierarchy to export\n";
  warnx << "-s <SK keyfile>       : Path to the secret key file\n";
  warnx << "Optional directives:\n";
  warnx << "-v                    : Verbose debugging output\n";
  warnx << "-b <blocksize>        : Page size of underlying database\n";
  warnx << "Backup directives:\n";
  warnx << "-x                    : do backup\n";
  warnx << "-w                    : wipe the root directory clean\n";
   exit (1);
}


int
main (int argc, char **argv)
{
  setprogname (argv[0]);

  char *exp_dir = NULL;
  char *sk_file = NULL;
  
  verbose_mode = false;
  initialize = false;
  blocksize = 8192;
  lsd_socket = "/tmp/chord-sock";

  int ch;
  while ((ch = getopt (argc, argv, "b:d:s:S:vwx")) != -1)
    switch (ch) {
    case 'b':
      if (!convertint (optarg, &blocksize)
	  || blocksize < 512 || blocksize > 0x400000)
	usage ();
      break;
    case 'd':
      exp_dir = optarg;
      break;
    case 's':
      sk_file = optarg;
      break;
    case 'v':
      verbose_mode = true;
      break;
    case 'w':
      wipe_flag = true;
      break;
    case 'x':
      x_flag = true;
      break;
    case 'S':
      lsd_socket = optarg;
      break;
    case '?':
    default:
      usage ();
    }
  argc -= optind;
  argv += optind;
  nfh = (blocksize - 100) / (sha1::hashsize*2);

  if ( (argc > 0) || !exp_dir || !sk_file )
    usage ();

  if (wipe_flag && !x_flag) {
    warn << "-w but not -x??\n";
    usage ();
  }

  if (verbose_mode) {
    warnx << "export directory : " << exp_dir << "\n";
    warnx << "SK keyfile       : " << sk_file << "\n";
  }



  sigcb(SIGUSR1, wrap (&sfsrodb_core_sigusr1));

  return (sfsrodb_main (exp_dir, sk_file));
}