tcfdb.c

Tokyo Cabinet的Tokyo Cabinet 是一个DBM的实现。这里的数据库由一系列key-value对的记录构成。key和value都可以是任意长度的字节序列,既可以是二进制也可以是字符

/*************************************************************************************************
 * private features
 *************************************************************************************************/


/* Serialize meta data into a buffer.
   `fdb' specifies the fixed-length database object.
   `hbuf' specifies the buffer. */
static void tcfdbdumpmeta(TCFDB *fdb, char *hbuf){
  memset(hbuf, 0, FDBHEADSIZ);
  sprintf(hbuf, "%s\n%s:%d\n", FDBMAGICDATA, _TC_FORMATVER, _TC_LIBVER);
  memcpy(hbuf + FDBTYPEOFF, &(fdb->type), sizeof(fdb->type));
  memcpy(hbuf + FDBFLAGSOFF, &(fdb->flags), sizeof(fdb->flags));
  uint64_t llnum;
  llnum = fdb->rnum;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + FDBRNUMOFF, &llnum, sizeof(llnum));
  llnum = fdb->fsiz;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + FDBFSIZOFF, &llnum, sizeof(llnum));
  llnum = fdb->width;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + FDBWIDTHOFF, &llnum, sizeof(llnum));
  llnum = fdb->limsiz;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + FDBLIMSIZOFF, &llnum, sizeof(llnum));
  llnum = fdb->min;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + FDBMINOFF, &llnum, sizeof(llnum));
  llnum = fdb->max;
  llnum = TCHTOILL(llnum);
  memcpy(hbuf + FDBMAXOFF, &llnum, sizeof(llnum));
}


/* Deserialize meta data from a buffer.
   `fdb' specifies the fixed-length database object.
   `hbuf' specifies the buffer. */
static void tcfdbloadmeta(TCFDB *fdb, const char *hbuf){
  memcpy(&(fdb->type), hbuf + FDBTYPEOFF, sizeof(fdb->type));
  memcpy(&(fdb->flags), hbuf + FDBFLAGSOFF, sizeof(fdb->flags));
  uint64_t llnum;
  memcpy(&llnum, hbuf + FDBRNUMOFF, sizeof(llnum));
  fdb->rnum = TCITOHLL(llnum);
  memcpy(&llnum, hbuf + FDBFSIZOFF, sizeof(llnum));
  fdb->fsiz = TCITOHLL(llnum);
  memcpy(&llnum, hbuf + FDBWIDTHOFF, sizeof(llnum));
  fdb->width = TCITOHLL(llnum);
  memcpy(&llnum, hbuf + FDBLIMSIZOFF, sizeof(llnum));
  fdb->limsiz = TCITOHLL(llnum);
  memcpy(&llnum, hbuf + FDBMINOFF, sizeof(llnum));
  fdb->min = TCITOHLL(llnum);
  memcpy(&llnum, hbuf + FDBMAXOFF, sizeof(llnum));
  fdb->max = TCITOHLL(llnum);
}


/* Clear all members.
   `fdb' specifies the fixed-length database object. */
static void tcfdbclear(TCFDB *fdb){
  assert(fdb);
  fdb->mmtx = NULL;
  fdb->amtx = NULL;
  fdb->rmtxs = NULL;
  fdb->tmtx = NULL;
  fdb->wmtx = NULL;
  fdb->eckey = NULL;
  fdb->type = TCDBTFIXED;
  fdb->flags = 0;
  fdb->width = FDBDEFWIDTH;
  fdb->limsiz = FDBDEFLIMSIZ;
  fdb->wsiz = 0;
  fdb->rsiz = 0;
  fdb->limid = 0;
  fdb->path = NULL;
  fdb->fd = -1;
  fdb->omode = 0;
  fdb->rnum = 0;
  fdb->fsiz = 0;
  fdb->min = 0;
  fdb->max = 0;
  fdb->iter = 0;
  fdb->map = NULL;
  fdb->array = NULL;
  fdb->ecode = TCESUCCESS;
  fdb->fatal = false;
  fdb->inode = 0;
  fdb->mtime = 0;
  fdb->tran = false;
  fdb->walfd = -1;
  fdb->walend = 0;
  fdb->dbgfd = -1;
  fdb->cnt_writerec = -1;
  fdb->cnt_readrec = -1;
  fdb->cnt_truncfile = -1;
  TCDODEBUG(fdb->cnt_writerec = 0);
  TCDODEBUG(fdb->cnt_readrec = 0);
  TCDODEBUG(fdb->cnt_truncfile = 0);
}


/* Set the open flag.
   `fdb' specifies the fixed-length database object.
   `flag' specifies the flag value.
   `sign' specifies the sign. */
static void tcfdbsetflag(TCFDB *fdb, int flag, bool sign){
  assert(fdb);
  char *fp = (char *)fdb->map + FDBFLAGSOFF;
  if(sign){
    *fp |= (uint8_t)flag;
  } else {
    *fp &= ~(uint8_t)flag;
  }
  fdb->flags = *fp;
}


/* Initialize the write ahead logging file.
   `fdb' specifies the fixed-length database object.
   If successful, the return value is true, else, it is false. */
static bool tcfdbwalinit(TCFDB *fdb){
  assert(fdb);
  if(lseek(fdb->walfd, 0, SEEK_SET) == -1){
    tcfdbsetecode(fdb, TCESEEK, __FILE__, __LINE__, __func__);
    return false;
  }
  if(ftruncate(fdb->walfd, 0) == -1){
    tcfdbsetecode(fdb, TCETRUNC, __FILE__, __LINE__, __func__);
    return false;
  }
  uint64_t llnum = fdb->fsiz;
  llnum = TCHTOILL(llnum);
  if(!tcwrite(fdb->walfd, &llnum, sizeof(llnum))){
    tcfdbsetecode(fdb, TCEWRITE, __FILE__, __LINE__, __func__);
    return false;
  }
  fdb->walend = fdb->fsiz;
  if(!tcfdbwalwrite(fdb, 0, FDBHEADSIZ)) return false;
  return true;
}


/* Write an event into the write ahead logging file.
   `fdb' specifies the fixed-length database object.
   `off' specifies the offset of the region to be updated.
   `size' specifies the size of the region.
   If successful, the return value is true, else, it is false. */
static bool tcfdbwalwrite(TCFDB *fdb, uint64_t off, int64_t size){
  assert(fdb && off >= 0 && size >= 0);
  if(off + size > fdb->walend) size = fdb->walend - off;
  if(size < 1) return true;
  char stack[FDBIOBUFSIZ];
  char *buf;
  if(size + sizeof(off) + sizeof(size) <= FDBIOBUFSIZ){
    buf = stack;
  } else {
    TCMALLOC(buf, size + sizeof(off) + sizeof(size));
  }
  char *wp = buf;
  uint64_t llnum = TCHTOILL(off);
  memcpy(wp, &llnum, sizeof(llnum));
  wp += sizeof(llnum);
  uint32_t lnum = TCHTOIL(size);
  memcpy(wp, &lnum, sizeof(lnum));
  wp += sizeof(lnum);
  memcpy(wp, fdb->map + off, size);
  wp += size;
  if(!FDBLOCKWAL(fdb)) return false;
  if(!tcwrite(fdb->walfd, buf, wp - buf)){
    tcfdbsetecode(fdb, TCEWRITE, __FILE__, __LINE__, __func__);
    if(buf != stack) TCFREE(buf);
    FDBUNLOCKWAL(fdb);
    return false;
  }
  if(buf != stack) TCFREE(buf);
  if((fdb->omode & FDBOTSYNC) && fsync(fdb->walfd) == -1){
    tcfdbsetecode(fdb, TCESYNC, __FILE__, __LINE__, __func__);
    FDBUNLOCKWAL(fdb);
    return false;
  }
  FDBUNLOCKWAL(fdb);
  return true;
}


/* Restore the database from the write ahead logging file.
   `fdb' specifies the fixed-length database object.
   `path' specifies the path of the database file.
   If successful, the return value is true, else, it is false. */
static int tcfdbwalrestore(TCFDB *fdb, const char *path){
  assert(fdb && path);
  char *tpath = tcsprintf("%s%c%s", path, MYEXTCHR, FDBWALSUFFIX);
  int walfd = open(tpath, O_RDONLY, FDBFILEMODE);
  TCFREE(tpath);
  if(walfd < 0) return false;
  bool err = false;
  uint64_t walsiz = 0;
  struct stat sbuf;
  if(fstat(walfd, &sbuf) == 0){
    walsiz = sbuf.st_size;
  } else {
    tcfdbsetecode(fdb, TCESTAT, __FILE__, __LINE__, __func__);
    err = true;
  }
  if(walsiz >= sizeof(walsiz) + FDBHEADSIZ){
    int dbfd = fdb->fd;
    int tfd = -1;
    if(!(fdb->omode & FDBOWRITER)){
      tfd = open(path, O_WRONLY, FDBFILEMODE);
      if(tfd >= 0){
        dbfd = tfd;
      } else {
        int ecode = TCEOPEN;
        switch(errno){
        case EACCES: ecode = TCENOPERM; break;
        case ENOENT: ecode = TCENOFILE; break;
        }
        tcfdbsetecode(fdb, ecode, __FILE__, __LINE__, __func__);
        err = true;
      }
    }
    uint64_t fsiz = 0;
    if(tcread(walfd, &fsiz, sizeof(fsiz))){
      fsiz = TCITOHLL(fsiz);
    } else {
      tcfdbsetecode(fdb, TCEREAD, __FILE__, __LINE__, __func__);
      err = true;
    }
    TCLIST *list = tclistnew();
    uint64_t waloff = sizeof(fsiz);
    char stack[FDBIOBUFSIZ];
    while(waloff < walsiz){
      uint64_t off;
      uint32_t size;
      if(!tcread(walfd, stack, sizeof(off) + sizeof(size))){
        tcfdbsetecode(fdb, TCEREAD, __FILE__, __LINE__, __func__);
        err = true;
        break;
      }
      memcpy(&off, stack, sizeof(off));
      off = TCITOHLL(off);
      memcpy(&size, stack + sizeof(off), sizeof(size));
      size = TCITOHL(size);
      char *buf;
      if(sizeof(off) + size <= FDBIOBUFSIZ){
        buf = stack;
      } else {
        TCMALLOC(buf, sizeof(off) + size);
      }
      *(uint64_t *)buf = off;
      if(!tcread(walfd, buf + sizeof(off), size)){
        tcfdbsetecode(fdb, TCEREAD, __FILE__, __LINE__, __func__);
        err = true;
        if(buf != stack) TCFREE(buf);
        break;
      }
      TCLISTPUSH(list, buf, sizeof(off) + size);
      if(buf != stack) TCFREE(buf);
      waloff += sizeof(off) + sizeof(size) + size;
    }
    for(int i = TCLISTNUM(list) - 1; i >= 0; i--){
      const char *rec;
      int size;
      TCLISTVAL(rec, list, i, size);
      uint64_t off = *(uint64_t *)rec;
      rec += sizeof(off);
      size -= sizeof(off);
      if(lseek(dbfd, off, SEEK_SET) == -1){
        tcfdbsetecode(fdb, TCESEEK, __FILE__, __LINE__, __func__);
        err = true;
        break;
      }
      if(!tcwrite(dbfd, rec, size)){
        tcfdbsetecode(fdb, TCEWRITE, __FILE__, __LINE__, __func__);
        err = true;
        break;
      }
    }
    tclistdel(list);
    if(ftruncate(dbfd, fsiz) == -1){
      tcfdbsetecode(fdb, TCETRUNC, __FILE__, __LINE__, __func__);
      err = true;
    }
    if((fdb->omode & FDBOTSYNC) && fsync(dbfd) == -1){
      tcfdbsetecode(fdb, TCESYNC, __FILE__, __LINE__, __func__);
      err = true;
    }
    if(tfd >= 0 && close(tfd) == -1){
      tcfdbsetecode(fdb, TCECLOSE, __FILE__, __LINE__, __func__);
      err = true;
    }
  } else {
    err = true;
  }
  if(close(walfd) == -1){
    tcfdbsetecode(fdb, TCECLOSE, __FILE__, __LINE__, __func__);
    err = true;
  }
  return !err;
}


/* Remove the write ahead logging file.
   `fdb' specifies the fixed-length database object.
   `path' specifies the path of the database file.
   If successful, the return value is true, else, it is false. */
static bool tcfdbwalremove(TCFDB *fdb, const char *path){
  assert(fdb && path);
  char *tpath = tcsprintf("%s%c%s", path, MYEXTCHR, FDBWALSUFFIX);
  bool err = false;
  if(unlink(tpath) == -1 && errno != ENOENT){
    tcfdbsetecode(fdb, TCEUNLINK, __FILE__, __LINE__, __func__);
    err = true;
  }
  TCFREE(tpath);
  return !err;
}


/* Open a database file and connect a fixed-length database object.
   `fdb' specifies the fixed-length database object.
   `path' specifies the path of the database file.
   `omode' specifies the connection mode.
   If successful, the return value is true, else, it is false. */
static bool tcfdbopenimpl(TCFDB *fdb, const char *path, int omode){
  assert(fdb && path);
  int mode = O_RDONLY;
  if(omode & FDBOWRITER){
    mode = O_RDWR;
    if(omode & FDBOCREAT) mode |= O_CREAT;
  }
  int fd = open(path, mode, FDBFILEMODE);
  if(fd < 0){
    int ecode = TCEOPEN;
    switch(errno){
    case EACCES: ecode = TCENOPERM; break;
    case ENOENT: ecode = TCENOFILE; break;
    }
    tcfdbsetecode(fdb, ecode, __FILE__, __LINE__, __func__);
    return false;
  }
  if(!(omode & FDBONOLCK)){
    if(!tclock(fd, omode & FDBOWRITER, omode & FDBOLCKNB)){
      tcfdbsetecode(fdb, TCELOCK, __FILE__, __LINE__, __func__);
      close(fd);
      return false;
    }
  }
  if((omode & FDBOWRITER) && (omode & FDBOTRUNC)){
    if(ftruncate(fd, 0) == -1){
      tcfdbsetecode(fdb, TCETRUNC, __FILE__, __LINE__, __func__);
      close(fd);
      return false;
    }
    if(!tcfdbwalremove(fdb, path)){
      close(fd);
      return false;
    }
  }
  struct stat sbuf;
  if(fstat(fd, &sbuf) == -1 || !S_ISREG(sbuf.st_mode)){
    tcfdbsetecode(fdb, TCESTAT, __FILE__, __LINE__, __func__);
    close(fd);
    return false;
  }
  char hbuf[FDBHEADSIZ];
  if((omode & FDBOWRITER) && sbuf.st_size < 1){
    fdb->flags = 0;
    fdb->rnum = 0;
    fdb->fsiz = FDBHEADSIZ;
    fdb->min = 0;
    fdb->max = 0;
    tcfdbdumpmeta(fdb, hbuf);
    if(!tcwrite(fd, hbuf, FDBHEADSIZ)){
      tcfdbsetecode(fdb, TCEWRITE, __FILE__, __LINE__, __func__);
      close(fd);
      return false;
    }
    sbuf.st_size = fdb->fsiz;
  }
  if(lseek(fd, 0, SEEK_SET) == -1){
    tcfdbsetecode(fdb, TCESEEK, __FILE__, __LINE__, __func__);
    close(fd);
    return false;
  }
  if(!tcread(fd, hbuf, FDBHEADSIZ)){
    tcfdbsetecode(fdb, TCEREAD, __FILE__, __LINE__, __func__);
    close(fd);
    return false;
  }
  int type = fdb->type;
  tcfdbloadmeta(fdb, hbuf);
  if((fdb->flags & FDBFOPEN) && tcfdbwalrestore(fdb, path)){
    if(lseek(fd, 0, SEEK_SET) == -1){
      tcfdbsetecode(fdb, TCESEEK, __FILE__, __LINE__, __func__);
      close(fd);
      return false;
    }
    if(!tcread(fd, hbuf, FDBHEADSIZ)){
      tcfdbsetecode(fdb, TCEREAD, __FILE__, __LINE__, __func__);
      close(fd);
      return false;
    }
    tcfdbloadmeta(fdb, hbuf);
    if(!tcfdbwalremove(fdb, path)){
      close(fd);
      return false;
    }
  }
  if(!(omode & FDBONOLCK)){
    if(memcmp(hbuf, FDBMAGICDATA, strlen(FDBMAGICDATA)) || fdb->type != type ||
       fdb->width < 1 || sbuf.st_size < fdb->fsiz || fdb->limsiz < FDBHEADSIZ ||
       fdb->fsiz > fdb->limsiz){
      tcfdbsetecode(fdb, TCEMETA, __FILE__, __LINE__, __func__);
      close(fd);
      return false;
    }
    if(sbuf.st_size > fdb->fsiz) fdb->fsiz = sbuf.st_size;
  }
  void *map = mmap(0, fdb->limsiz, PROT_READ | ((omode & FDBOWRITER) ? PROT_WRITE : 0),
                   MAP_SHARED, fd, 0);
  if(map == MAP_FAILED){
    tcfdbsetecode(fdb, TCEMMAP, __FILE__, __LINE__, __func__);
    close(fd);
    return false;
  }
  if(fdb->width <= UINT8_MAX){
    fdb->wsiz = sizeof(uint8_t);
  } else if(fdb->width <= UINT16_MAX){
    fdb->wsiz = sizeof(uint16_t);
  } else {
    fdb->wsiz = sizeof(uint32_t);
  }
  fdb->rsiz = fdb->width + fdb->wsiz;
  fdb->limid = (fdb->limsiz - FDBHEADSIZ) / fdb->rsiz;
  fdb->path = tcstrdup(path);
  fdb->fd = fd;
  fdb->omode = omode;
  fdb->iter = 0;
  fdb->map = map;
  fdb->array = (unsigned char *)map + FDBHEADSIZ;
  fdb->ecode = TCESUCCESS;
  fdb->fatal = false;
  fdb->inode = (uint64_t)sbuf.st_ino;