tctdb.c

Tokyo Cabinet的Tokyo Cabinet 是一个DBM的实现。这里的数据库由一系列key-value对的记录构成。key和value都可以是任意长度的字节序列,既可以是二进制也可以是字符

/*************************************************************************************************
 * The table database API of Tokyo Cabinet
 *                                                      Copyright (C) 2006-2009 Mikio Hirabayashi
 * This file is part of Tokyo Cabinet.
 * Tokyo Cabinet is free software; you can redistribute it and/or modify it under the terms of
 * the GNU Lesser General Public License as published by the Free Software Foundation; either
 * version 2.1 of the License or any later version.  Tokyo Cabinet is distributed in the hope
 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 * You should have received a copy of the GNU Lesser General Public License along with Tokyo
 * Cabinet; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307 USA.
 *************************************************************************************************/


#include "tcutil.h"
#include "tchdb.h"
#include "tcbdb.h"
#include "tctdb.h"
#include "myconf.h"

#define TDBOPAQUESIZ   64                // size of using opaque field
#define TDBLEFTOPQSIZ  64                // size of left opaque field
#define TDBPAGEBUFSIZ  32768             // size of a buffer to read each page

#define TDBDEFBNUM     131071            // default bucket number
#define TDBDEFAPOW     4                 // default alignment power
#define TDBDEFFPOW     10                // default free block pool power
#define TDBDEFLCNUM    4096              // default number of leaf cache
#define TDBDEFNCNUM    512               // default number of node cache
#define TDBDEFXMSIZ    ((1LL<<20)*64)    // default size of the extra mapped memory
#define TDBIDXSUFFIX   "idx"             // suffix of column index file
#define TDBIDXLMEMB    64                // number of members in each leaf of the index
#define TDBIDXNMEMB    256               // number of members in each node of the index
#define TDBIDXLSMAX    8192              // maximum size of each leaf of the index
#define TDBNUMCNTCOL   "_num"            // column name of number counting
#define TDBCNTBUFSIZ   1024              // size of a buffer for number counting
#define TDBHINTUSIZ    256               // unit size of the hint string
#define TDBORDRATIO    0.2               // ratio of records to use the order index

enum {                                   // enumeration for duplication behavior
  TDBPDOVER,                             // overwrite an existing value
  TDBPDKEEP,                             // keep the existing value
  TDBPDCAT                               // concatenate values
};

typedef struct {                         // type of structure for a sort key
  const char *kbuf;                      // pointer to the primary key
  int ksiz;                              // size of the primary key
  char *vbuf;                            // pointer to the value
  int vsiz;                              // size of the value
} TDBSORTKEY;


/* private macros */
#define TDBLOCKMETHOD(TC_tdb, TC_wr) \
  ((TC_tdb)->mmtx ? tctdblockmethod((TC_tdb), (TC_wr)) : true)
#define TDBUNLOCKMETHOD(TC_tdb) \
  ((TC_tdb)->mmtx ? tctdbunlockmethod(TC_tdb) : true)
#define TDBTHREADYIELD(TC_tdb) \
  do { if((TC_tdb)->mmtx) sched_yield(); } while(false)


/* private function prototypes */
static void tctdbclear(TCTDB *tdb);
static bool tctdbopenimpl(TCTDB *tdb, const char *path, int omode);
static bool tctdbcloseimpl(TCTDB *tdb);
static bool tctdbputimpl(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols, int dmode);
static bool tctdboutimpl(TCTDB *tdb, const char *pkbuf, int pksiz);
static TCMAP *tctdbgetimpl(TCTDB *tdb, const void *pkbuf, int pksiz);
static double tctdbaddnumber(TCTDB *tdb, const void *pkbuf, int pksiz, double num);
static bool tctdboptimizeimpl(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts);
static bool tctdbvanishimpl(TCTDB *tdb);
static bool tctdbcopyimpl(TCTDB *tdb, const char *path);
static bool tctdbtranbeginimpl(TCTDB *tdb);
static bool tctdbtrancommitimpl(TCTDB *tdb);
static bool tctdbtranabortimpl(TCTDB *tdb);
static bool tctdbsetindeximpl(TCTDB *tdb, const char *name, int type);
static int64_t tctdbgenuidimpl(TCTDB *tdb, int64_t inc);
static TCLIST *tctdbqrysearchimpl(TDBQRY *qry);
static TCMAP *tctdbqryidxfetch(TDBQRY *qry, TDBCOND *cond, TDBIDX *idx);
static bool tctdbqryonecondmatch(TDBQRY *qry, TDBCOND *cond, const char *pkbuf, int pksiz);
static bool tctdbqryallcondmatch(TDBQRY *qry, const char *pkbuf, int pksiz);
static bool tctdbqrycondmatch(int op, const char *expr, int esiz, const char *vbuf, int vsiz);
static bool tctdbqrycondcheckstrand(const char *tval, const char *oval);
static bool tctdbqrycondcheckstror(const char *tval, const char *oval);
static bool tctdbqrycondcheckstroreq(const char *vbuf, const char *expr);
static bool tctdbqrycondchecknumbt(const char *vbuf, const char *expr);
static bool tctdbqrycondchecknumoreq(const char *vbuf, const char *expr);
static int tdbcmppkeynumasc(const TCLISTDATUM *a, const TCLISTDATUM *b);
static int tdbcmppkeynumdesc(const TCLISTDATUM *a, const TCLISTDATUM *b);
static int tdbcmpsortkeystrasc(const TDBSORTKEY *a, const TDBSORTKEY *b);
static int tdbcmpsortkeystrdesc(const TDBSORTKEY *a, const TDBSORTKEY *b);
static int tdbcmpsortkeynumasc(const TDBSORTKEY *a, const TDBSORTKEY *b);
static int tdbcmpsortkeynumdesc(const TDBSORTKEY *a, const TDBSORTKEY *b);
static bool tctdbidxput(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols);
static bool tctdbidxout(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols);
static bool tctdbforeachimpl(TCTDB *tdb, TCITER iter, void *op);
static int tctdbqryprocoutcb(const void *pkbuf, int pksiz, TCMAP *cols, void *op);
static bool tctdblockmethod(TCTDB *tdb, bool wr);
static bool tctdbunlockmethod(TCTDB *tdb);


/* debugging function prototypes */
void tctdbprintmeta(TCTDB *tdb);



/*************************************************************************************************
 * API
 *************************************************************************************************/


/* Get the message string corresponding to an error code. */
const char *tctdberrmsg(int ecode){
  return tcerrmsg(ecode);
}


/* Create a table database object. */
TCTDB *tctdbnew(void){
  TCTDB *tdb;
  TCMALLOC(tdb, sizeof(*tdb));
  tctdbclear(tdb);
  tdb->hdb = tchdbnew();
  tchdbtune(tdb->hdb, TDBDEFBNUM, TDBDEFAPOW, TDBDEFFPOW, 0);
  tchdbsetxmsiz(tdb->hdb, TDBDEFXMSIZ);
  return tdb;
}


/* Delete a table database object. */
void tctdbdel(TCTDB *tdb){
  assert(tdb);
  if(tdb->open) tctdbclose(tdb);
  tchdbdel(tdb->hdb);
  if(tdb->mmtx){
    pthread_rwlock_destroy(tdb->mmtx);
    TCFREE(tdb->mmtx);
  }
  TCFREE(tdb);
}


/* Get the last happened error code of a table database object. */
int tctdbecode(TCTDB *tdb){
  assert(tdb);
  return tchdbecode(tdb->hdb);
}


/* Set mutual exclusion control of a table database object for threading. */
bool tctdbsetmutex(TCTDB *tdb){
  assert(tdb);
  if(!TCUSEPTHREAD) return true;
  if(tdb->mmtx || tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  TCMALLOC(tdb->mmtx, sizeof(pthread_rwlock_t));
  bool err = false;
  if(pthread_rwlock_init(tdb->mmtx, NULL) != 0) err = true;
  if(err){
    TCFREE(tdb->mmtx);
    tdb->mmtx = NULL;
    return false;
  }
  return tchdbsetmutex(tdb->hdb);
}


/* Set the tuning parameters of a table database object. */
bool tctdbtune(TCTDB *tdb, int64_t bnum, int8_t apow, int8_t fpow, uint8_t opts){
  assert(tdb);
  if(tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  tdb->opts = opts;
  uint8_t hopts = 0;
  if(opts & TDBTLARGE) hopts |= HDBTLARGE;
  if(opts & TDBTDEFLATE) hopts |= HDBTDEFLATE;
  if(opts & TDBTBZIP) hopts |= HDBTBZIP;
  if(opts & TDBTTCBS) hopts |= HDBTTCBS;
  if(opts & TDBTEXCODEC) hopts |= HDBTEXCODEC;
  bnum = (bnum > 0) ? bnum : TDBDEFBNUM;
  apow = (apow >= 0) ? apow : TDBDEFAPOW;
  fpow = (fpow >= 0) ? fpow : TDBDEFFPOW;
  return tchdbtune(tdb->hdb, bnum, apow, fpow, hopts);
}


/* Set the caching parameters of a table database object. */
bool tctdbsetcache(TCTDB *tdb, int32_t rcnum, int32_t lcnum, int32_t ncnum){
  assert(tdb);
  if(tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  if(lcnum > 0) tdb->lcnum = lcnum;
  if(ncnum > 0) tdb->ncnum = ncnum;
  return tchdbsetcache(tdb->hdb, rcnum);
}


/* Set the size of the extra mapped memory of a table database object. */
bool tctdbsetxmsiz(TCTDB *tdb, int64_t xmsiz){
  assert(tdb);
  if(tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  return tchdbsetxmsiz(tdb->hdb, xmsiz);
}


/* Open a database file and connect a table database object. */
bool tctdbopen(TCTDB *tdb, const char *path, int omode){
  assert(tdb && path);
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbopenimpl(tdb, path, omode);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}


/* Close a table database object. */
bool tctdbclose(TCTDB *tdb){
  assert(tdb);
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(!tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbcloseimpl(tdb);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}


/* Store a record into a table database object. */
bool tctdbput(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
  assert(tdb && pkbuf && pksiz >= 0 && cols);
  int vsiz;
  if(tcmapget(cols, "", 0, &vsiz)){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(!tdb->open || !tdb->wmode){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDOVER);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}


/* Store a string record into a table database object with a zero separated column string. */
bool tctdbput2(TCTDB *tdb, const void *pkbuf, int pksiz, const void *cbuf, int csiz){
  assert(tdb && pkbuf && pksiz >= 0 && cbuf && csiz >= 0);
  TCMAP *cols = tcstrsplit4(cbuf, csiz);
  bool rv = tctdbput(tdb, pkbuf, pksiz, cols);
  tcmapdel(cols);
  return rv;
}


/* Store a string record into a table database object with a tab separated column string. */
bool tctdbput3(TCTDB *tdb, const char *pkstr, const char *cstr){
  assert(tdb && pkstr && cstr);
  TCMAP *cols = tcstrsplit3(cstr, "\t");
  bool rv = tctdbput(tdb, pkstr, strlen(pkstr), cols);
  tcmapdel(cols);
  return rv;
}


/* Store a new record into a table database object. */
bool tctdbputkeep(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
  assert(tdb && pkbuf && pksiz >= 0 && cols);
  int vsiz;
  if(tcmapget(cols, "", 0, &vsiz)){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(!tdb->open || !tdb->wmode){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDKEEP);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}


/* Store a new string record into a table database object with a zero separated column string. */
bool tctdbputkeep2(TCTDB *tdb, const void *pkbuf, int pksiz, const void *cbuf, int csiz){
  assert(tdb && pkbuf && pksiz >= 0 && cbuf && csiz >= 0);
  TCMAP *cols = tcstrsplit4(cbuf, csiz);
  bool rv = tctdbputkeep(tdb, pkbuf, pksiz, cols);
  tcmapdel(cols);
  return rv;
}


/* Store a new string record into a table database object with a tab separated column string. */
bool tctdbputkeep3(TCTDB *tdb, const char *pkstr, const char *cstr){
  assert(tdb && pkstr && cstr);
  TCMAP *cols = tcstrsplit3(cstr, "\t");
  bool rv = tctdbputkeep(tdb, pkstr, strlen(pkstr), cols);
  tcmapdel(cols);
  return rv;
}


/* Concatenate columns of the existing record in a table database object. */
bool tctdbputcat(TCTDB *tdb, const void *pkbuf, int pksiz, TCMAP *cols){
  assert(tdb && pkbuf && pksiz >= 0 && cols);
  int vsiz;
  if(tcmapget(cols, "", 0, &vsiz)){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    return false;
  }
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(!tdb->open || !tdb->wmode){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdbputimpl(tdb, pkbuf, pksiz, cols, TDBPDCAT);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}


/* Concatenate columns in a table database object with a zero separated column string. */
bool tctdbputcat2(TCTDB *tdb, const void *pkbuf, int pksiz, const void *cbuf, int csiz){
  assert(tdb && pkbuf && pksiz >= 0 && cbuf && csiz >= 0);
  TCMAP *cols = tcstrsplit4(cbuf, csiz);
  bool rv = tctdbputcat(tdb, pkbuf, pksiz, cols);
  tcmapdel(cols);
  return rv;
}


/* Concatenate columns in a table database object with with a tab separated column string. */
bool tctdbputcat3(TCTDB *tdb, const char *pkstr, const char *cstr){
  assert(tdb && pkstr && cstr);
  TCMAP *cols = tcstrsplit3(cstr, "\t");
  bool rv = tctdbputcat(tdb, pkstr, strlen(pkstr), cols);
  tcmapdel(cols);
  return rv;
}


/* Remove a record of a table database object. */
bool tctdbout(TCTDB *tdb, const void *pkbuf, int pksiz){
  assert(tdb && pkbuf && pksiz >= 0);
  if(!TDBLOCKMETHOD(tdb, true)) return false;
  if(!tdb->open || !tdb->wmode){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return false;
  }
  bool rv = tctdboutimpl(tdb, pkbuf, pksiz);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}


/* Remove a string record of a table database object. */
bool tctdbout2(TCTDB *tdb, const char *pkstr){
  assert(tdb && pkstr);
  return tctdbout(tdb, pkstr, strlen(pkstr));
}


/* Retrieve a record in a table database object. */
TCMAP *tctdbget(TCTDB *tdb, const void *pkbuf, int pksiz){
  assert(tdb && pkbuf && pksiz >= 0);
  if(!TDBLOCKMETHOD(tdb, false)) return NULL;
  if(!tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return NULL;
  }
  TCMAP *rv = tctdbgetimpl(tdb, pkbuf, pksiz);
  TDBUNLOCKMETHOD(tdb);
  return rv;
}


/* Retrieve a record in a table database object as a zero separated column string. */
char *tctdbget2(TCTDB *tdb, const void *pkbuf, int pksiz, int *sp){
  assert(tdb && pkbuf && pksiz >= 0 && sp);
  TCMAP *cols = tctdbget(tdb, pkbuf, pksiz);
  if(!cols) return NULL;
  char *cbuf = tcstrjoin4(cols, sp);
  tcmapdel(cols);
  return cbuf;
}


/* Retrieve a string record in a table database object as a tab separated column string. */
char *tctdbget3(TCTDB *tdb, const char *pkstr){
  assert(tdb && pkstr);
  TCMAP *cols = tctdbget(tdb, pkstr, strlen(pkstr));
  if(!cols) return NULL;
  char *cstr = tcstrjoin3(cols, '\t');
  tcmapdel(cols);
  return cstr;
}


/* Get the size of the value of a record in a table database object. */
int tctdbvsiz(TCTDB *tdb, const void *pkbuf, int pksiz){
  assert(tdb && pkbuf && pksiz >= 0);
  if(!TDBLOCKMETHOD(tdb, false)) return -1;
  if(!tdb->open){
    tctdbsetecode(tdb, TCEINVALID, __FILE__, __LINE__, __func__);
    TDBUNLOCKMETHOD(tdb);
    return -1;
  }
  int rv = tchdbvsiz(tdb->hdb, pkbuf, pksiz);
  TDBUNLOCKMETHOD(tdb);