curia.c

harvest是一个下载html网页得机器人

/*************************************************************************************************
 * Implementation of Curia
 *                                                      Copyright (C) 2000-2003 Mikio Hirabayashi
 * This file is part of QDBM, Quick Database Manager.
 * QDBM is free software; you can redistribute it and/or modify it under the terms of the GNU
 * Lesser General Public License as published by the Free Software Foundation; either version
 * 2.1 of the License or any later version.  QDBM is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
 * details.
 * You should have received a copy of the GNU Lesser General Public License along with QDBM; if
 * not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
 * 02111-1307 USA.
 *************************************************************************************************/


#include "curia.h"
#include "myconf.h"

#define CR_NAMEMAX     512               /* max size of a database name */
#define CR_DPMAX       512               /* max number of division of a database */
#define CR_DIRMODE     00755             /* permission of a creating directory */
#define CR_FILEMODE    00644             /* permission of a creating file */
#define CR_PATHBUFSIZ  1024              /* size of a path buffer */
#define CR_DEFDNUM     5                 /* default number of division of a database */
#define CR_ATTRBNUM    16                /* bucket number of attrubute database */
#define CR_DPNAME      "depot"           /* name of each sub database */
#define CR_KEYDNUM     "dnum"            /* key of division number */
#define CR_KEYLRNUM    "lrnum"           /* key of the number of large objects */
#define CR_LOBDIR      "lob"             /* name of the directory of large objects */
#define CR_LOBDDEPTH   2                 /* depth of the directories of large objects */


/* private function prototypes */
static char *crstrdup(const char *str);
static int crdpgetnum(DEPOT *depot, const char *kbuf, int ksiz);
static char *crgetlobpath(CURIA *curia, const char *kbuf, int ksiz);
static int crmklobdir(const char *path);
static int crrmlobdir(const char *path);
static int crwrite(int fd, const void *buf, int size);
static int crread(int fd, void *buf, int size);



/*************************************************************************************************
 * public objects
 *************************************************************************************************/


/* Get a database handle. */
CURIA *cropen(const char *name, int omode, int bnum, int dnum){
  DEPOT *attr, **depots;
  char path[CR_PATHBUFSIZ], *tname;
  int i, j, dpomode, inode, lrnum;
  struct stat sbuf;
  CURIA *curia;
  assert(name);
  if(dnum < 1) dnum = CR_DEFDNUM;
  if(dnum > CR_DPMAX) dnum = CR_DPMAX;
  if(strlen(name) > CR_NAMEMAX){
    dpecode = DP_EMISC;
    return NULL;
  }
  dpomode = DP_OREADER;
  if(omode & CR_OWRITER){
    dpomode = DP_OWRITER;
    if(omode & CR_OCREAT) dpomode |= DP_OCREAT;
    if(omode & CR_OTRUNC) dpomode |= DP_OTRUNC;
  }
  if(omode & CR_ONOLCK) dpomode |= DP_ONOLCK;
  attr = NULL;
  lrnum = 0;
  if((omode & CR_OWRITER) && (omode & CR_OCREAT)){
    if(mkdir(name, CR_DIRMODE) == -1 && errno != EEXIST){
      dpecode = DP_EMKDIR;
      return NULL;
    }
    for(i = 0; i < dnum; i++){
      sprintf(path, "%s%c%04d", name, MYPATHCHR, i + 1);
      if(mkdir(path, CR_DIRMODE) == -1 && errno != EEXIST){
        dpecode = DP_EMKDIR;
        return NULL;
      }
    }
    sprintf(path, "%s%c%s", name, MYPATHCHR, CR_DPNAME);
    if(!(attr = dpopen(path, dpomode, CR_ATTRBNUM))) return NULL;
    if(!dpput(attr, CR_KEYDNUM, -1, (char *)&dnum, sizeof(int), DP_DOVER)){
      dpclose(attr);
      return NULL;
    }
  }
  if(!attr){
    sprintf(path, "%s%c%s", name, MYPATHCHR, CR_DPNAME);
    if(!(attr = dpopen(path, dpomode, 1))) return NULL;
    if(!(omode & CR_OTRUNC)){
      if((dnum = crdpgetnum(attr, CR_KEYDNUM, -1)) < 1 ||
         (lrnum = crdpgetnum(attr, CR_KEYLRNUM, -1)) < 0){
        dpclose(attr);
        dpecode = DP_EBROKEN;
        return NULL;
      }
    }
  }
  if(omode & CR_OTRUNC){
    for(i = 0; i < CR_DPMAX; i++){
      sprintf(path, "%s%c%04d%c%s", name, MYPATHCHR, i + 1, MYPATHCHR, CR_DPNAME);
      if(unlink(path) == -1 && errno != ENOENT){
        dpclose(attr);
        dpecode = DP_EUNLINK;
        return NULL;
      }
      sprintf(path, "%s%c%04d%c%s", name, MYPATHCHR, i + 1, MYPATHCHR, CR_LOBDIR);
      if(!crrmlobdir(path)){
        dpclose(attr);
        return NULL;
      }
      if(i >= dnum){
        sprintf(path, "%s%c%04d", name, MYPATHCHR, i + 1);
        if(rmdir(path) == -1 && errno != ENOENT){
          dpclose(attr);
          dpecode = DP_ERMDIR;
          return NULL;
        }
      }
    }
    errno = 0;
  }
  if(stat(name, &sbuf) == -1){
    dpclose(attr);
    dpecode = DP_ESTAT;
    return NULL;
  }
  inode = sbuf.st_ino;
  if(!(depots = malloc(dnum * sizeof(DEPOT *)))){
    dpclose(attr);
    dpecode = DP_EALLOC;
    return NULL;
  }
  for(i = 0; i < dnum; i++){
    sprintf(path, "%s%c%04d%c%s", name, MYPATHCHR, i + 1, MYPATHCHR, CR_DPNAME);
    if(!(depots[i] = dpopen(path, dpomode, bnum))){
      for(j = 0; j < i; j++){
        dpclose(depots[j]);
      }
      free(depots);
      dpclose(attr);
      return NULL;
    }
  }
  curia = malloc(sizeof(CURIA));
  tname = crstrdup(name);
  if(!curia || !tname){
    free(curia);
    free(tname);
    for(i = 0; i < dnum; i++){
      dpclose(depots[i]);
    }
    free(depots);
    dpclose(attr);
    dpecode = DP_EALLOC;
    return NULL;
  }
  curia->name = tname;
  curia->wmode = (omode & CR_OWRITER);
  curia->inode = inode;
  curia->attr = attr;
  curia->depots = depots;
  curia->dnum = dnum;
  curia->inum = 0;
  curia->lrnum = lrnum;
  return curia;
}


/* Close a database handle. */
int crclose(CURIA *curia){
  int i, err;
  assert(curia);
  err = FALSE;
  for(i = 0; i < curia->dnum; i++){
    if(!dpclose(curia->depots[i])) err = TRUE;
  }
  free(curia->depots);
  if(curia->wmode){
    if(!dpput(curia->attr, CR_KEYLRNUM, -1, (char *)&(curia->lrnum), sizeof(int), DP_DOVER))
      err = TRUE;
  }
  if(!dpclose(curia->attr)) err = TRUE;
  free(curia->name);
  free(curia);
  return err ? FALSE : TRUE;
}


/* Store a record. */
int crput(CURIA *curia, const char *kbuf, int ksiz, const char *vbuf, int vsiz, int dmode){
  int dpdmode;
  int tnum;
  assert(curia && kbuf && vbuf);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  switch(dmode){
  case CR_DKEEP: dpdmode = DP_DKEEP; break;
  case CR_DCAT: dpdmode = DP_DCAT; break;
  default: dpdmode = DP_DOVER; break;
  }
  tnum = dpouterhash(kbuf, ksiz) % curia->dnum;
  return dpput(curia->depots[tnum], kbuf, ksiz, vbuf, vsiz, dpdmode);
}


/* Delete a record. */
int crout(CURIA *curia, const char *kbuf, int ksiz){
  int tnum;
  assert(curia && kbuf);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  tnum = dpouterhash(kbuf, ksiz) % curia->dnum;
  return dpout(curia->depots[tnum], kbuf, ksiz);
}


/* Retrieve a record. */
char *crget(CURIA *curia, const char *kbuf, int ksiz, int start, int max, int *sp){
  int tnum;
  assert(curia && kbuf && start >= 0);
  tnum = dpouterhash(kbuf, ksiz) % curia->dnum;
  return dpget(curia->depots[tnum], kbuf, ksiz, start, max, sp);
}


/* Get the size of the value of a record. */
int crvsiz(CURIA *curia, const char *kbuf, int ksiz){
  int tnum;
  assert(curia && kbuf);
  tnum = dpouterhash(kbuf, ksiz) % curia->dnum;
  return dpvsiz(curia->depots[tnum], kbuf, ksiz);
}


/* Initialize the iterator of a database handle. */
int criterinit(CURIA *curia){
  int i, err;
  assert(curia);
  err = FALSE;
  for(i = 0; i < curia->dnum; i++){
    if(!dpiterinit(curia->depots[i])){
      err = TRUE;
      break;
    }
  }
  curia->inum = 0;
  return err ? FALSE : TRUE;
}


/* Get the next key of the iterator. */
char *criternext(CURIA *curia, int *sp){
  char *kbuf;
  assert(curia);
  kbuf = NULL;
  while(curia->inum < curia->dnum && !(kbuf = dpiternext(curia->depots[curia->inum], sp))){
    if(dpecode != DP_ENOITEM) return NULL;
    (curia->inum)++;
  }
  return kbuf;
}


/* Set alignment of a database handle. */
int crsetalign(CURIA *curia, int align){
  int i, err;
  assert(curia);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  err = FALSE;
  for(i = 0; i < curia->dnum; i++){
    if(!dpsetalign(curia->depots[i], align)){
      err = TRUE;
      break;
    }
  }
  return err ? FALSE : TRUE;
}


/* Synchronize contents of updating a database with the files and the devices. */
int crsync(CURIA *curia){
  int i, err;
  assert(curia);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  err = FALSE;
  if(!dpput(curia->attr, CR_KEYLRNUM, -1, (char *)&(curia->lrnum), sizeof(int), DP_DOVER) ||
     !dpsync(curia->attr)) err = TRUE;
  for(i = 0; i < curia->dnum; i++){
    if(!dpsync(curia->depots[i])){
      err = TRUE;
      break;
    }
  }
  return err ? FALSE : TRUE;
}


/* Optimize a database. */
int croptimize(CURIA *curia, int bnum){
  int i, err;
  assert(curia);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  err = FALSE;
  for(i = 0; i < curia->dnum; i++){
    if(!dpoptimize(curia->depots[i], bnum)){
      err = TRUE;
      break;
    }
  }
  curia->inum = 0;
  return err ? FALSE : TRUE;
}


/* Get the name of a database. */
char *crname(CURIA *curia){
  char *name;
  assert(curia);
  if(!(name = crstrdup(curia->name))){
    dpecode = DP_EALLOC;
    return NULL;
  }
  return name;
}


/* Get the total size of database files. */
int crfsiz(CURIA *curia){
  int i, sum, rv;
  assert(curia);
  sum = 0;
  for(i = 0; i < curia->dnum; i++){
    rv = dpfsiz(curia->depots[i]);
    if(rv == -1) return -1;
    sum += rv;
  }
  return sum;
}


/* Get the total number of the elements of each bucket array. */
int crbnum(CURIA *curia){
  int i, sum, rv;
  assert(curia);
  sum = 0;
  for(i = 0; i < curia->dnum; i++){
    rv = dpbnum(curia->depots[i]);
    if(rv == -1) return -1;
    sum += rv;
  }
  return sum;
}


/* Get the total number of the used elements of each bucket array. */
int crbusenum(CURIA *curia){
  int i, sum, rv;
  assert(curia);
  sum = 0;
  for(i = 0; i < curia->dnum; i++){
    rv = dpbusenum(curia->depots[i]);
    if(rv == -1) return -1;
    sum += rv;
  }
  return sum;
}


/* Get the number of the records stored in a database. */
int crrnum(CURIA *curia){
  int i, sum, rv;
  assert(curia);
  sum = 0;
  for(i = 0; i < curia->dnum; i++){
    rv = dprnum(curia->depots[i]);
    if(rv == -1) return -1;
    sum += rv;
  }
  return sum;
}


/* Check whether a database handle is a writer or not. */
int crwritable(CURIA *curia){
  assert(curia);
  return curia->wmode;
}


/* Check whether a database has a fatal error or not. */