depot.c

harvest是一个下载html网页得机器人

}


/* Get the next key of the iterator. */
char *dpiternext(DEPOT *depot, int *sp){
  int off, head[DP_RHNUM];
  char *kbuf;
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return NULL;
  }
  off = DP_HEADSIZ + depot->bnum * sizeof(int);
  off = off > depot->ioff ? off : depot->ioff;
  while(off < depot->fsiz){
    if(!dprechead(depot, off, head)){
      depot->fatal = TRUE;
      return NULL;
    }
    if(head[DP_RHIFLAGS] & DP_RECFDEL){
      off += dprecsize(head);
    } else {
      if(!(kbuf = dpreckey(depot, off, head))){
        depot->fatal = TRUE;
        return NULL;
      }
      depot->ioff = off + dprecsize(head);
      if(sp) *sp = head[DP_RHIKSIZ];
      return kbuf;
    }
  }
  dpecode = DP_ENOITEM;
  return NULL;
}


/* Set alignment of a database handle. */
int dpsetalign(DEPOT *depot, int align){
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return FALSE;
  }
  if(!depot->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  depot->align = align;
  return TRUE;
}


/* Synchronize contents of updating a database with the file and the device. */
int dpsync(DEPOT *depot){
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return FALSE;
  }
  if(!depot->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  *((int *)(depot->map + DP_FSIZOFF)) = depot->fsiz;
  *((int *)(depot->map + DP_RNUMOFF)) = depot->rnum;
  if(msync(depot->map, depot->msiz, MS_SYNC) == -1){
    dpecode = DP_EMAP;
    depot->fatal = TRUE;
    return FALSE;
  }
  if(fsync(depot->fd) == -1){
    dpecode = DP_ESYNC;
    depot->fatal = TRUE;
    return FALSE;
  }
  return TRUE;
}


/* Optimize a database. */
int dpoptimize(DEPOT *depot, int bnum){
  DEPOT *tdepot;
  char *name;
  int i, err, off, head[DP_RHNUM], ksizs[DP_OPTRUNIT], vsizs[DP_OPTRUNIT], unum;
  char *kbufs[DP_OPTRUNIT], *vbufs[DP_OPTRUNIT];
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return FALSE;
  }
  if(!depot->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  if(!(name = malloc(strlen(depot->name) + strlen(DP_TMPFSUF) + 1))){
    dpecode = DP_EALLOC;
    depot->fatal = FALSE;
    return FALSE;
  }
  sprintf(name, "%s%s", depot->name, DP_TMPFSUF);
  bnum = dpgetprime(bnum > 0 ? bnum : (int)(depot->rnum * (1.0 / DP_OPTBLOAD)) + 1);
  if(!(tdepot = dpopen(name, DP_OWRITER | DP_OCREAT | DP_OTRUNC, bnum))){
    free(name);
    depot->fatal = TRUE;
    return FALSE;
  }
  free(name);
  if(!dpsetflags(tdepot, dpgetflags(depot))){
    dpclose(tdepot);
    depot->fatal = TRUE;
    return FALSE;
  }
  tdepot->align = depot->align;
  err = FALSE;
  off = DP_HEADSIZ + depot->bnum * sizeof(int);
  unum = 0;
  while(off < depot->fsiz){
    if(!dprechead(depot, off, head)){
      err = TRUE;
      break;
    }
    if(!(head[DP_RHIFLAGS] & DP_RECFDEL)){
      kbufs[unum] = dpreckey(depot, off, head);
      ksizs[unum] = head[DP_RHIKSIZ];
      vbufs[unum] = dprecval(depot, off, head, 0, -1);
      vsizs[unum] = head[DP_RHIVSIZ];
      unum++;
      if(unum >= DP_OPTRUNIT){
        for(i = 0; i < unum; i++){
          if(kbufs[i] && vbufs[i]){
            if(!dpput(tdepot, kbufs[i], ksizs[i], vbufs[i], vsizs[i], DP_DKEEP)) err = TRUE;
          } else {
            err = TRUE;
          }
          free(kbufs[i]);
          free(vbufs[i]);
          if(err) break;
        }
        unum = 0;
      }
    }
    off += dprecsize(head);
    if(err) break;
  }
  for(i = 0; i < unum; i++){
    if(kbufs[i] && vbufs[i]){
      if(!dpput(tdepot, kbufs[i], ksizs[i], vbufs[i], vsizs[i], DP_DKEEP)) err = TRUE;
    } else {
      err = TRUE;
    }
    free(kbufs[i]);
    free(vbufs[i]);
    if(err) break;
  }
  if(!dpsync(tdepot)) err = TRUE;
  if(err){
    unlink(tdepot->name);
    dpclose(tdepot);
    depot->fatal = TRUE;
    return FALSE;
  }
  if(munmap(depot->map, depot->msiz) == -1){
    dpclose(tdepot);
    dpecode = DP_EMAP;
    depot->fatal = TRUE;
    return FALSE;
  }
  depot->map = MAP_FAILED;
  if(ftruncate(depot->fd, 0) == -1){
    dpclose(tdepot);
    unlink(tdepot->name);
    dpecode = DP_ETRUNC;
    depot->fatal = TRUE;
    return FALSE;
  }
  if(dpfcopy(depot->fd, 0, tdepot->fd, 0) == -1){
    dpclose(tdepot);
    unlink(tdepot->name);
    depot->fatal = TRUE;
    return FALSE;
  }
  depot->fsiz = tdepot->fsiz;
  depot->bnum = tdepot->bnum;
  depot->ioff = 0;
  depot->mroff = -1;
  depot->mrsiz = -1;
  depot->msiz = tdepot->msiz;
  depot->map = mmap(0, depot->msiz, PROT_READ | PROT_WRITE, MAP_SHARED, depot->fd, 0);
  if(depot->map == MAP_FAILED){
    dpecode = DP_EMAP;
    depot->fatal = TRUE;
    return FALSE;
  }
  depot->buckets = (int *)(depot->map + DP_HEADSIZ);
  if(!(name = dpname(tdepot))){
    dpclose(tdepot);
    unlink(tdepot->name);
    depot->fatal = TRUE;
    return FALSE;
  }
  if(!dpclose(tdepot)){
    free(name);
    unlink(tdepot->name);
    depot->fatal = TRUE;
    return FALSE;
  }
  if(unlink(name) == -1){
    free(name);
    dpecode = DP_EUNLINK;
    depot->fatal = TRUE;
    return FALSE;
  }
  free(name);
  return TRUE;
}


/* Get the name of a database. */
char *dpname(DEPOT *depot){
  char *name;
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return NULL;
  }
  if(!(name = dpstrdup(depot->name))){
    dpecode = DP_EALLOC;
    depot->fatal = TRUE;
    return NULL;
  }
  return name;
}


/* Get the size of a database file. */
int dpfsiz(DEPOT *depot){
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return -1;
  }
  return depot->fsiz;
}


/* Get the number of the elements of the bucket array. */
int dpbnum(DEPOT *depot){
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return -1;
  }
  return depot->bnum;
}


/* Get the number of the used elements of the bucket array. */
int dpbusenum(DEPOT *depot){
  int i, hits;
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return -1;
  }
  hits = 0;
  for(i = 0; i < depot->bnum; i++){
    if(depot->buckets[i]) hits++;
  }
  return hits;
}


/* Get the number of the records stored in a database. */
int dprnum(DEPOT *depot){
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return -1;
  }
  return depot->rnum;
}


/* Check whether a database handle is a writer or not. */
int dpwritable(DEPOT *depot){
  assert(depot);
  return depot->wmode;
}


/* Check whether a database has a fatal error or not. */
int dpfatalerror(DEPOT *depot){
  assert(depot);
  return depot->fatal;
}


/* Get the inode number of a database file. */
int dpinode(DEPOT *depot){
  assert(depot);
  return depot->inode;
}


/* Get the file descriptor of a database file. */
int dpfdesc(DEPOT *depot){
  assert(depot);
  return depot->fd;
}


/* Remove a database file. */
int dpremove(const char *name){
  struct stat sbuf;
  DEPOT *depot;
  assert(name);
  if(stat(name, &sbuf) == -1){
    dpecode = DP_ESTAT;
    return FALSE;
  }
  if((depot = dpopen(name, DP_OWRITER | DP_OTRUNC, -1)) != NULL) dpclose(depot);
  if(unlink(name) == -1){
    dpecode = DP_EUNLINK;
    return FALSE;
  }
  return TRUE;
}


/* Hash function used inside Depot. */
int dpinnerhash(const char *kbuf, int ksiz){
  assert(kbuf);
  if(ksiz < 0) ksiz = strlen(kbuf);
  return dpfirsthash(kbuf, ksiz);
}


/* Hash function which is independent from the hash functions used inside Depot. */
int dpouterhash(const char *kbuf, int ksiz){
  assert(kbuf);
  if(ksiz < 0) ksiz = strlen(kbuf);
  return dpthirdhash(kbuf, ksiz);
}


/* Get a natural prime number not less than a number. */
int dpprimenum(int num){
  assert(num > 0);
  return dpgetprime(num);
}



/*************************************************************************************************
 * Functions for Experts
 *************************************************************************************************/


/* Synchronize updating contents on memory. */
int dpmemsync(DEPOT *depot){
  assert(depot);
  if(depot->fatal){
    dpecode = DP_EFATAL;
    return FALSE;
  }
  if(!depot->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  *((int *)(depot->map + DP_FSIZOFF)) = depot->fsiz;
  *((int *)(depot->map + DP_RNUMOFF)) = depot->rnum;
  if(msync(depot->map, depot->msiz, MS_ASYNC) == -1){
    dpecode = DP_EMAP;
    depot->fatal = TRUE;
    return FALSE;
  }
  return TRUE;
}


/* Get flags of a database. */
int dpgetflags(DEPOT *depot){
  assert(depot);
  return *(depot->map + DP_FLAGSOFF);
}


/* Set flags of a database. */
int dpsetflags(DEPOT *depot, int flags){
  assert(depot);
  if(!depot->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  *(depot->map + DP_FLAGSOFF) = flags;
  return TRUE;
}



/*************************************************************************************************
 * private objects
 *************************************************************************************************/


/* Check whether the byte order of the platform is big endian or not.
   The return value is true if bigendian, else, it is false. */
static int dpbigendian(void){
  char buf[sizeof(int)];
  *(int *)buf = 1;
  return !buf[0];
}


/* Get a copied string.
   `str' specifies an original string.
   The return value is a copied string whose region is allocated by `malloc'. */
static char *dpstrdup(const char *str){
  int len;
  char *buf;
  assert(str);
  len = strlen(str);
  if(!(buf = malloc(len + 1))) return NULL;
  memcpy(buf, str, len + 1);
  return buf;
}


/* Lock a file descriptor.
   `fd' specifies a file descriptor.
   `ex' specifies whether an exclusive lock or a shared lock is performed.
   The return value is true if successful, else, it is false. */
static int dplock(int fd, int ex){
  struct flock lock;
  assert(fd >= 0);
  memset(&lock, 0, sizeof(struct flock));
  lock.l_type = ex ? F_WRLCK : F_RDLCK;
  lock.l_whence = SEEK_SET;
  lock.l_start = 0;
  lock.l_len = 0;
  lock.l_pid = 0;
  while(fcntl(fd, F_SETLKW, &lock) == -1){
    if(errno != EINTR){
      dpecode = DP_ELOCK;
      return FALSE;
    }
  }
  return TRUE;
}


/* Write into a file.
   `fd' specifies a file descriptor.
   `buf' specifies a buffer to write.
   `size' specifies the size of the buffer.
   The return value is the size of the written buffer, or, -1 on failure. */
static int dpwrite(int fd, const void *buf, int size){
  const char *lbuf;
  int rv, wb;
  assert(fd >= 0 && buf && size >= 0);
  lbuf = buf;
  rv = 0;
  do {
    wb = write(fd, lbuf, size);
    switch(wb){
    case -1: if(errno != EINTR) return -1;
    case 0: break;
    default:
      lbuf += wb;
      size -= wb;
      rv += wb;
      break;
    }
  } while(size > 0);
  return rv;
}


/* Write into a file at an offset.
   `fd' specifies a file descriptor.
   `off' specifies an offset of the file.
   `buf' specifies a buffer to write.
   `size' specifies the size of the buffer.
   The return value is true if successful, else, it is false. */
static int dpseekwrite(int fd, int off, const void *buf, int size){
  char *lbuf;
  assert(fd >= 0 && buf && size >= 0);