villa.c

harvest是一个下载html网页得机器人

  if(!villa->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  if(!villa->tran){
    dpecode = DP_EMISC;
    return FALSE;
  }
  err = FALSE;
  cbmapiterinit(villa->leafc);
  while((tmp = cbmapiternext(villa->leafc, NULL)) != NULL){
    pid = *(int *)tmp;
    leaf = (VLLEAF *)cbmapget(villa->leafc, (char *)&pid, sizeof(int), NULL);
    if(leaf->dirty){
      if(!vlleafsave(villa, leaf)) err = TRUE;
    }
  }
  cbmapiterinit(villa->nodec);
  while((tmp = cbmapiternext(villa->nodec, NULL)) != NULL){
    pid = *(int *)tmp;
    node = (VLNODE *)cbmapget(villa->nodec, (char *)&pid, sizeof(int), NULL);
    if(node->dirty){
      if(!vlnodesave(villa, node)) err = TRUE;
    }
  }
  if(!dpsetalign(villa->depot, 0)) err = TRUE;
  if(!vldpputnum(villa->depot, VL_ROOTKEY, villa->root)) err = TRUE;
  if(!vldpputnum(villa->depot, VL_LASTKEY, villa->last)) err = TRUE;
  if(!vldpputnum(villa->depot, VL_LNUMKEY, villa->lnum)) err = TRUE;
  if(!vldpputnum(villa->depot, VL_NNUMKEY, villa->nnum)) err = TRUE;
  if(!vldpputnum(villa->depot, VL_RNUMKEY, villa->rnum)) err = TRUE;
  if(!dpmemsync(villa->depot)) err = TRUE;
  villa->tran = FALSE;
  villa->rbroot = -1;
  villa->rblast = -1;
  villa->rblnum = -1;
  villa->rbnnum = -1;
  villa->rbrnum = -1;
  return err ? FALSE : TRUE;
}


/* Abort the transaction. */
int vltranabort(VILLA *villa){
  int err, pid;
  const char *tmp;
  VLLEAF *leaf;
  VLNODE *node;
  assert(villa);
  if(!villa->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  if(!villa->tran){
    dpecode = DP_EMISC;
    return FALSE;
  }
  err = FALSE;
  cbmapiterinit(villa->leafc);
  while((tmp = cbmapiternext(villa->leafc, NULL)) != NULL){
    pid = *(int *)tmp;
    if(!(leaf = (VLLEAF *)cbmapget(villa->leafc, (char *)&pid, sizeof(int), NULL))){
      err = TRUE;
      continue;
    }
    if(leaf->dirty){
      leaf->dirty = FALSE;
      if(!vlleafcacheout(villa, pid)) err = TRUE;
    }
  }
  cbmapiterinit(villa->nodec);
  while((tmp = cbmapiternext(villa->nodec, NULL)) != NULL){
    pid = *(int *)tmp;
    if(!(node = (VLNODE *)cbmapget(villa->nodec, (char *)&pid, sizeof(int), NULL))){
      err = TRUE;
      continue;
    }
    if(node->dirty){
      node->dirty = FALSE;
      if(!vlnodecacheout(villa, pid)) err = TRUE;
    }
  }
  villa->tran = FALSE;
  villa->root = villa->rbroot;
  villa->last = villa->rblast;
  villa->lnum = villa->rblnum;
  villa->nnum = villa->rbnnum;
  villa->rnum = villa->rbrnum;
  return err ? FALSE : TRUE;
}


/* Remove a database file. */
int vlremove(const char *name){
  assert(name);
  return dpremove(name);
}



/*************************************************************************************************
 * private objects
 *************************************************************************************************/


/* Compare keys of two records by lexical order.
   `aptr' specifies the pointer to the region of one key.
   `asiz' specifies the size of the region of one key.
   `bptr' specifies the pointer to the region of the other key.
   `bsiz' specifies the size of the region of the other key.
   The return value is positive if the former is big, negative if the latter is big, 0 if both
   are equivalent. */
static int vllexcompare(const char *aptr, int asiz, const char *bptr, int bsiz){
  int i, min;
  assert(aptr && asiz >= 0 && bptr && bsiz >= 0);
  min = asiz < bsiz ? asiz : bsiz;
  for(i = 0; i < min; i++){
    if(((unsigned char *)aptr)[i] != ((unsigned char *)bptr)[i])
      return ((unsigned char *)aptr)[i] - ((unsigned char *)bptr)[i];
  }
  if(asiz == bsiz) return 0;
  return asiz - bsiz;
}


/* Compare keys of two records as native integers.
   `aptr' specifies the pointer to the region of one key.
   `asiz' specifies the size of the region of one key.
   `bptr' specifies the pointer to the region of the other key.
   `bsiz' specifies the size of the region of the other key.
   The return value is positive if the former is big, negative if the latter is big, 0 if both
   are equivalent. */
static int vlintcompare(const char *aptr, int asiz, const char *bptr, int bsiz){
  int anum, bnum;
  assert(aptr && asiz >= 0 && bptr && bsiz >= 0);
  if(asiz != bsiz) return asiz - bsiz;
  anum = (asiz == sizeof(int) ? *(int *)aptr : INT_MIN);
  bnum = (bsiz == sizeof(int) ? *(int *)bptr : INT_MIN);
  return anum - bnum;
}


/* Compare keys of two records as numbers of big endian.
   `aptr' specifies the pointer to the region of one key.
   `asiz' specifies the size of the region of one key.
   `bptr' specifies the pointer to the region of the other key.
   `bsiz' specifies the size of the region of the other key.
   The return value is positive if the former is big, negative if the latter is big, 0 if both
   are equivalent. */
static int vlnumcompare(const char *aptr, int asiz, const char *bptr, int bsiz){
  int i;
  assert(aptr && asiz >= 0 && bptr && bsiz >= 0);
  if(asiz != bsiz) return asiz - bsiz;
  for(i = 0; i < asiz; i++){
    if(aptr[i] != bptr[i]) return aptr[i] - bptr[i];
  }
  return 0;
}


/* Compare keys of two records as numeric strings of octal, decimal or hexadecimal.
   `aptr' specifies the pointer to the region of one key.
   `asiz' specifies the size of the region of one key.
   `bptr' specifies the pointer to the region of the other key.
   `bsiz' specifies the size of the region of the other key.
   The return value is positive if the former is big, negative if the latter is big, 0 if both
   are equivalent. */
static int vldeccompare(const char *aptr, int asiz, const char *bptr, int bsiz){
  assert(aptr && asiz >= 0 && bptr && bsiz >= 0);
  return strtod(aptr, NULL) - strtod(bptr, NULL);
}


/* Store a record composed of a pair of integers.
   `depot' specifies an internal database handle.
   `knum' specifies an integer of the key.
   `vnum' specifies an integer of the value.
   The return value is true if successful, else, it is false. */
static int vldpputnum(DEPOT *depot, int knum, int vnum){
  assert(depot);
  return dpput(depot, (char *)&knum, sizeof(int), (char *)&vnum, sizeof(int), DP_DOVER);
}


/* Retrieve a record composed of a pair of integers.
   `depot' specifies an internal database handle.
   `knum' specifies an integer of the key.
   `vip' specifies the pointer to a variable to assign the result to.
   The return value is true if successful, else, it is false. */
static int vldpgetnum(DEPOT *depot, int knum, int *vnp){
  char *vbuf;
  int vsiz;
  assert(depot && vnp);
  vbuf = dpget(depot, (char *)&knum, sizeof(int), 0, -1, &vsiz);
  if(!vbuf || vsiz != sizeof(int)){
    free(vbuf);
    return FALSE;
  }
  *vnp = *(int *)vbuf;
  free(vbuf);
  return TRUE;
}


/* Set a buffer for a variable length number.
   `buf' specifies the pointer to the buffer.
   `num' specifies the number.
   The return value is the size of valid region. */
static int vlsetvnumbuf(char *buf, int num){
  div_t d;
  int len;
  assert(buf && num >= 0);
  if(num == 0){
    ((signed char *)buf)[0] = 0;
    return 1;
  }
  len = 0;
  while(num > 0){
    d = div(num, 128);
    num = d.quot;
    ((signed char *)buf)[len] = d.rem;
    if(num > 0) ((signed char *)buf)[len] = -(((signed char *)buf)[len]) - 1;
    len++;
  }
  return len;
}


/* Read a variable length buffer.
   `buf' specifies the pointer to the buffer.
   `size' specifies the limit size to read.
   `sp' specifies the pointer to a variable to which the size of the read region assigned.
   The return value is the value of the buffer. */
static int vlreadvnumbuf(const char *buf, int size, int *sp){
  int i, num, base;
  assert(buf && size > 0 && sp);
  num = 0;
  base = 1;
  if(size < 2){
    *sp = 1;
    return ((signed char *)buf)[0];
  }
  for(i = 0; i < size; i++){
    if(((signed char *)buf)[i] >= 0){
      num += ((signed char *)buf)[i] * base;
      break;
    }
    num += base * (((signed char *)buf)[i] + 1) * -1;
    base *= 128;
  }
  *sp = i + 1;
  return num;
}


/* Create a new leaf.
   `villa' specifies a database handle.
   `prev' specifies the ID number of the previous leaf.
   `next' specifies the ID number of the previous leaf.
   The return value is a handle of the leaf. */
static VLLEAF *vlleafnew(VILLA *villa, int prev, int next){
  VLLEAF lent;
  assert(villa);
  lent.id = villa->lnum + VL_LEAFIDMIN;
  lent.dirty = TRUE;
  lent.recs = cblistopen();
  lent.prev = prev;
  lent.next = next;
  villa->lnum++;
  cbmapput(villa->leafc, (char *)&(lent.id), sizeof(int), (char *)&lent, sizeof(VLLEAF), TRUE);
  return (VLLEAF *)cbmapget(villa->leafc, (char *)&(lent.id), sizeof(int), NULL);
}


/* Remove a leaf from the cache.
   `villa' specifies a database handle.
   `id' specifies the ID number of the leaf.
   The return value is true if successful, else, it is false. */
static int vlleafcacheout(VILLA *villa, int id){
  VLLEAF *leaf;
  VLREC *recp;
  int i, j, err, ln;
  assert(villa && id >= VL_LEAFIDMIN);
  if(!(leaf = (VLLEAF *)cbmapget(villa->leafc, (char *)&id, sizeof(int), NULL))) return FALSE;
  err = FALSE;
  if(leaf->dirty){
    if(!vlleafsave(villa, leaf)) err = TRUE;
  }
  ln = CB_LISTNUM(leaf->recs);
  for(i = 0; i < ln; i++){
    recp = (VLREC *)CB_LISTVAL(leaf->recs, i, NULL);
    cbdatumclose(recp->key);
    cbdatumclose(recp->first);
    if(recp->rest){
      for(j = 0; j < CB_LISTNUM(recp->rest); j++){
        free(cblistpop(recp->rest, NULL));
      }
      cblistclose(recp->rest);
    }
  }
  cblistclose(leaf->recs);
  cbmapout(villa->leafc, (char *)&id, sizeof(int));
  return err ? FALSE : TRUE;
}


/* Save a leaf into the database.
   `villa' specifies a database handle.
   `leaf' specifies a leaf handle.
   The return value is true if successful, else, it is false. */
static int vlleafsave(VILLA *villa, VLLEAF *leaf){
  CBDATUM *buf;
  char vnumbuf[VL_VNUMBUFSIZ], *zbuf;
  const char *vbuf;
  VLREC *recp;
  int i, j, ksiz, vnum, vsiz, prev, next, vnumsiz, ln, zsiz;
  assert(villa && leaf);
  buf =  cbdatumopen(NULL, 0);
  prev = leaf->prev;
  if(prev == -1) prev = VL_NODEIDMIN - 1;
  vnumsiz = vlsetvnumbuf(vnumbuf, prev);
  cbdatumcat(buf, vnumbuf, vnumsiz);
  next = leaf->next;
  if(next == -1) next = VL_NODEIDMIN - 1;
  vnumsiz = vlsetvnumbuf(vnumbuf, next);
  cbdatumcat(buf, vnumbuf, vnumsiz);
  ln = CB_LISTNUM(leaf->recs);
  for(i = 0; i < ln; i++){
    recp = (VLREC *)CB_LISTVAL(leaf->recs, i, NULL);
    ksiz = CB_DATUMSIZE(recp->key);
    vnumsiz = vlsetvnumbuf(vnumbuf, ksiz);
    cbdatumcat(buf, vnumbuf, vnumsiz);
    cbdatumcat(buf, CB_DATUMPTR(recp->key), ksiz);
    vnum = 1 + (recp->rest ? CB_LISTNUM(recp->rest) : 0);
    vnumsiz = vlsetvnumbuf(vnumbuf, vnum);
    cbdatumcat(buf, vnumbuf, vnumsiz);
    vsiz = CB_DATUMSIZE(recp->first);
    vnumsiz = vlsetvnumbuf(vnumbuf, vsiz);
    cbdatumcat(buf, vnumbuf, vnumsiz);
    cbdatumcat(buf, CB_DATUMPTR(recp->first), vsiz);
    if(recp->rest){
      for(j = 0; j < CB_LISTNUM(recp->rest); j++){
        vbuf = cblistval(recp->rest, j, &vsiz);
        vnumsiz = vlsetvnumbuf(vnumbuf, vsiz);
        cbdatumcat(buf, vnumbuf, vnumsiz);
        cbdatumcat(buf, vbuf, vsiz);
      }
    }
  }
  if(_qdbm_deflate){
    if(!(zbuf = _qdbm_deflate(CB_DATUMPTR(buf), CB_DATUMSIZE(buf), &zsiz))){
      cbdatumclose(buf);
      if(dpecode == DP_EMODE) dpecode = DP_EMISC;
      return FALSE;
    }
    villa->avglsiz = (villa->avglsiz * 9 + zsiz) / 10;
    if(!dpsetalign(villa->depot, villa->avglsiz * VL_ALIGNRATIO) ||
       !dpput(villa->depot, (char *)&(leaf->id), sizeof(int), zbuf, zsiz, DP_DOVER)){
      cbdatumclose(buf);
      if(dpecode == DP_EMODE) dpecode = DP_EBROKEN;
      return FALSE;
    }
    free(zbuf);
  } else {
    villa->avglsiz = (villa->avglsiz * 9 + CB_DATUMSIZE(buf)) / 10;
    if(!dpsetalign(villa->depot, villa->avglsiz * VL_ALIGNRATIO) ||
       !dpput(villa->depot, (char *)&(leaf->id), sizeof(int),
              CB_DATUMPTR(buf), CB_DATUMSIZE(buf), DP_DOVER)){
      cbdatumclose(buf);
      if(dpecode == DP_EMODE) dpecode = DP_EBROKEN;
      return FALSE;
    }
  }
  cbdatumclose(buf);
  leaf->dirty = FALSE;
  return TRUE;
}


/* Load a leaf from the database.
   `villa' specifies a database handle.
   `id' specifies the ID number of the leaf.
   If successful, the return value is the pointer to the leaf, else, it is `NULL'. */
static VLLEAF *vlleafload(VILLA *villa, int id){
  char *buf, *rp, *kbuf, *vbuf, *zbuf;
  int i, size, step, ksiz, vnum, vsiz, prev, next, zsiz;
  VLLEAF *leaf, lent;
  VLREC rec;
  assert(villa && id >= VL_LEAFIDMIN);
  if((leaf = (VLLEAF *)cbmapget(villa->leafc, (char *)&id, sizeof(int), NULL)) != NULL){
    cbmapmove(villa->leafc, (char *)&id, sizeof(int), FALSE);
    return leaf;
  }
  ksiz = -1;
  prev = -1;
  next = -1;
  if(!(buf = dpget(villa->depot, (char *)&id, sizeof(int), 0, -1, &size))) return NULL;
  if(_qdbm_inflate){
    if(!(zbuf = _qdbm_inflate(buf, size, &zsiz))){
      dpecode = DP_EBROKEN;
      free(buf);
      return NULL;
    }
    free(buf);
    buf = zbuf;
    size = zsiz;
  }
  rp = buf;
  if(size >= 1){
    prev = vlreadvnumbuf(rp, size, &step);
    rp += step;
    size -= step;
    if(prev >= VL_NODEIDMIN - 1) prev = -1;
  }
  if(size >= 1){
    next = vlreadvnumbuf(rp, size, &step);
    rp += step;
    size -= step;
    if(next >= VL_NODEIDMIN - 1) next = -1;
  }
  lent.id = id;
  lent.dirty = FALSE;
  lent.recs = cblistopen();
  lent.prev = prev;
  lent.next = next;
  while(size >= 1){
    ksiz = vlreadvnumbuf(rp, size, &step);
    rp += step;
    size -= step;
    if(size < ksiz) break;
    kbuf = rp;
    rp += ksiz;
    size -= ksiz;
    vnum = vlreadvnumbuf(rp, size, &step);