cabin.c

harvest是一个下载html网页得机器人

/* Copy a map. */
CBMAP *cbmapdup(CBMAP *map){
  CBMAP *newmap;
  const char *kbuf, *vbuf;
  int ksiz, vsiz;
  assert(map);
  cbmapiterinit(map);
  newmap = cbmapopen();
  while((kbuf = cbmapiternext(map, &ksiz)) != NULL){
    vbuf = cbmapget(map, kbuf, ksiz, &vsiz);
    cbmapput(newmap, kbuf, ksiz, vbuf, vsiz, FALSE);
  }
  cbmapiterinit(map);
  return newmap;
}


/* Close a map handle. */
void cbmapclose(CBMAP *map){
  CBMAPDATUM *datum, *next;
  datum = map->first;
  while(datum){
    next = (CBMAPDATUM *)(datum->next);
    free(datum->kbuf);
    free(datum->vbuf);
    free(datum);
    datum = next;
  }
  free(map->buckets);
  free(map);
}


/* Store a record. */
int cbmapput(CBMAP *map, const char *kbuf, int ksiz, const char *vbuf, int vsiz, int over){
  CBMAPDATUM *datum, **entp;
  int bidx, hash, kcmp;
  assert(map && kbuf && vbuf);
  if(ksiz < 0) ksiz = strlen(kbuf);
  if(vsiz < 0) vsiz = strlen(vbuf);
  bidx = cbfirsthash(kbuf, ksiz) % CB_MAPBNUM;
  datum = map->buckets[bidx];
  entp = map->buckets + bidx;
  hash = cbsecondhash(kbuf, ksiz);
  while(datum){
    if(hash > datum->hash){
      entp = (CBMAPDATUM **)&(datum->left);
      datum = (CBMAPDATUM *)datum->left;
    } else if(hash < datum->hash){
      entp = (CBMAPDATUM **)&(datum->right);
      datum = (CBMAPDATUM *)datum->right;
    } else {
      kcmp = cbkeycmp(kbuf, ksiz, datum->kbuf, datum->ksiz);
      if(kcmp < 0){
        entp = (CBMAPDATUM **)&(datum->left);
        datum = (CBMAPDATUM *)datum->left;
      } else if(kcmp > 0){
        entp = (CBMAPDATUM **)&(datum->right);
        datum = (CBMAPDATUM *)datum->right;
      } else {
        if(!over) return FALSE;
        free(datum->vbuf);
        datum->vbuf = cbmemdup(vbuf, vsiz);
        datum->vsiz = vsiz;
        return TRUE;
      }
    }
  }
  datum = cbmalloc(sizeof(*datum));
  datum->kbuf = cbmemdup(kbuf, ksiz);
  datum->ksiz = ksiz;
  datum->vbuf = cbmemdup(vbuf, vsiz);
  datum->vsiz = vsiz;
  datum->hash = hash;
  datum->left = NULL;
  datum->right = NULL;
  datum->prev = (char *)map->last;
  datum->next = NULL;
  *entp = datum;
  if(!map->first) map->first = datum;
  if(map->last) map->last->next = (char *)datum;
  map->last = datum;
  map->rnum++;
  return TRUE;
}


/* Delete a record. */
int cbmapout(CBMAP *map, const char *kbuf, int ksiz){
  CBMAPDATUM *datum, **entp, *tmp;
  int bidx, hash, kcmp;
  assert(map && kbuf);
  if(ksiz < 0) ksiz = strlen(kbuf);
  bidx = cbfirsthash(kbuf, ksiz) % CB_MAPBNUM;
  datum = map->buckets[bidx];
  entp = map->buckets + bidx;
  hash = cbsecondhash(kbuf, ksiz);
  while(datum){
    if(hash > datum->hash){
      entp = (CBMAPDATUM **)&(datum->left);
      datum = (CBMAPDATUM *)datum->left;
    } else if(hash < datum->hash){
      entp = (CBMAPDATUM **)&(datum->right);
      datum = (CBMAPDATUM *)datum->right;
    } else {
      kcmp = cbkeycmp(kbuf, ksiz, datum->kbuf, datum->ksiz);
      if(kcmp < 0){
        entp = (CBMAPDATUM **)&(datum->left);
        datum = (CBMAPDATUM *)datum->left;
      } else if(kcmp > 0){
        entp = (CBMAPDATUM **)&(datum->right);
        datum = (CBMAPDATUM *)datum->right;
      } else {
        if(datum->prev) ((CBMAPDATUM *)(datum->prev))->next = datum->next;
        if(datum->next) ((CBMAPDATUM *)(datum->next))->prev = datum->prev;
        if(datum == map->first) map->first = (CBMAPDATUM *)datum->next;
        if(datum == map->last) map->last = (CBMAPDATUM *)datum->prev;
        if(datum->left && !datum->right){
          *entp = (CBMAPDATUM *)datum->left;
        } else if(!datum->left && datum->right){
          *entp = (CBMAPDATUM *)datum->right;
        } else if(!datum->left && !datum->left){
          *entp = NULL;
        } else {
          *entp = (CBMAPDATUM *)datum->left;
          tmp = *entp;
          while(TRUE){
            if(hash > tmp->hash){
              if(tmp->left){
                tmp = (CBMAPDATUM *)tmp->left;
              } else {
                tmp->left = datum->right;
                break;
              }
            } else if(hash < tmp->hash){
              if(tmp->right){
                tmp = (CBMAPDATUM *)tmp->right;
              } else {
                tmp->right = datum->right;
                break;
              }
            } else {
              kcmp = cbkeycmp(kbuf, ksiz, datum->kbuf, datum->ksiz);
              if(kcmp < 0){
                if(tmp->left){
                  tmp = (CBMAPDATUM *)tmp->left;
                } else {
                  tmp->left = datum->right;
                  break;
                }
              } else {
                if(tmp->right){
                  tmp = (CBMAPDATUM *)tmp->right;
                } else {
                  tmp->right = datum->right;
                  break;
                }
              }
            }
          }
        }
        free(datum->kbuf);
        free(datum->vbuf);
        free(datum);
        map->rnum--;
        return TRUE;
      }
    }
  }
  return FALSE;
}


/* Retrieve a record. */
const char *cbmapget(const CBMAP *map, const char *kbuf, int ksiz, int *sp){
  CBMAPDATUM *datum;
  int hash, kcmp;
  assert(map && kbuf);
  if(ksiz < 0) ksiz = strlen(kbuf);
  datum = map->buckets[cbfirsthash(kbuf, ksiz)%CB_MAPBNUM];
  hash = cbsecondhash(kbuf, ksiz);
  while(datum){
    if(hash > datum->hash){
      datum = (CBMAPDATUM *)datum->left;
    } else if(hash < datum->hash){
      datum = (CBMAPDATUM *)datum->right;
    } else {
      kcmp = cbkeycmp(kbuf, ksiz, datum->kbuf, datum->ksiz);
      if(kcmp < 0){
        datum = (CBMAPDATUM *)datum->left;
      } else if(kcmp > 0){
        datum = (CBMAPDATUM *)datum->right;
      } else {
        if(sp) *sp = datum->vsiz;
        return datum->vbuf;
      }
    }
  }
  return NULL;
}


/* Move a record to the edge. */
int cbmapmove(CBMAP *map, const char *kbuf, int ksiz, int head){
  CBMAPDATUM *datum;
  int hash, kcmp;
  assert(map && kbuf);
  if(ksiz < 0) ksiz = strlen(kbuf);
  datum = map->buckets[cbfirsthash(kbuf, ksiz)%CB_MAPBNUM];
  hash = cbsecondhash(kbuf, ksiz);
  while(datum){
    if(hash > datum->hash){
      datum = (CBMAPDATUM *)datum->left;
    } else if(hash < datum->hash){
      datum = (CBMAPDATUM *)datum->right;
    } else {
      kcmp = cbkeycmp(kbuf, ksiz, datum->kbuf, datum->ksiz);
      if(kcmp < 0){
        datum = (CBMAPDATUM *)datum->left;
      } else if(kcmp > 0){
        datum = (CBMAPDATUM *)datum->right;
      } else {
        if(head){
          if(map->first == datum) return TRUE;
          if(map->last == datum) map->last = (CBMAPDATUM *)(datum->prev);
          if(datum->prev) ((CBMAPDATUM *)(datum->prev))->next = datum->next;
          if(datum->next) ((CBMAPDATUM *)(datum->next))->prev = datum->prev;
          datum->prev = NULL;
          datum->next = (char *)(map->first);
          map->first->prev = (char *)datum;
          map->first = datum;
        } else {
          if(map->last == datum) return TRUE;
          if(map->first == datum) map->first = (CBMAPDATUM *)(datum->next);
          if(datum->prev) ((CBMAPDATUM *)(datum->prev))->next = datum->next;
          if(datum->next) ((CBMAPDATUM *)(datum->next))->prev = datum->prev;
          datum->prev = (char *)(map->last);
          datum->next = NULL;
          map->last->next = (char *)datum;
          map->last = datum;
        }
        return TRUE;
      }
    }
  }
  return FALSE;
}


/* Initialize the iterator of a map handle. */
void cbmapiterinit(CBMAP *map){
  assert(map);
  map->cur = map->first;
}


/* Get the next key of the iterator. */
const char *cbmapiternext(CBMAP *map, int *sp){
  CBMAPDATUM *datum;
  assert(map);
  if(!map->cur) return NULL;
  datum = map->cur;
  map->cur = (CBMAPDATUM *)datum->next;
  if(sp) *sp = datum->ksiz;
  return datum->kbuf;
}


/* Get the number of the records stored in a map. */
int cbmaprnum(const CBMAP *map){
  assert(map);
  return map->rnum;
}


/* Get the list hanele contains all keys in a map. */
CBLIST *cbmapkeys(CBMAP *map){
  CBLIST *list;
  const char *kbuf;
  int ksiz;
  assert(map);
  list = cblistopen();
  cbmapiterinit(map);
  while((kbuf = cbmapiternext(map, &ksiz)) != NULL){
    cblistpush(list, kbuf, ksiz);
  }
  return list;
}


/* Get the list hanele contains all values in a map. */
CBLIST *cbmapvals(CBMAP *map){
  CBLIST *list;
  const char *kbuf, *vbuf;
  int ksiz, vsiz;
  assert(map);
  list = cblistopen();
  cbmapiterinit(map);
  while((kbuf = cbmapiternext(map, &ksiz)) != NULL){
    vbuf = cbmapget(map, kbuf, ksiz, &vsiz);
    cblistpush(list, vbuf, vsiz);
  }
  return list;
}


/* Serialize a map into a byte array. */
char *cbmapdump(CBMAP *map, int *sp){
  char *buf, vnumbuf[CB_VNUMBUFSIZ];
  const char *kbuf, *vbuf;
  int bsiz, vnumsiz, rn, ksiz, vsiz;
  assert(map && sp);
  rn = cbmaprnum(map);
  vnumsiz = cbsetvnumbuf(vnumbuf, rn);
  buf = cbmalloc(vnumsiz + 1);
  memcpy(buf, vnumbuf, vnumsiz);
  bsiz = vnumsiz;
  cbmapiterinit(map);
  while((kbuf = cbmapiternext(map, &ksiz)) != NULL){
    vbuf = cbmapget(map, kbuf, ksiz, &vsiz);
    vnumsiz = cbsetvnumbuf(vnumbuf, ksiz);
    buf = cbrealloc(buf, bsiz + vnumsiz + ksiz + 1);
    memcpy(buf + bsiz, vnumbuf, vnumsiz);
    bsiz += vnumsiz;
    memcpy(buf + bsiz, kbuf, ksiz);
    bsiz += ksiz;
    vnumsiz = cbsetvnumbuf(vnumbuf, vsiz);
    buf = cbrealloc(buf, bsiz + vnumsiz + vsiz + 1);
    memcpy(buf + bsiz, vnumbuf, vnumsiz);
    bsiz += vnumsiz;
    memcpy(buf + bsiz, vbuf, vsiz);
    bsiz += vsiz;
  }
  *sp = bsiz;
  return buf;
}


/* Redintegrate a serialized map. */
CBMAP *cbmapload(const char *ptr, int size){
  CBMAP *map;
  const char *rp, *kbuf, *vbuf;
  int i, step, rn, ksiz, vsiz;
  assert(ptr && size >= 0);
  map = cbmapopen();
  rp = ptr;
  rn = cbreadvnumbuf(rp, size, &step);
  rp += step;
  size -= step;
  if(rn > size) return map;
  for(i = 0; i < rn; i++){
    if(size < 1) break;
    ksiz = cbreadvnumbuf(rp, size, &step);
    rp += step;
    size -= step;
    if(ksiz > size) break;
    kbuf = rp;
    rp += ksiz;
    if(size < 1) break;
    vsiz = cbreadvnumbuf(rp, size, &step);
    rp += step;
    size -= step;
    if(vsiz > size) break;
    vbuf = rp;
    rp += vsiz;
    cbmapput(map, kbuf, ksiz, vbuf, vsiz, TRUE);
  }
  return map;
}


/* Allocate a formatted string on memory. */
char *cbsprintf(const char *format, ...){
  va_list ap;
  char *buf, cbuf[CB_SPBUFSIZ], *str;
  int len, cblen, num, slen;
  unsigned int unum;
  double dnum;
  va_start(ap, format);
  assert(format);
  buf = cbmalloc(1);
  len = 0;
  while(*format != '\0'){
    if(*format == '%'){
      cbuf[0] = '%';
      cblen = 1;
      format++;
      while(strchr("0123456789 .+-", *format) && *format != '\0' && cblen < CB_SPBUFSIZ - 1){
        cbuf[cblen++] = *format;
        format++;
      }
      cbuf[cblen] = '\0';
      if(atoi(cbuf + 1) > CB_SPMAXWIDTH - 16){
        sprintf(cbuf, "(err)");
      } else {
        cbuf[cblen++] = *format;
        cbuf[cblen] = '\0';
      }
      switch(*format){
      case 'd':
        num = va_arg(ap, int);
        buf = cbrealloc(buf, len + CB_SPMAXWIDTH + 1);
        len += sprintf(buf + len, cbuf, num);
        break;
      case 'o': case 'u': case 'x': case 'X': case 'c':
        unum = va_arg(ap, unsigned int);
        buf = cbrealloc(buf, len + CB_SPMAXWIDTH + 1);
        len += sprintf(buf + len, cbuf, unum);
        break;
      case 'e': case 'E': case 'f': case 'g': case 'G':
        dnum = va_arg(ap, double);
        buf = cbrealloc(buf, len + CB_SPMAXWIDTH + 1);
        len += sprintf(buf + len, cbuf, dnum);
        break;
      case 's':
        str = va_arg(ap, char *);
        slen = strlen(str);
        buf = cbrealloc(buf, len + slen + 1);
        memcpy(buf + len, str, slen);
        len += slen;
        break;
      case '%':
        buf = cbrealloc(buf, len + 1);
        buf[len++] = '%';
        break;
      default:
        break;
      }
    } else {
      buf = cbrealloc(buf, len + 1);
      buf[len++] = *format;
    }
    format++;
  }
  buf[len] = '\0';
  va_end(ap);
  return buf;
}


/* Replace some patterns in a string. */
char *cbreplace(const char *str, CBMAP *pairs){
  int i, bsiz, wi, rep, ksiz, vsiz;
  char *buf;
  const char *key, *val;
  assert(str && pairs);
  bsiz = CB_DATUMUNIT;
  buf = cbmalloc(bsiz);
  wi = 0;
  while(*str != '\0'){
    rep = FALSE;
    cbmapiterinit(pairs);
    while((key = cbmapiternext(pairs, &ksiz)) != NULL){
      for(i = 0; i < ksiz; i++){
        if(str[i] == '\0' || str[i] != key[i]) break;
      }
      if(i == ksiz){
        val = cbmapget(pairs, key, ksiz, &vsiz);
        if(wi + vsiz >= bsiz){
          bsiz = bsiz * 2 + vsiz;
          buf = cbrealloc(buf, bsiz);
        }
        memcpy(buf + wi, val, vsiz);
        wi += vsiz;
        str += ksiz;
        rep = TRUE;
        break;
      }
    }
    if(!rep){
      if(wi + 1 >= bsiz){
        bsiz = bsiz * 2 + 1;
        buf = cbrealloc(buf, bsiz);
      }
      buf[wi++] = *str;
      str++;
    }
  }
  buf = cbrealloc(buf, wi + 1);
  buf[wi] = '\0';
  return buf;
}


/* Make a list by split a serial datum. */
CBLIST *cbsplit(const char *ptr, int size, const char *delim){
  CBLIST *list;
  int bi, step;
  list = cblistopen();
  if(ptr){
    if(size < 0) size = strlen(ptr);
    if(delim){
      for(bi = 0; bi < size; bi += step){
        step = 0;
        while(!strchr(delim, ptr[bi+step])){
          step++;
        }
        cblistpush(list, ptr + bi, step);
        step++;
      }
      if(size > 0 && strchr(delim, ptr[size-1])) cblistpush(list, "", 0);
    } else {
      for(bi = 0; bi < size; bi += step){
        step = 0;
        while(ptr[bi+step]){
          step++;
        }
        cblistpush(list, ptr + bi, step);
        step++;
      }
      if(size > 0 && ptr[size-1] == 0) cblistpush(list, "", 0);
    }
  }
  return list;
}


/* Read whole data of a file. */
char *cbreadfile(const char *name, int *sp){
  int fd, size, rv;
  char iobuf[CB_IOBUFSIZ], *buf;
  assert(name);
  if((fd = open(name, O_RDONLY, 0)) == -1) return NULL;
  buf = cbmalloc(1);
  size = 0;
  while((rv = read(fd, iobuf, CB_IOBUFSIZ)) > 0){
    buf = cbrealloc(buf, size + rv + 1);
    memcpy(buf + size, iobuf, rv);
    size += rv;