depot.c

harvest是一个下载html网页得机器人

  if(size < 1) return TRUE;
  lbuf = (char *)buf;
  if(off < 0){
    if(lseek(fd, 0, SEEK_END) == -1){
      dpecode = DP_ESEEK;
      return FALSE;
    }
  } else {
    if(lseek(fd, off, SEEK_SET) != off){
      dpecode = DP_ESEEK;
      return FALSE;
    }
  }
  if(dpwrite(fd, lbuf, size) != size){
    dpecode = DP_EWRITE;
    return FALSE;
  }
  return TRUE;
}


/* Write an integer into a file at an offset.
   `fd' specifies a file descriptor.
   `off' specifies an offset of the file.
   `num' specifies an integer.
   The return value is true if successful, else, it is false. */
static int dpseekwritenum(int fd, int off, int num){
  assert(fd >= 0);
  return dpseekwrite(fd, off, &num, sizeof(int));
}


/* Read from a file and store the data into a buffer.
   `fd' specifies a file descriptor.
   `buffer' specifies a buffer to store into.
   `size' specifies the size to read with.
   The return value is the size read with, or, -1 on failure. */
static int dpread(int fd, void *buf, int size){
  char *lbuf;
  int i, bs;
  assert(fd >= 0 && buf && size >= 0);
  lbuf = buf;
  for(i = 0; i < size && (bs = read(fd, lbuf + i, size - i)) != 0; i += bs){
    if(bs == -1 && errno != EINTR) return -1;
  }
  return i;
}


/* Read from a file at an offset and store the data into a buffer.
   `fd' specifies a file descriptor.
   `off' specifies an offset of the file.
   `buffer' specifies a buffer to store into.
   `size' specifies the size to read with.
   The return value is true if successful, else, it is false. */
static int dpseekread(int fd, int off, void *buf, int size){
  char *lbuf;
  assert(fd >= 0 && off >= 0 && buf && size >= 0);
  lbuf = (char *)buf;
  if(lseek(fd, off, SEEK_SET) != off){
    dpecode = DP_ESEEK;
    return FALSE;
  }
  if(dpread(fd, lbuf, size) != size){
    dpecode = DP_EREAD;
    return FALSE;
  }
  return TRUE;
}


/* Copy data between files.
   `destfd' specifies a file descriptor of a destination file.
   `destoff' specifies an offset of the destination file.
   `srcfd' specifies a file descriptor of a source file.
   `srcoff' specifies an offset of the source file.
   The return value is the size copied with, or, -1 on failure. */
static int dpfcopy(int destfd, int destoff, int srcfd, int srcoff){
  char iobuf[DP_IOBUFSIZ];
  int sum, iosiz;
  if(lseek(srcfd, srcoff, SEEK_SET) == -1 || lseek(destfd, destoff, SEEK_SET) == -1){
    dpecode = DP_ESEEK;
    return -1;
  }
  sum = 0;
  while((iosiz = dpread(srcfd, iobuf, DP_IOBUFSIZ)) > 0){
    if(dpwrite(destfd, iobuf, iosiz) == -1){
      dpecode = DP_EWRITE;
      return -1;
    }
    sum += iosiz;
  }
  if(iosiz < 0){
    dpecode = DP_EREAD;
    return -1;
  }
  return sum;
}


/* Get the first hash value.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the key.
   The return value is 31 bit hash value of the key. */
static int dpfirsthash(const char *kbuf, int ksiz){
  unsigned int sum;
  int i;
  assert(kbuf && ksiz >= 0);
  if(ksiz == sizeof(int)){
    memcpy(&sum, kbuf, sizeof(int));
  } else {
    sum = 751;
  }
  for(i = 0; i < ksiz; i++){
    sum = sum * 31 + ((const unsigned char *)kbuf)[i];
  }
  return (sum * 87767623) & 0x7FFFFFFF;
}


/* Get the second hash value.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the key.
   The return value is 31 bit hash value of the key. */
static int dpsecondhash(const char *kbuf, int ksiz){
  unsigned int sum;
  int i;
  assert(kbuf && ksiz >= 0);
  sum = 19780211;
  for(i = ksiz - 1; i >= 0; i--){
    sum = sum * 37 + ((const unsigned char *)kbuf)[i];
  }
  return (sum * 43321879) & 0x7FFFFFFF;
}


/* Get the third hash value.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the key.
   The return value is 31 bit hash value of the key. */
static int dpthirdhash(const char *kbuf, int ksiz){
  unsigned int sum;
  int i;
  assert(kbuf && ksiz >= 0);
  sum = 774831917;
  for(i = ksiz - 1; i >= 0; i--){
    sum = sum * 29 + ((const unsigned char *)kbuf)[i];
  }
  return (sum * 5157883) & 0x7FFFFFFF;
}


/* Get a natural prime number not less than a number.
   `num' specified a natural number.
   The return value is a prime number not less than the specified number. */
static int dpgetprime(int num){
  int primes[] = {
    1, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 43, 47, 53, 59, 61, 71, 79, 83,
    89, 103, 109, 113, 127, 139, 157, 173, 191, 199, 223, 239, 251, 283, 317, 349,
    383, 409, 443, 479, 509, 571, 631, 701, 761, 829, 887, 953, 1021, 1151, 1279,
    1399, 1531, 1663, 1789, 1913, 2039, 2297, 2557, 2803, 3067, 3323, 3583, 3833,
    4093, 4603, 5119, 5623, 6143, 6653, 7159, 7673, 8191, 9209, 10223, 11261,
    12281, 13309, 14327, 15359, 16381, 18427, 20479, 22511, 24571, 26597, 28669,
    30713, 32749, 36857, 40949, 45053, 49139, 53239, 57331, 61417, 65521, 73727,
    81919, 90107, 98299, 106487, 114679, 122869, 131071, 147451, 163819, 180221,
    196597, 212987, 229373, 245759, 262139, 294911, 327673, 360439, 393209, 425977,
    458747, 491503, 524287, 589811, 655357, 720887, 786431, 851957, 917503, 982981,
    1048573, 1179641, 1310719, 1441771, 1572853, 1703903, 1835003, 1966079,
    2097143, 2359267, 2621431, 2883577, 3145721, 3407857, 3670013, 3932153,
    4194301, 4718579, 5242877, 5767129, 6291449, 6815741, 7340009, 7864301,
    8388593, 9437179, 10485751, 11534329, 12582893, 13631477, 14680063, 15728611,
    16777213, 18874367, 20971507, 23068667, 25165813, 27262931, 29360087, 31457269,
    33554393, 37748717, 41943023, 46137319, 50331599, 54525917, 58720253, 62914549,
    67108859, 75497467, 83886053, 92274671, 100663291, 109051903, 117440509,
    125829103, 134217689, 150994939, 167772107, 184549373, 201326557, 218103799,
    234881011, 251658227, 268435399, 301989881, 335544301, 369098707, 402653171,
    436207613, 469762043, 503316469, 536870909, 603979769, 671088637, 738197503,
    805306357, 872415211, 939524087, 1006632947, 1073741789, 1207959503,
    1342177237, 1476394991, 1610612711, 1744830457, 1879048183, 2013265907, -1
  };
  int i;
  assert(num > 0);
  for(i = 0; primes[i] > 0; i++){
    if(num <= primes[i]) return primes[i];
  }
  return primes[i-1];
}


/* Get the padding size of a record.
   `vsiz' specifies the size of the value of a record.
   The return value is the padding size of a record. */
static int dppadsize(DEPOT *depot, int vsiz){
  int mod;
  assert(depot && vsiz >= 0);
  if(depot->align > 0){
    mod = vsiz % depot->align;
    return mod == 0 ? 0 : depot->align - mod;
  } else if(depot->align < 0){
    return (int)(vsiz * (2.0 / (1 << -(depot->align))));
  }
  return 0;
}


/* Get the size of a record in a database file.
   `head' specifies the header of  a record.
   The return value is the size of a record in a database file. */
static int dprecsize(int *head){
  assert(head);
  return DP_RHNUM * sizeof(int) + head[DP_RHIKSIZ] + head[DP_RHIVSIZ] + head[DP_RHIPSIZ];
}


/* Read the header of a record.
   `depot' specifies a database handle.
   `off' specifies an offset of the database file.
   `head' specifies a buffer for the header.
   The return value is true if successful, else, it is false. */
static int dprechead(DEPOT *depot, int off, int *head){
  assert(depot && off >= 0 && head);
  if(off > depot->fsiz){
    dpecode = DP_EBROKEN;
    return FALSE;
  }
  if(!dpseekread(depot->fd, off, head, DP_RHNUM * sizeof(int))) return FALSE;
  if(head[DP_RHIKSIZ] < 0 || head[DP_RHIVSIZ] < 0 || DP_RHIPSIZ < 0 ||
     head[DP_RHILEFT] < 0 || head[DP_RHIRIGHT] < 0){
    dpecode = DP_EBROKEN;
    return FALSE;
  }
  return TRUE;
}


/* Read the entitiy of the key of a record.
   `depot' specifies a database handle.
   `off' specifies an offset of the database file.
   `head' specifies the header of a record.
   The return value is a key data whose region is allocated by `malloc', or NULL on failure. */
static char *dpreckey(DEPOT *depot, int off, int *head){
  char *kbuf;
  int ksiz;
  assert(depot && off >= 0);
  ksiz = head[DP_RHIKSIZ];
  if(!(kbuf = malloc(ksiz + 1))){
    dpecode = DP_EALLOC;
    return NULL;
  }
  if(!dpseekread(depot->fd, off + DP_RHNUM * sizeof(int), kbuf, ksiz)){
    free(kbuf);
    return NULL;
  }
  kbuf[ksiz] = '\0';
  return kbuf;
}


/* Read the entitiy of the value of a record.
   `depot' specifies a database handle.
   `off' specifies an offset of the database file.
   `head' specifies the header of a record.
   `start' specifies the offset address of the beginning of the region of the value to be read.
   `max' specifies the max size to be read.  If it is negative, the size to read is unlimited.
   The return value is a value data whose region is allocated by `malloc', or NULL on failure. */
static char *dprecval(DEPOT *depot, int off, int *head, int start, int max){
  char *vbuf;
  int vsiz;
  assert(depot && off >= 0 && start >= 0);
  head[DP_RHIVSIZ] -= start;
  if(max < 0){
    vsiz = head[DP_RHIVSIZ];
  } else {
    vsiz = max < head[DP_RHIVSIZ] ? max : head[DP_RHIVSIZ];
  }
  if(!(vbuf = malloc(vsiz + 1))){
    dpecode = DP_EALLOC;
    return NULL;
  }
  if(!dpseekread(depot->fd, off + DP_RHNUM * sizeof(int) + head[DP_RHIKSIZ] + start, vbuf, vsiz)){
    free(vbuf);
    return NULL;
  }
  vbuf[vsiz] = '\0';
  return vbuf;
}


/* Compare two keys.
   `abuf' specifies the pointer to the region of the former.
   `asiz' specifies the size of the region.
   `bbuf' specifies the pointer to the region of the latter.
   `bsiz' specifies the size of the region.
   The return value is 0 if two equals, positive if the formar is big, else, negative. */
static int dpkeycmp(const char *abuf, int asiz, const char *bbuf, int bsiz){
  assert(abuf && asiz >= 0 && bbuf && bsiz >= 0);
  if(asiz > bsiz) return 1;
  if(asiz < bsiz) return -1;
  return memcmp(abuf, bbuf, asiz);
}


/* Search a record.
   `depot' specifies a database handle.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the region.
   `hash' specifies the second hash value of the key.
   `bip' specifies the pointer to the region to assign the index of the corresponding record.
   `offp' specifies the pointer to the region to assign the last visited node in the hash chain,
   or, -1 if the hash chain is empty.
   `entp' specifies the offset of the last used joint, or, -1 if the hash chain is empty.
   `head' specifies the pointer to the region to store the header of the last visited record in.
   `delhit' specifies whether a deleted record corresponds or not.
   The return value is 0 if successful, 1 if there is no corresponding record, -1 on error. */
static int dprecsearch(DEPOT *depot, const char *kbuf, int ksiz, int hash,
                       int *bip, int *offp, int *entp, int *head, int delhit){
  int off, entoff, thash, kcmp;
  char *tkey;
  assert(depot && kbuf && ksiz >= 0 && hash >= 0 && bip && offp && entp && head);
  *bip = dpfirsthash(kbuf, ksiz) % depot->bnum;
  off = depot->buckets[*bip];
  *offp = -1;
  *entp = -1;
  entoff = -1;
  while(off != 0){
    if(!dprechead(depot, off, head)) return -1;
    thash = head[DP_RHIHASH];
    if(hash > thash){
      entoff = off + DP_RHILEFT * sizeof(int);
      off = head[DP_RHILEFT];
    } else if(hash < thash){
      entoff = off + DP_RHIRIGHT * sizeof(int);
      off = head[DP_RHIRIGHT];
    } else {
      if(!(tkey = dpreckey(depot, off, head))) return -1;
      kcmp = dpkeycmp(kbuf, ksiz, tkey, head[DP_RHIKSIZ]);
      free(tkey);
      if(kcmp > 0){
        entoff = off + DP_RHILEFT * sizeof(int);
        off = head[DP_RHILEFT];
      } else if(kcmp < 0){
        entoff = off + DP_RHIRIGHT * sizeof(int);
        off = head[DP_RHIRIGHT];
      } else {
        if(!delhit && (head[DP_RHIFLAGS] & DP_RECFDEL)){
          entoff = off + DP_RHILEFT * sizeof(int);
          off = head[DP_RHILEFT];
        } else {
          *offp = off;
          *entp = entoff;
          return 0;
        }
      }
    }
  }
  *offp = off;
  *entp = entoff;
  return 1;
}


/* Overwrite a record.
   `depot' specifies a database handle.
   `off' specifies the offset of the database file.
   `rsiz' specifies the size of the existing record.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the region.
   `vbuf' specifies the pointer to the region of a value.
   `vsiz' specifies the size of the region.
   `hash' specifies the second hash value of the key.
   `left' specifies the offset of the left child.
   `right' specifies the offset of the right child.
   The return value is true if successful, or, false on failure. */
static int dprecrewrite(DEPOT *depot, int off, int rsiz, const char *kbuf, int ksiz,
                        const char *vbuf, int vsiz, int hash, int left, int right){
  int head[DP_RHNUM], psiz, hoff, koff, voff, c;
  assert(depot && off >= 1 && rsiz > 0 && kbuf && ksiz >= 0 && vbuf && vsiz >= 0);
  psiz = rsiz - sizeof(head) - ksiz - vsiz;
  head[DP_RHIFLAGS] = 0;
  head[DP_RHIHASH] = hash;
  head[DP_RHIKSIZ] = ksiz;
  head[DP_RHIVSIZ] = vsiz;
  head[DP_RHIPSIZ] = psiz;
  head[DP_RHILEFT] = left;
  head[DP_RHIRIGHT] = right;
  hoff = off;
  koff = hoff + sizeof(head);
  voff = koff + ksiz;
  c = 0;
  if(!dpseekwrite(depot->fd, hoff, head, sizeof(head)) ||
     !dpseekwrite(depot->fd, koff, kbuf, ksiz) ||
     !dpseekwrite(depot->fd, voff, vbuf, vsiz) ||
     (psiz > 0 && !dpseekwrite(depot->fd, voff + vsiz + psiz - 1, &c, 1))) return FALSE;
  return TRUE;
}


/* Write a record at the end of a database file.
   `depot' specifies a database handle.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the region.
   `vbuf' specifies the pointer to the region of a value.
   `vsiz' specifies the size of the region.
   `hash' specifies the second hash value of the key.
   `left' specifies the offset of the left child.
   `right' specifies the offset of the right child.
   The return value is the offset of the record, or, -1 on failure. */
static int dprecappend(DEPOT *depot, const char *kbuf, int ksiz, const char *vbuf, int vsiz,
                       int hash, int left, int right){
  int head[DP_RHNUM], psiz, off, hoff, koff, voff, c;
  assert(depot && kbuf && ksiz >= 0 && vbuf && vsiz >= 0);
  psiz = dppadsize(depot, vsiz);
  head[DP_RHIFLAGS] = 0;
  head[DP_RHIHASH] = hash;
  head[DP_RHIKSIZ] = ksiz;
  head[DP_RHIVSIZ] = vsiz;
  head[DP_RHIPSIZ] = psiz;
  head[DP_RHILEFT] = left;
  head[DP_RHIRIGHT] = right;
  off = depot->fsiz;
  hoff = off;
  koff = hoff + sizeof(head);
  voff = koff + ksiz;
  c = 0;
  if(!dpseekwrite(depot->fd, hoff, head, sizeof(head)) ||
     !dpseekwrite(depot->fd, koff, kbuf, ksiz) ||
     !dpseekwrite(depot->fd, voff, vbuf, vsiz) ||
     (psiz > 0 && !dpseekwrite(depot->fd, voff + vsiz + psiz - 1, &c, 1))) return -1;
  depot->fsiz += sizeof(head) + vsiz + ksiz + psiz;
  return off;
}


/* Overwrite the value of a record.
   `depot' specifies a database handle.
   `off' specifies the offset of the database file.
   `head' specifies the header of the record.
   `vbuf' specifies the pointer to the region of a value.
   `vsiz' specifies the size of the region.
   `cat' specifies whether it is concatenate mode or not.
   The return value is true if successful, or, false on failure. */
static int dprecover(DEPOT *depot, int off, int *head, const char *vbuf, int vsiz, int cat){
  int hsiz, hoff, voff;
  assert(depot && off >= 0 && head && vbuf && vsiz >= 0);
  if(depot->mroff == off) depot->mroff = -1;
  if(cat){
    head[DP_RHIFLAGS] = 0;
    head[DP_RHIPSIZ] -= vsiz;
    head[DP_RHIVSIZ] += vsiz;
    hsiz = DP_RHNUM * sizeof(int);
    hoff = off;
    voff = hoff + DP_RHNUM * sizeof(int) + head[DP_RHIKSIZ] + head[DP_RHIVSIZ] - vsiz;
  } else {
    head[DP_RHIFLAGS] = 0;
    head[DP_RHIPSIZ] += head[DP_RHIVSIZ] - vsiz;
    head[DP_RHIVSIZ] = vsiz;
    hsiz = DP_RHNUM * sizeof(int);
    hoff = off;
    voff = hoff + DP_RHNUM * sizeof(int) + head[DP_RHIKSIZ];
  }
  if(!dpseekwrite(depot->fd, hoff, head, hsiz) ||
     !dpseekwrite(depot->fd, voff, vbuf, vsiz)) return FALSE;
  return TRUE;
}


/* Delete a record.
   `depot' specifies a database handle.
   `off' specifies the offset of the database file.
   `head' specifies the header of the record.
   `reusable' specifies whether the region is reusable or not.
   The return value is true if successful, or, false on failure. */
static int dprecdelete(DEPOT *depot, int off, int *head, int reusable){
  assert(depot && off >= 0 && head);
  if(reusable){
    depot->mroff = off;
    depot->mrsiz = dprecsize(head);
  }
  return dpseekwritenum(depot->fd, off + DP_RHIFLAGS * sizeof(int),
                        DP_RECFDEL | (reusable ? DP_RECFREUSE : 0));
}



/* END OF FILE */