curia.c

harvest是一个下载html网页得机器人

int crfatalerror(CURIA *curia){
  int i;
  assert(curia);
  if(dpfatalerror(curia->attr)) return TRUE;
  for(i = 0; i < curia->dnum; i++){
    if(dpfatalerror(curia->depots[i])) return TRUE;
  }
  return FALSE;
}


/* Get the inode number of a database directory. */
int crinode(CURIA *curia){
  assert(curia);
  return curia->inode;
}


/* Remove a database directory. */
int crremove(const char *name){
  struct stat sbuf;
  CURIA *curia;
  char path[CR_PATHBUFSIZ];
  assert(name);
  if(stat(name, &sbuf) == -1){
    dpecode = DP_ESTAT;
    return FALSE;
  }
  if((curia = cropen(name, CR_OWRITER | CR_OTRUNC, 1, 1)) != NULL) crclose(curia);
  sprintf(path, "%s%c0001%c%s", name, MYPATHCHR, MYPATHCHR, CR_DPNAME);
  dpremove(path);
  sprintf(path, "%s%c0001", name, MYPATHCHR);
  if(rmdir(path) == -1){
    dpecode = DP_ERMDIR;
    return FALSE;
  }
  sprintf(path, "%s%c%s", name, MYPATHCHR, CR_DPNAME);
  if(!dpremove(path)) return FALSE;
  if(rmdir(name) == -1){
    dpecode = DP_ERMDIR;
    return FALSE;
  }
  return TRUE;
}


/* Store a large object. */
int crputlob(CURIA *curia, const char *kbuf, int ksiz, const char *vbuf, int vsiz, int dmode){
  char *path;
  int mode, fd, err, be;
  struct stat sbuf;
  assert(curia && kbuf && vbuf);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  if(ksiz < 0) ksiz = strlen(kbuf);
  if(vsiz < 0) vsiz = strlen(vbuf);
  if(!(path = crgetlobpath(curia, kbuf, ksiz))) return FALSE;
  if(!crmklobdir(path)){
    free(path);
    return FALSE;
  }
  be = stat(path, &sbuf) != -1 && S_ISREG(sbuf.st_mode);
  mode = O_RDWR | O_CREAT;
  if(dmode & CR_DKEEP) mode |= O_EXCL;
  if(dmode & CR_DCAT){
    mode |= O_APPEND;
  } else {
    mode |= O_TRUNC;
  }
  if((fd = open(path, mode, CR_FILEMODE)) == -1){
    free(path);
    dpecode = DP_EOPEN;
    if(dmode == CR_DKEEP) dpecode = DP_EKEEP;
    return FALSE;
  }
  free(path);
  err = FALSE;
  if(crwrite(fd, vbuf, vsiz) == -1){
    err = TRUE;
    dpecode = DP_EWRITE;
  }
  if(close(fd) == -1){
    err = TRUE;
    dpecode = DP_ECLOSE;
  }
  if(!err && !be) (curia->lrnum)++;
  return err ? FALSE : TRUE;
}


/* Delete a large object. */
int croutlob(CURIA *curia, const char *kbuf, int ksiz){
  char *path;
  int err, be;
  struct stat sbuf;
  assert(curia && kbuf);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  if(ksiz < 0) ksiz = strlen(kbuf);
  if(!(path = crgetlobpath(curia, kbuf, ksiz))) return FALSE;
  be = stat(path, &sbuf) != -1 && S_ISREG(sbuf.st_mode);
  err = FALSE;
  if(unlink(path) == -1){
    err = TRUE;
    dpecode = DP_ENOITEM;
  }
  free(path);
  if(!err && be) (curia->lrnum)--;
  return err ? FALSE : TRUE;
}


/* Retrieve a large object. */
char *crgetlob(CURIA *curia, const char *kbuf, int ksiz, int start, int max, int *sp){
  char *path, *buf;
  struct stat sbuf;
  int fd, size;
  assert(curia && kbuf && start >= 0);
  if(ksiz < 0) ksiz = strlen(kbuf);
  if(!(path = crgetlobpath(curia, kbuf, ksiz))) return NULL;
  if((fd = open(path, O_RDONLY, CR_FILEMODE)) == -1){
    free(path);
    dpecode = DP_ENOITEM;
    return NULL;
  }
  free(path);
  if(fstat(fd, &sbuf) == -1){
    close(fd);
    dpecode = DP_ESTAT;
    return NULL;
  }
  if(start > sbuf.st_size){
    close(fd);
    dpecode = DP_ENOITEM;
    return NULL;
  }
  if(lseek(fd, start, SEEK_SET) == -1){
    close(fd);
    dpecode = DP_ESEEK;
    return NULL;
  }
  if(max < 0) max = sbuf.st_size;
  if(!(buf = malloc(max + 1))){
    close(fd);
    dpecode = DP_EALLOC;
    return NULL;
  }
  size = crread(fd, buf, max);
  close(fd);
  if(size == -1){
    free(buf);
    dpecode = DP_EREAD;
    return NULL;
  }
  buf[size] = '\0';
  if(sp) *sp = size;
  return buf;
}


/* Get the size of the value of a large object. */
int crvsizlob(CURIA *curia, const char *kbuf, int ksiz){
  char *path;
  struct stat sbuf;
  assert(curia && kbuf);
  if(ksiz < 0) ksiz = strlen(kbuf);
  if(!(path = crgetlobpath(curia, kbuf, ksiz))) return -1;
  if(stat(path, &sbuf) == -1){
    free(path);
    dpecode = DP_ENOITEM;
    return -1;
  }
  free(path);
  return sbuf.st_size;
}


/* Get the number of the large objects stored in a database. */
int crrnumlob(CURIA *curia){
  assert(curia);
  return curia->lrnum;
}



/*************************************************************************************************
 * Functions for Experts
 *************************************************************************************************/


/* Synchronize updating contents on memory. */
int crmemsync(CURIA *curia){
  int i, err;
  assert(curia);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  err = FALSE;
  if(!dpput(curia->attr, CR_KEYLRNUM, -1, (char *)&(curia->lrnum), sizeof(int), DP_DOVER) ||
     !dpmemsync(curia->attr)) err = TRUE;
  for(i = 0; i < curia->dnum; i++){
    if(!dpmemsync(curia->depots[i])){
      err = TRUE;
      break;
    }
  }
  return err ? FALSE : TRUE;
}


/* Get flags of a database. */
int crgetflags(CURIA *curia){
  assert(curia);
  return dpgetflags(curia->attr);
}


/* Set flags of a database. */
int crsetflags(CURIA *curia, int flags){
  assert(curia);
  if(!curia->wmode){
    dpecode = DP_EMODE;
    return FALSE;
  }
  return dpsetflags(curia->attr, flags);
}



/*************************************************************************************************
 * private objects
 *************************************************************************************************/


/* Get a copied string.
   `str' specifies an original string.
   The return value is a copied string whose region is allocated by `malloc'. */
static char *crstrdup(const char *str){
  int len;
  char *buf;
  assert(str);
  len = strlen(str);
  if(!(buf = malloc(len + 1))) return NULL;
  memcpy(buf, str, len + 1);
  return buf;
}


/* Get an integer from a database.
   `depot' specifies an inner database handle.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the key.
   The return value is the integer of the corresponding record. */
static int crdpgetnum(DEPOT *depot, const char *kbuf, int ksiz){
  char *vbuf;
  int vsiz, rv;
  if(!(vbuf = dpget(depot, kbuf, ksiz, 0, -1, &vsiz)) || vsiz != sizeof(int)){
    free(vbuf);
    return INT_MIN;
  }
  rv = *(int *)vbuf;
  free(vbuf);
  return rv;
}


/* Get the path of a large object.
   `curia' specifies a database handle.
   `kbuf' specifies the pointer to the region of a key.
   `ksiz' specifies the size of the key.
   The return value is a path string whose region is allocated by `malloc'. */
static char *crgetlobpath(CURIA *curia, const char *kbuf, int ksiz){
  char prefix[CR_PATHBUFSIZ], *wp, *path;
  int i, hash;
  assert(curia && kbuf && ksiz >= 0);
  wp = prefix;
  wp += sprintf(wp, "%s%c%04d%c%s%c",
                curia->name, MYPATHCHR, dpouterhash(kbuf, ksiz) % curia->dnum + 1,
                MYPATHCHR, CR_LOBDIR, MYPATHCHR);
  hash = dpinnerhash(kbuf, ksiz);
  for(i = 0; i < CR_LOBDDEPTH; i++){
    wp += sprintf(wp, "%02X%c", hash % 0x100, MYPATHCHR);
    hash /= 0x100;
  }
  if(!(path = malloc(strlen(prefix) + ksiz * 2 + 1))){
    dpecode = DP_EALLOC;
    return NULL;
  }
  wp = path;
  wp += sprintf(path, "%s", prefix);
  for(i = 0; i < ksiz; i++){
    wp += sprintf(wp, "%02X", ((unsigned char *)kbuf)[i]);
  }
  return path;
}


/* Create directories included in a path.
   `path' specifies a path.
   The return value is true if successful, else, it is false. */
static int crmklobdir(const char *path){
  char elem[CR_PATHBUFSIZ], *wp;
  const char *dp;
  int err, len;
  wp = elem;
  err = FALSE;
  while(*path != '\0' && (dp = strchr(path, MYPATHCHR)) != NULL){
    len = dp - path;
    if(wp != elem) wp += sprintf(wp, "%c", MYPATHCHR);
    memcpy(wp, path, len);
    wp[len] = '\0';
    wp += len;
    if(mkdir(elem, CR_DIRMODE) == -1 && errno != EEXIST) err = TRUE;
    path = dp + 1;
  }
  if(err) dpecode = DP_EMKDIR;
  return err ? FALSE : TRUE;
}


/* Remofe file and directories under a directory.
   `path' specifies a path.
   The return value is true if successful, else, it is false. */
static int crrmlobdir(const char *path){
  char elem[CR_PATHBUFSIZ];
  DIR *DD;
  struct dirent *dp;
  assert(path);
  if(unlink(path) != -1){
    return TRUE;
  } else {
    if(errno == ENOENT) return TRUE;
    if(!(DD = opendir(path))){
      dpecode = DP_EMISC;
      return FALSE;
    }
    while((dp = readdir(DD)) != NULL){
      if(!strcmp(dp->d_name, MYCDIRSTR) || !strcmp(dp->d_name, MYPDIRSTR)) continue;
      sprintf(elem, "%s%c%s", path, MYPATHCHR, dp->d_name);
      if(!crrmlobdir(elem)){
        closedir(DD);
        return FALSE;
      }
    }
  }
  if(closedir(DD) == -1){
    dpecode = DP_EMISC;
    return FALSE;
  }
  if(rmdir(path) == -1){
    dpecode = DP_ERMDIR;
    return FALSE;
  }
  return TRUE;
}


/* Write into a file.
   `fd' specifies a file descriptor.
   `buf' specifies a buffer to write.
   `size' specifies the size of the buffer.
   The return value is the size of the written buffer, or, -1 on failure. */
static int crwrite(int fd, const void *buf, int size){
  char *lbuf;
  int rv, wb;
  assert(fd >= 0 && buf && size >= 0);
  lbuf = (char *)buf;
  rv = 0;
  do {
    wb = write(fd, lbuf, size);
    switch(wb){
    case -1: if(errno != EINTR) return -1;
    case 0: break;
    default:
      lbuf += wb;
      size -= wb;
      rv += wb;
      break;
    }
  } while(size > 0);
  return rv;
}


/* Read from a file and store the data into a buffer.
   `fd' specifies a file descriptor.
   `buffer' specifies a buffer to store into.
   `size' specifies the size to read with.
   The return value is the size read with, or, -1 on failure. */
static int crread(int fd, void *buf, int size){
  char *lbuf;
  int i, bs;
  assert(fd >= 0 && buf && size >= 0);
  lbuf = buf;
  for(i = 0; i < size && (bs = read(fd, lbuf + i, size - i)) != 0; i += bs){
    if(bs == -1 && errno != EINTR) return -1;
  }
  return i;
}



/* END OF FILE */