odmgr.c

harvest是一个下载html网页得机器人

/*************************************************************************************************
 * Utility for debugging Odeum and its applications
 *                                                      Copyright (C) 2000-2003 Mikio Hirabayashi
 * This file is part of QDBM, Quick Database Manager.
 * QDBM is free software; you can redistribute it and/or modify it under the terms of the GNU
 * Lesser General Public License as published by the Free Software Foundation; either version
 * 2.1 of the License or any later version.  QDBM is distributed in the hope that it will be
 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
 * details.
 * You should have received a copy of the GNU Lesser General Public License along with QDBM; if
 * not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
 * 02111-1307 USA.
 *************************************************************************************************/


#include <depot.h>
#include <cabin.h>
#include <odeum.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#undef TRUE
#define TRUE           1                 /* boolean true */
#undef FALSE
#define FALSE          0                 /* boolean false */

#define MAXSRCHWORDS   256               /* max number of search words */
#define WOCCRPOINT     10000             /* points per occurence */
#define MAXKEYWORDS    8                 /* max number of keywords */
#define SUMMARYWIDTH   16                /* width of each phrase in a summary */
#define MAXSUMMARY     128               /* max number of words in a summary */


/* for RISC OS */
#if defined(__riscos__) || defined(__riscos)
#include <unixlib/local.h>
int __riscosify_control = __RISCOSIFY_NO_PROCESS;
#endif


/* global variables */
const char *progname;                    /* program name */


/* function prototypes */
int main(int argc, char **argv);
void usage(void);
char *readstdin(int *sp);
int runcreate(int argc, char **argv);
int runput(int argc, char **argv);
int runout(int argc, char **argv);
int runget(int argc, char **argv);
int runsearch(int argc, char **argv);
int runlist(int argc, char **argv);
int runoptimize(int argc, char **argv);
int runinform(int argc, char **argv);
int runremove(int argc, char **argv);
int runbreak(int argc, char **argv);
void pdperror(const char *name);
void printdoc(const ODDOC *doc, int tb, int hb, int score, ODEUM *odeum, const CBLIST *skeys);
char *docsummary(const ODDOC *doc, const CBLIST *kwords, int num, int hilight);
CBMAP *listtomap(const CBLIST *list);
int docreate(const char *name);
int doput(const char *name, const char *text, const char *uri, const char *title,
          const char *author, const char *date, int wmax, int keep);
int doout(const char *name, const char *uri, int id);
int doget(const char *name, const char *uri, int id, int tb, int hb);
int dosearch(const char *name, const char *text, int max, int or, int idf,
             int tb, int hb, int nb);
int dolist(const char *name, int tb, int hb);
int dooptimize(const char *name);
int doinform(const char *name);
int doremove(const char *name);
int dobreak(const char *text, int hb, int kb, int sb);


/* main routine */
int main(int argc, char **argv){
  int rv, tsiz;
  char *tmp;
  progname = argv[0];
  if(argc < 2) usage();
  rv = 0;
  if(!strcmp(argv[1], "create")){
    rv = runcreate(argc, argv);
  } else if(!strcmp(argv[1], "put")){
    rv = runput(argc, argv);
  } else if(!strcmp(argv[1], "out")){
    rv = runout(argc, argv);
  } else if(!strcmp(argv[1], "get")){
    rv = runget(argc, argv);
  } else if(!strcmp(argv[1], "search")){
    rv = runsearch(argc, argv);
  } else if(!strcmp(argv[1], "list")){
    rv = runlist(argc, argv);
  } else if(!strcmp(argv[1], "optimize")){
    rv = runoptimize(argc, argv);
  } else if(!strcmp(argv[1], "inform")){
    rv = runinform(argc, argv);
  } else if(!strcmp(argv[1], "remove")){
    rv = runremove(argc, argv);
  } else if(!strcmp(argv[1], "break")){
    rv = runbreak(argc, argv);
  } else if(!strcmp(argv[1], "version") || !strcmp(argv[1], "--version")){
    printf("Powered by QDBM version %s", dpversion);
    if((tmp = cbdeflate("", 0, &tsiz)) != NULL){
      printf(" (ZLIB enabled)");
      free(tmp);
    }
    printf("\n");
    printf("Copyright (c) 2000-2003 Mikio Hirabayashi\n");
    rv = 0;
  } else {
    usage();
  }
  return rv;
}


/* print the usage and exit */
void usage(void){
  fprintf(stderr, "%s: administration utility for Odeum\n", progname);
  fprintf(stderr, "\n");
  fprintf(stderr, "usage:\n");
  fprintf(stderr, "  %s create name\n", progname);
  fprintf(stderr, "  %s put [-uri str] [-title str] [-author str] [-date str]"
          " [-wmax num] [-keep] name [file]\n", progname);
  fprintf(stderr, "  %s out [-id] name expr\n", progname);
  fprintf(stderr, "  %s get [-id] [-t|-h] name expr\n", progname);
  fprintf(stderr, "  %s search [-max num] [-or] [-idf] [-t|-h|-n] name words...\n", progname);
  fprintf(stderr, "  %s list [-t|-h] name\n", progname);
  fprintf(stderr, "  %s optimize name\n", progname);
  fprintf(stderr, "  %s inform name\n", progname);
  fprintf(stderr, "  %s remove name\n", progname);
  fprintf(stderr, "  %s break [-h|-k|-s] [file]\n", progname);
  fprintf(stderr, "  %s version\n", progname);
  exit(1);
}


/* read the standard input */
char *readstdin(int *sp){
  char *buf;
  int i, blen, c;
  blen = 256;
  buf = cbmalloc(blen);
  for(i = 0; (c = getchar()) != EOF; i++){
    if(i >= blen - 1) buf = cbrealloc(buf, blen *= 2);
    buf[i] = c;
  }
  buf[i] = '\0';
  *sp = i;
  return buf;
}


/* parse arguments of create command */
int runcreate(int argc, char **argv){
  char *name;
  int i, rv;
  name = NULL;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      usage();
    } else if(!name){
      name = argv[i];
    } else {
      usage();
    }
  }
  if(!name) usage();
  rv = docreate(name);
  return rv;
}


/* parse arguments of put command */
int runput(int argc, char **argv){
  char *name, *file, *uri, *title, *author, *date, *text;
  int i, wmax, keep, size, rv;
  name = NULL;
  file = NULL;
  uri = NULL;
  title = NULL;
  author = NULL;
  date = NULL;
  wmax = -1;
  keep = FALSE;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-uri")){
        if(++i >= argc) usage();
        uri = argv[i];
      } else if(!strcmp(argv[i], "-uri")){
        if(++i >= argc) usage();
        uri = argv[i];
      } else if(!strcmp(argv[i], "-title")){
        if(++i >= argc) usage();
        title = argv[i];
      } else if(!strcmp(argv[i], "-author")){
        if(++i >= argc) usage();
        author = argv[i];
      } else if(!strcmp(argv[i], "-date")){
        if(++i >= argc) usage();
        date = argv[i];
      } else if(!strcmp(argv[i], "-wmax")){
        if(++i >= argc) usage();
        wmax = atoi(argv[i]);
      } else if(!strcmp(argv[i], "-keep")){
        keep = TRUE;
      } else {
        usage();
      }
    } else if(!name){
      name = argv[i];
    } else if(!file){
      file = argv[i];
    } else {
      usage();
    }
  }
  if(!name) usage();
  if(!uri) uri = file;
  if(!uri) usage();
  if(file){
    if(!(text = cbreadfile(file, &size))){
      fprintf(stderr, "%s: %s: cannot open\n", progname, file);
      return 1;
    }
  } else {
    text = readstdin(&size);
  }
  rv = doput(name, text, uri, title, author, date, wmax, keep);
  free(text);
  return rv;
}


/* parse arguments of out command */
int runout(int argc, char **argv){
  char *name, *expr;
  int i, ib, id, rv;
  name = NULL;
  expr = NULL;
  ib = FALSE;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-id")){
        ib = TRUE;
      } else {
        usage();
      }
    } else if(!name){
      name = argv[i];
    } else if(!expr){
      expr = argv[i];
    } else {
      usage();
    }
  }
  if(!name || !expr) usage();
  id = -1;
  if(ib){
    id = atoi(expr);
    if(id < 1) usage();
  }
  rv = doout(name, expr, id);
  return rv;
}


/* parse arguments of get command */
int runget(int argc, char **argv){
  char *name, *expr;
  int i, ib, tb, hb, id, rv;
  name = NULL;
  expr = NULL;
  ib = FALSE;
  tb = FALSE;
  hb = FALSE;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-id")){
        ib = TRUE;
      } else if(!strcmp(argv[i], "-t")){
        tb = TRUE;
      } else if(!strcmp(argv[i], "-h")){
        hb = TRUE;
      } else {
        usage();
      }
    } else if(!name){
      name = argv[i];
    } else if(!expr){
      expr = argv[i];
    } else {
      usage();
    }
  }
  if(!name || !expr) usage();
  id = -1;
  if(ib){
    id = atoi(expr);
    if(id < 1) usage();
  }
  rv = doget(name, expr, id, tb, hb);
  return rv;
}


/* parse arguments of search command */
int runsearch(int argc, char **argv){
  char *name, *srchwords[MAXSRCHWORDS];
  int i, wnum, max, or, idf, tb, hb, nb, rv;
  CBDATUM *text;
  name = NULL;
  wnum = 0;
  max = -1;
  or = FALSE;
  idf = FALSE;
  tb = FALSE;
  hb = FALSE;
  nb = FALSE;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-max")){
        if(++i >= argc) usage();
        max = atoi(argv[i]);
      } else if(!strcmp(argv[i], "-or")){
        or = TRUE;
      } else if(!strcmp(argv[i], "-idf")){
        idf = TRUE;
      } else if(!strcmp(argv[i], "-t")){
        tb = TRUE;
      } else if(!strcmp(argv[i], "-h")){
        hb = TRUE;
      } else if(!strcmp(argv[i], "-n")){
        nb = TRUE;
      } else {
        usage();
      }
    } else if(!name){
      name = argv[i];
    } else if(wnum < MAXSRCHWORDS){
      srchwords[wnum++] = argv[i];
    }
  }
  if(!name) usage();
  text = cbdatumopen("", 0);
  for(i = 0; i < wnum; i++){
    if(i > 0) cbdatumcat(text, " ", 1);
    cbdatumcat(text, srchwords[i], -1);
  }
  rv = dosearch(name, cbdatumptr(text), max, or, idf, tb, hb, nb);
  cbdatumclose(text);
  return rv;
}


/* parse arguments of list command */
int runlist(int argc, char **argv){
  char *name;
  int i, tb, hb, rv;
  name = NULL;
  tb = FALSE;
  hb = FALSE;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-t")){
        tb = TRUE;
      } else if(!strcmp(argv[i], "-h")){
        hb = TRUE;
      } else {
        usage();
      }
    } else if(!name){
      name = argv[i];
    } else {
      usage();
    }
  }
  if(!name) usage();
  rv = dolist(name, tb, hb);
  return rv;
}


/* parse arguments of optimize command */
int runoptimize(int argc, char **argv){
  char *name;
  int i, rv;
  name = NULL;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      usage();
    } else if(!name){
      name = argv[i];
    } else {
      usage();
    }
  }
  if(!name) usage();
  rv = dooptimize(name);
  return rv;
}


/* parse arguments of inform command */
int runinform(int argc, char **argv){
  char *name;
  int i, rv;
  name = NULL;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      usage();
    } else if(!name){
      name = argv[i];
    } else {
      usage();
    }
  }
  if(!name) usage();
  rv = doinform(name);
  return rv;
}


/* parse arguments of remove command */
int runremove(int argc, char **argv){
  char *name;
  int i, rv;
  name = NULL;
  for(i = 2; i < argc; i++){
    if(!name && argv[i][0] == '-'){
      usage();
    } else if(!name){
      name = argv[i];
    } else {
      usage();
    }
  }
  if(!name) usage();
  rv = doremove(name);
  return rv;
}


/* parse arguments of break command */
int runbreak(int argc, char **argv){
  char *file, *text;
  int i, hb, kb, sb, size, rv;
  file = NULL;
  hb = FALSE;
  kb = FALSE;
  sb = FALSE;
  for(i = 2; i < argc; i++){
    if(!file && argv[i][0] == '-'){
      if(!strcmp(argv[i], "-h")){
        hb = TRUE;
      } else if(!strcmp(argv[i], "-k")){
        kb = TRUE;
      } else if(!strcmp(argv[i], "-s")){
        sb = TRUE;
      } else {
        usage();
      }
    } else if(!file){
      file = argv[i];
    } else {
      usage();
    }
  }
  if(file){
    if(!(text = cbreadfile(file, &size))){
      fprintf(stderr, "%s: %s: cannot open\n", progname, file);
      return 1;
    }
  } else {
    text = readstdin(&size);
  }
  rv = dobreak(text, hb, kb, sb);
  free(text);
  return rv;
}


/* print an error message */
void pdperror(const char *name){
  fprintf(stderr, "%s: %s: %s\n", progname, name, dperrmsg(dpecode));
}


/* print the contents of a document */
void printdoc(const ODDOC *doc, int tb, int hb, int score, ODEUM *odeum, const CBLIST *skeys){
  const CBLIST *words;
  CBMAP *scores;
  CBLIST *kwords;
  const char *title, *author, *word, *date;
  char *summary;
  int i, wsiz;
  title = oddocgetattr(doc, "title");
  author = oddocgetattr(doc, "author");
  date = oddocgetattr(doc, "date");
  if(hb){
    printf("ID: %d\n", oddocid(doc));
    printf("URI: %s\n", oddocuri(doc));
    if(title) printf("TITLE: %s\n", title);
    if(author) printf("AUTHOR: %s\n", author);
    if(date) printf("DATE: %s\n", date);