config.c

网络爬虫程序

      case PARAM_TRANSPARENT:
        if(mopt.args[0][0])
        {
          http_proxy *pr = http_proxy_parse(mopt.args[0]);

          if(!pr)
            usage_short();
          else
            *((http_proxy **) cpar->val_adr) = pr;
        }
        else
        {
          cfg_set_to_default(cpar);
        }
        break;
      case PARAM_PROXY:
        if(mopt.args[0][0])
        {
          http_proxy *pr = http_proxy_parse(mopt.args[0]);

          if(!pr)
            usage_short();
          else
            *((dllist **) cpar->val_adr) =
              dllist_append(*((dllist **) cpar->val_adr), (dllist_t) pr);
        }
        else
        {
          cfg_set_to_default(cpar);
        }
        break;
      case PARAM_FUNC:
        {
          int (*_cfg_func) (char *, char *, char *, char *);

          _cfg_func = cpar->val_adr;

          if(_cfg_func)
          {
            if(_cfg_func(mopt.args[0], mopt.args[1], mopt.args[2],
                mopt.args[3]))
            {
              usage_short();
            }
          }
        }
        break;
      case PARAM_JSTRANS:
#ifdef HAVE_REGEX
        {
          js_transform_t *jt;

          jt = js_transform_new(mopt.args[0], mopt.args[1], mopt.args[2],
          mopt.args[3], (long) cpar->mdefault_val);

          if(!jt)
          {
            xprintf(0, gettext("Invalid parameters for \"-%s\" option\n"),
              cfg_get_option_string(cpar, mopt.option_type));
            usage_short();
          }
          else
          {
            cfg.js_transform = dllist_append(cfg.js_transform, (dllist_t) jt);
          }
        }
#endif
        break;
      case PARAM_NUMLIST:
        {
          dllist *ptr = tl_numlist_split(mopt.args[0], ",");

          if(!ptr && mopt.args[0][0])
          {
            xprintf(0,
              gettext("Invalid number list \"%s\" for option \"-%s\"\n"),
              mopt.args[0], cfg_get_option_string(cpar, mopt.option_type));
            usage_short();
          }
          cfg_set_to_default(cpar);
          if(cpar->mval_adr)
            *((bool_t *) cpar->mval_adr) = (bool_t) (long) cpar->mdefault_val;
          *((dllist **) cpar->val_adr) = ptr;
        }
        break;
      case PARAM_FTPHS:
        {
          ftp_handshake_info *fhi;
          fhi = ftp_handshake_info_parse(mopt.args[0], mopt.args[1]);

          if(!fhi)
          {
            xprintf(0,
              gettext
              ("Invalid FTP login handshake string \"%s\" for option \"-%s\"\n"),
              mopt.args[1], cfg_get_option_string(cpar, mopt.option_type));
            usage_short();
          }
          *((dllist **) cpar->val_adr) =
            dllist_append(*((dllist **) cpar->val_adr), (dllist_t) fhi);
        }
        break;
      case PARAM_TAGPAT:
        {
          tag_pattern_t *tp;
          tp = tag_pattern_new((long) cpar->mdefault_val,
            mopt.args[0], mopt.args[1], mopt.args[2]);
          if(!tp)
            usage_short();
          *((dllist **) cpar->val_adr) =
            dllist_append(*((dllist **) cpar->val_adr), (dllist_t) tp);
        }
        break;
      }
    }

    if(moptrv == MOPT_PARAM)
    {
      url_info *ui;
      ui = url_info_new(mopt.args[0]);
      cfg.request = dllist_append(cfg.request, (dllist_t) ui);
      cfg.total_cnt++;
    }
  }
  mopt_destroy(&mopt);
}

static int cfg_load_fd(bufio * fd)
{
  int i;
  bool_t found;
  int nr;
  double dnr;
  char *p;
  char lnbuf[4096];
  char *lns;
  pavuk_mode temp_mode;
  int rv = 0;

  while(bufio_readln(fd, lnbuf, sizeof(lnbuf)) > 0)
  {
    strip_nl(lnbuf);
    for(lns = lnbuf; *lns && tl_ascii_isspace(*lns); lns++);
    if(*lns == '#' || !*lns)
      continue;

    found = FALSE;

    for(i = 0; i < NUM_ELEM(params); i++)
    {
      if(!params[i].par_entry)
        continue;
      if(!strncasecmp(lns, params[i].par_entry, strlen(params[i].par_entry)))
      {
        if(params[i].type & PARAM_UNSUPPORTED)
        {
          xprintf(0,
            gettext
            ("WARNING: option \"-%s\" not supported in current configuration!\n"),
            params[i].par_entry);
          continue;
        }

        if(params[i].type & PARAM_FOREIGN)
          continue;

        lns += strlen(params[i].par_entry);
        for(; *lns && tl_ascii_isspace(*lns); lns++);
        for(p = lns + strlen(lns) - 1; p >= lns && tl_ascii_isspace(*p); p--)
          *p = '\0';

        if(!*lns)
        {
          cfg_set_to_default(&(params[i]));
          continue;
        }

        found = TRUE;
        switch (params[i].type)
        {
        case PARAM_NUM:
          nr = _atoi(lns);
          if(errno == ERANGE)
          {
            xprintf(0, gettext("Please specify number \"%s\"\n"),
              params[i].par_entry);
            rv = 1;
          }
          else
            *((int *) params[i].val_adr) = nr;
          break;
        case PARAM_PBOOL:
        case PARAM_NBOOL:
          if(!strcasecmp(lns, "false"))
          {
            *((bool_t *) params[i].val_adr) = FALSE;
          }
          else if(!strcasecmp(lns, "true"))
          {
            *((bool_t *) params[i].val_adr) = TRUE;
          }
          else
          {
            xprintf(0,
              gettext("Only \"true\" & \"false\" is allowed : \"%s\"\n"),
              lns);
            rv = 1;
          }
          break;
        case PARAM_PORT_RANGE:
          if(sscanf(lns, "%ld:%ld",
              (long *) params[i].val_adr,
              (long *) params[i].mval_adr) != 2
            || *((long *) params[i].val_adr) <= 1023 ||
            *((long *) params[i].mval_adr) > 65535 ||
            *((long *) params[i].val_adr) >= *((long *) params[i].mval_adr))
          {
            xprintf(0, gettext("Invalid port range \"%s\"\n"), lns);
            rv = 1;
          }
          break;
        case PARAM_STR:
        case PARAM_PASS:
          cfg_set_to_default(&params[i]);
          *((char **) params[i].val_adr) = *lns ? tl_strdup(lns) : NULL;
          break;
        case PARAM_PATH:
          cfg_set_to_default(&params[i]);
#ifdef __CYGWIN__
          *((char **) params[i].val_adr) =
            *lns ? cvt_win32_to_unix_path(lns) : NULL;
#else
          *((char **) params[i].val_adr) =
            *lns ? get_abs_file_path_oss(lns) : NULL;
#endif
          break;
        case PARAM_STRLIST:
          cfg_set_to_default(&params[i]);
          if(params[i].mval_adr)
            *((bool_t *) params[i].mval_adr) =
              (bool_t) (long) params[i].mdefault_val;
          if(*lns)
            *((char ***) params[i].val_adr) = tl_str_split(lns, ",");
          break;
        case PARAM_CONN:
          cfg_set_to_default(&params[i]);
          p = strchr(lns, ':');
          if(p)
          {
            nr = _atoi(p + 1);
            if(errno == ERANGE)
            {
              struct servent *se;

              if((se = getservbyname(p + 1, "tcp")))
              {
                nr = ntohs(se->s_port);
              }
              else
              {
                xprintf(0, gettext("Unknown port \"%s\"\n"), p + 1);
                rv = 1;
              }
            }
            if(params[i].mval_adr)
              *((int *) params[i].mval_adr) = (int) nr;
          }
          *((char **) params[i].val_adr) =
            p ? tl_strndup(lns, p - lns) : tl_strdup(lns);
          break;
        case PARAM_AUTHSCH:
          nr = authinfo_get_type(lns);
          if(nr == HTTP_AUTH_NONE)
          {
            xprintf(0, gettext("Bad auth scheme \"%s\"\n"), lns);
            rv = 1;
          }
          else
            *((int *) params[i].val_adr) = nr;
          break;
        case PARAM_MODE:
          temp_mode = mode_get_by_str(lns);
          if(temp_mode == MODE_UNKNOWN)
          {
            xprintf(0, gettext("Unknow operation mode \"%s\"\n"), lns);
            rv = 1;
          }
          else
            *((pavuk_mode *) params[i].val_adr) = temp_mode;
          break;
        case PARAM_TIME:
          {
            time_t ttm = time_scn_cmd(lns);
            if(!params[i].val_adr)
            {
              xprintf(0, gettext("Bad time parameter \"%s\"\n"), lns);
              rv = 1;
            }
            else
              *(time_t *) params[i].val_adr = ttm;
          }
          break;
        case PARAM_HTMLTAG:
          htmltag_set_disabled(lns, (long) params[i].default_val);
          break;
        case PARAM_TWO_QSTR:
          cfg_set_to_default(&params[i]);
          if(lns && *lns)
          {
            char *xp = tl_strdup(lns);

            *(char **) params[i].val_adr = tl_strdup(get_1qstr(xp));
            *(char **) params[i].mval_adr = tl_strdup(get_1qstr(NULL));

            _free(xp);
          }
          else
          {
            *(char **) params[i].val_adr = NULL;
            *(char **) params[i].mval_adr = NULL;
          }
          break;
        case PARAM_DOUBLE:
          dnr = _atof(lns);
          if(errno == ERANGE)
          {
            xprintf(0, gettext("Please specify floating number \"%s\"\n"),
              params[i].par_entry);
            rv = 1;
          }
          else
            *(double *) params[i].val_adr = dnr;
          break;
        case PARAM_LFNAME:
          {
            char *ps1 = tl_strdup(get_1qstr(lns));
            char *ps2 = tl_strdup(get_1qstr(NULL));
            char *ps3 = tl_strdup(get_1qstr(NULL));
            lfname_type t;
            lfname *lfnm;

            if(!ps1 || !ps2 || !ps3)
            {
              t = LFNAME_UNKNOWN;
              xprintf(0, gettext("Please specify proper arguments for %s\n"),
                params[i].par_entry);
              rv = 1;
            }
            else if(!strcasecmp(ps1, "F"))
              t = LFNAME_FNMATCH;
#ifdef HAVE_REGEX
            else if(!strcasecmp(ps1, "R"))
              t = LFNAME_REGEX;
#endif
            else
            {
              t = LFNAME_UNKNOWN;
              xprintf(0,
                gettext("Please specify proper condition type for %s (%s)\n"),
                params[i].par_entry,
#ifdef HAVE_REGEX
                "F or R"
#else
                "F"
#endif
                );
              rv = 1;
            }
            if(t != LFNAME_UNKNOWN)
            {
              lfnm = lfname_new(t, ps2, ps3);
              if(lfnm)
                cfg.lfnames = dllist_append(cfg.lfnames, (dllist_t) lfnm);
              else
                rv = 1;
            }
            _free(ps1);
            _free(ps2);
            _free(ps3);
          }
          break;
        case PARAM_RE:
#ifdef HAVE_REGEX
          {
            re_entry *ree;
            if(!(ree = re_make(lns)))
            {
              xprintf(0, gettext("Please specify valid RE \"%s\"\n"),
                params[i].par_entry);
              rv = 1;
            }
            else
              *(dllist **) params[i].val_adr =
                dllist_append(*((dllist **) params[i].val_adr), (dllist_t) ree);
          }
#endif
          break;
        case PARAM_USTRAT:
          {
            strategie strtg = get_strategie_by_str(lns);
            if(strtg == SSTRAT_LAST)
            {
              xprintf(0,
                gettext("Unknown URL scheduling strategy - \"%s\"\n"), lns);
              rv = 1;
            }
            else
              *(strategie *) params[i].val_adr = strtg;
          }
          break;
        case PARAM_SSLVER:
          {
            int sslv = get_ssl_version_by_str(lns);
            if(sslv == -1)
            {
              xprintf(0, gettext("Unknown SSL version - \"%s\"\n"), lns);
              rv = 1;
            }
            else
              *(int *) params[i].val_adr = sslv;
          }
          break;
        case PARAM_HTTPHDR:
          {
            httphdr *hr = httphdr_parse(lns);
            if(!hr)
            {
              xprintf(0, gettext("Invalid additional HTTP header - \"%s\"\n"),
                lns);
              rv = 1;
            }
            else
              *((dllist **) params[i].val_adr) =
                dllist_append(*((dllist **) params[i].val_adr), (dllist_t) hr);
          }
          break;
        case PARAM_DEBUGL:
#ifdef DEBUG
          {
            int dl = debug_level_parse(lns);
            if(dl != -1)
              *((int *) params[i].val_adr) = dl;
            else
              rv = 1;
          }
#endif
          break;
        case PARAM_REQUEST:
          {
            url_info *ui = url_info_parse(lns);
            if(!ui)
            {
              xprintf(0, gettext("Invalid request specification - \"%s\"\n"),
                lns);
              rv = 1;
            }
            else
              *((dllist **) params[i].val_adr) =
                dllist_append(*((dllist **) params[i].val_adr), (dllist_t) ui);
          }
          break;
        case PARAM_TRANSPARENT:
          if(lns)
          {
            http_proxy *pr = http_proxy_parse(lns);
            if(pr)
              *((http_proxy **) params[i].val_adr) = pr;
            else
              rv = 1;
          }
          break;
        case PARAM_PROXY:
          if(lns)
          {
            http_proxy *pr = http_proxy_parse(lns);

            if(pr)
              *((dllist **) params[i].val_adr) =
                dllist_append(*((dllist **) params[i].val_adr), (dllist_t) pr);
            else
              rv = 1;
          }
          break;
        case PARAM_FUNC:
          break;
        case PARAM_JSTRANS:
#ifdef HAVE_REGEX
          {
            char *ps1 = tl_strdup(get_1qstr(lns));
            char *ps2 = tl_strdup(get_1qstr(NULL));
            char *ps3 = tl_strdup(get_1qstr(NULL));
            char *ps4 = tl_strdup(get_1qstr(NULL));

            js_transform_t *jt;

            jt = js_transform_new(ps1, ps2, ps3, ps4,
            (long) params[i].mdefault_val);

            if(!jt)
            {
              xprintf(0,