config.c

网络爬虫程序

  }
}

void cfg_set_all_to_default(void)
{
  int i;

  for(i = 0; i < NUM_ELEM(params); i++)
    cfg_set_to_default(&(params[i]));
}

void cfg_setup_default(void)
{
  int i, x, j;

  for(i = 0; i < NUM_ELEM(params); i++)
  {
    if(params[i].type & PARAM_UNSUPPORTED)
      continue;

    if(params[i].type & PARAM_FOREIGN)
      continue;

    switch (params[i].type)
    {
    case PARAM_NUM:
      *((long *) params[i].val_adr) = (long) params[i].default_val;
      break;
    case PARAM_PBOOL:
      *((bool_t *) params[i].val_adr) = (bool_t) (long) params[i].default_val;
      break;
    case PARAM_NBOOL:
      *((bool_t *) params[i].val_adr) = (bool_t) (long) params[i].default_val;
      break;
    case PARAM_PORT_RANGE:
      *((long *) params[i].val_adr) = (long) params[i].default_val;
      *((long *) params[i].mval_adr) = (long) params[i].mdefault_val;
      break;
    case PARAM_PATH:
    case PARAM_STR:
    case PARAM_PASS:
      *((char **) params[i].val_adr) = (char *) params[i].default_val;
      break;
    case PARAM_STRLIST:
      *((char ***) params[i].val_adr) = (char **) params[i].default_val;
      if(params[i].mval_adr)
        *((bool_t *) params[i].mval_adr) =
          (bool_t) (long) params[i].mdefault_val;
      break;
    case PARAM_CONN:
      *((char **) params[i].val_adr) = (char *) params[i].default_val;
      if(params[i].mval_adr)
        *((long *) params[i].mval_adr) = (long) params[i].mdefault_val;
      break;
    case PARAM_AUTHSCH:
      *((long *) params[i].val_adr) = (long) params[i].default_val;
      break;
    case PARAM_MODE:
      *((long *) params[i].val_adr) = (long) params[i].default_val;
      break;
    case PARAM_TIME:
      *((time_t *) params[i].val_adr) = (time_t) 0;
      break;
    case PARAM_HTMLTAG:
      for(x = 0; x < html_link_tags_num(); x++)
        for(j = 0; html_link_tags[x].attribs[j].attrib; j++)
          html_link_tags[x].attribs[j].stat &= ~LINK_DISABLED;
      break;
    case PARAM_TWO_QSTR:
      *((char **) params[i].val_adr) = (char *) params[i].default_val;
      *((char **) params[i].mval_adr) = (char *) params[i].mdefault_val;
      break;
    case PARAM_DOUBLE:
      *((double *) params[i].val_adr) = *(double *) params[i].default_val;
      break;
    case PARAM_LFNAME:
      cfg.lfnames = NULL;
      break;
    case PARAM_RE:
#ifdef HAVE_REGEX
      *((dllist **) params[i].val_adr) = NULL;
#endif
      break;
    case PARAM_USTRAT:
      *((strategie *) params[i].val_adr) = (strategie) params[i].default_val;
      break;
    case PARAM_SSLVER:
      *((long *) params[i].val_adr) = (long) params[i].default_val;
      break;
    case PARAM_HTTPHDR:
      *((dllist **) params[i].val_adr) = NULL;
      break;
    case PARAM_DEBUGL:
      *((long *) params[i].val_adr) = (long) params[i].default_val;
      break;
    case PARAM_REQUEST:
      *((dllist **) params[i].val_adr) = NULL;
      break;
    case PARAM_TRANSPARENT:
      *((http_proxy **) params[i].val_adr) = NULL;
      break;
    case PARAM_PROXY:
      *((dllist **) params[i].val_adr) = NULL;
      break;
    case PARAM_FUNC:
      break;
    case PARAM_JSTRANS:
#ifdef HAVE_REGEX
      cfg.js_transform = NULL;
#endif
      break;
    case PARAM_NUMLIST:
      *((dllist **) params[i].val_adr) = (dllist *) params[i].default_val;
      if(params[i].mval_adr)
        *((bool_t *) params[i].mval_adr) =
          (bool_t) (long) params[i].mdefault_val;
      break;
    case PARAM_FTPHS:
      *((dllist **) params[i].val_adr) = (dllist *) params[i].default_val;
      break;
    case PARAM_TAGPAT:
      *((dllist **) params[i].val_adr) = (dllist *) params[i].default_val;
      break;
    }
  }
}

int cfg_get_num_params(cfg_param_t * cpar)
{
  long num;

  static const struct
  {
    par_type_t type;
    int num_params;
  } tab[] =
  {
    {PARAM_NUM, 1},
    {PARAM_PBOOL, 0},
    {PARAM_NBOOL, 0},
    {PARAM_STR, 1},
    {PARAM_PASS, 1},
    {PARAM_STRLIST, 1},
    {PARAM_CONN, 1},
    {PARAM_AUTHSCH, 1},
    {PARAM_MODE, 1},
    {PARAM_PATH, 1},
    {PARAM_TIME, 1},
    {PARAM_HTMLTAG, 1},
    {PARAM_TWO_QSTR, 2},
    {PARAM_DOUBLE, 1},
    {PARAM_LFNAME, 3},
    {PARAM_RE, 1},
    {PARAM_USTRAT, 1},
    {PARAM_SSLVER, 1},
    {PARAM_HTTPHDR, 1},
    {PARAM_DEBUGL, 1},
    {PARAM_REQUEST, 1},
    {PARAM_PROXY, 1},
    {PARAM_TRANSPARENT, 1},
    {PARAM_FUNC, 0},
    {PARAM_JSTRANS, 4},
    {PARAM_NUMLIST, 1},
    {PARAM_FTPHS, 2},
    {PARAM_TAGPAT, 3},
    {PARAM_PORT_RANGE, 1}
  };

  num = tab[cpar->type].num_params;

  if(cpar->type == PARAM_FUNC)
    num = (long) cpar->default_val;

  return num;
}

static char *cfg_get_option_string(cfg_param_t * param, int type)
{
  switch (type)
  {
  case MOPT_OPT_SHORT:
    return param->short_cmd;
  case MOPT_OPT_LONG:
    return param->long_cmd;
  case MOPT_OPT_COMPAT:
    return param->long_cmd;
  default:
    return "";
  }
}

void cfg_setup_cmdln(int argc, char **argv)
{
  int nr = 0;
  double dnr = 0.0;
  char *p = NULL;
  char **pl = NULL;
  mopt_t mopt;
  cfg_param_t *cpar;
  int moptrv;

  mopt_init(&mopt, NUM_ELEM(params), params, argc, argv);

  for(;;)
  {
    moptrv = mopt_get_next_param(&mopt, &cpar);

    if(moptrv == MOPT_END)
      break;

    if(moptrv == MOPT_ERR)
    {
      xprintf(0, gettext("Error parsing commandline\n"));
      usage_short();
      break;
    }

    if(moptrv == MOPT_MISSINGP)
    {
      xprintf(0,
        gettext("Not enough number of parameters for \"-%s\" option\n"),
        cfg_get_option_string(cpar, mopt.option_type));
      usage_short();
      break;
    }

    if(moptrv == MOPT_UNKNOWN)
    {
      xprintf(0, gettext("Unknown option %s\n"), argv[mopt.current]);
      usage_short();
      break;
    }

    if(moptrv == MOPT_BAD)
    {
      xprintf(0, gettext("Wrong format of option %s\n"), argv[mopt.current]);
      usage_short();
      break;
    }

    if(moptrv == MOPT_OK)
    {
      if(cpar->type & PARAM_UNSUPPORTED)
      {
        xprintf(0,
          gettext
          ("WARNING: option \"-%s\" not supported in current configuration!\n"),
          cfg_get_option_string(cpar, mopt.option_type));
        continue;
      }

      if(cpar->type & PARAM_FOREIGN)
        continue;

      switch (cpar->type)
      {
      case PARAM_NUM:
        nr = _atoi(mopt.args[0]);
        if(errno == ERANGE)
        {
          xprintf(0,
            gettext("Please specify number with parameter \"-%s\"\n"),
            cfg_get_option_string(cpar, mopt.option_type));
          usage_short();
        }
        *((int *) cpar->val_adr) = nr;
        break;
      case PARAM_PBOOL:
        *((bool_t *) cpar->val_adr) = TRUE;
        break;
      case PARAM_NBOOL:
        *((bool_t *) cpar->val_adr) = FALSE;
        break;
      case PARAM_PORT_RANGE:
        if(sscanf(mopt.args[0], "%ld:%ld",
            (long *) cpar->val_adr,
            (long *) cpar->mval_adr) != 2
          || *((long *) cpar->val_adr) <= 1023 ||
          *((long *) cpar->mval_adr) > 65535 ||
          *((long *) cpar->val_adr) >= *((long *) cpar->mval_adr))
        {
          xprintf(0, gettext("Invalid port range \"%s\"\n"), mopt.args[0]);
          usage_short();
        }
        break;
      case PARAM_PASS:
      case PARAM_STR:
        cfg_set_to_default(cpar);
        if(mopt.args[0][0])
          p = tl_strdup(mopt.args[0]);
        else
          p = NULL;
        *((char **) cpar->val_adr) = p;
        if(cpar->type == PARAM_PASS)
        {
          if(mopt.args[0][0])
          {
            memset(mopt.args[0], ' ', strlen(mopt.args[0]));
            strcpy(mopt.args[0], "*");
          }
        }
        break;
      case PARAM_PATH:
        cfg_set_to_default(cpar);
        if(mopt.args[0][0])
        {
#ifdef __CYGWIN__
          p = cvt_win32_to_unix_path(mopt.args[0]);
#else
          p = get_abs_file_path_oss(mopt.args[0]);
#endif
        }
        *((char **) cpar->val_adr) = p;
        break;
      case PARAM_STRLIST:
        cfg_set_to_default(cpar);
        pl = tl_str_split(mopt.args[0], ",");
        if(cpar->mval_adr)
          *((bool_t *) cpar->mval_adr) = (bool_t) (long) cpar->mdefault_val;
        *((char ***) cpar->val_adr) = pl;
        break;
      case PARAM_CONN:
        cfg_set_to_default(cpar);
        if(mopt.args[0][0])
        {
          p = strchr(mopt.args[0], ':');
          if(p)
          {
            nr = _atoi(p + 1);
            if(errno == ERANGE)
            {
              struct servent *se;

              if((se = getservbyname(p + 1, "tcp")))
              {
                nr = ntohs(se->s_port);
              }
              else
              {
                xprintf(0, gettext("Unknown port \"%s\"\n"), p + 1);
              }
            }
            if(cpar->mval_adr)
              *((int *) cpar->mval_adr) = (int) nr;
          }
        }
        else
          p = NULL;

        *((char **) cpar->val_adr) =
          p ? tl_strndup(mopt.args[0],
          p - mopt.args[0]) : tl_strdup(mopt.args[0]);
        break;
      case PARAM_AUTHSCH:
        nr = authinfo_get_type(mopt.args[0]);
        if(nr == HTTP_AUTH_NONE)
        {
          xprintf(0, gettext("Bad auth scheme \"%s\"\n"), mopt.args[0]);
          usage_short();
        }
        else
          *((int *) cpar->val_adr) = nr;
        break;
      case PARAM_MODE:
        cfg.mode = mode_get_by_str(mopt.args[0]);
        if(cfg.mode == MODE_UNKNOWN)
        {
          xprintf(0, gettext("Unknow operation mode \"%s\"\n"), mopt.args[0]);
          usage_short();
        }
        break;
      case PARAM_TIME:
        *((time_t *) cpar->val_adr) = time_scn_cmd(mopt.args[0]);
        break;
      case PARAM_HTMLTAG:
        htmltag_set_disabled(mopt.args[0], (long) cpar->default_val);
        break;
      case PARAM_TWO_QSTR:
        cfg_set_to_default(cpar);
        p = tl_strdup(mopt.args[0]);
        *((char **) cpar->val_adr) = p;

        p = tl_strdup(mopt.args[1]);
        *((char **) cpar->mval_adr) = p;
        break;
      case PARAM_DOUBLE:
        dnr = _atof(mopt.args[0]);
        if(errno == ERANGE)
        {
          xprintf(0,
            gettext
            ("Please specify floating number with parameter \"-%s\"\n"),
            cfg_get_option_string(cpar, mopt.option_type));
          usage_short();
        }
        *((double *) cpar->val_adr) = dnr;
        break;
      case PARAM_LFNAME:
        {
          lfname_type t;
          lfname *lfnm;

          if(!strcasecmp(mopt.args[0], "F"))
            t = LFNAME_FNMATCH;
#ifdef HAVE_REGEX
          else if(!strcasecmp(mopt.args[0], "R"))
            t = LFNAME_REGEX;
#endif
          else
          {
            t = LFNAME_UNKNOWN;
#ifdef HAVE_REGEX
#define __CONDITIONS "F or R"
#else
#define __CONDITIONS "F"
#endif
            xprintf(0,
              gettext("Please specify proper condition type for -%s (%s)\n"),
              cfg_get_option_string(cpar, mopt.option_type), __CONDITIONS);
#undef __CONDITIONS
            usage_short();
          }
          lfnm = lfname_new(t, mopt.args[1], mopt.args[2]);
          if(!lfnm)
            usage_short();

          cfg.lfnames = dllist_append(cfg.lfnames, (dllist_t) lfnm);
        }
        break;
      case PARAM_RE:
#ifdef HAVE_REGEX
        {
          re_entry *ree = NULL;

          if(!(ree = re_make(mopt.args[0])))
          {
            xprintf(0,
              gettext("Please specify valid RE with parameter \"-%s\"\n"),
              cfg_get_option_string(cpar, mopt.option_type));
            usage_short();
          }
          *((dllist **) cpar->val_adr) =
            dllist_append(*((dllist **) cpar->val_adr), (dllist_t) ree);
        }
#endif
        break;
      case PARAM_USTRAT:
        *(strategie *) cpar->val_adr = get_strategie_by_str(mopt.args[0]);
        if(*(strategie *) cpar->val_adr == SSTRAT_LAST)
        {
          xprintf(0, gettext("Unknown URL scheduling strategy - \"%s\"\n"),
            mopt.args[0]);
          usage_short();
        }
        break;
      case PARAM_SSLVER:
        *(int *) cpar->val_adr = get_ssl_version_by_str(mopt.args[0]);
        if(*(int *) cpar->val_adr == -1)
        {
          xprintf(0, gettext("Unknown SSL version - \"%s\"\n"), mopt.args[0]);
          usage_short();
        }
        break;
      case PARAM_HTTPHDR:
        {
          httphdr *hr = httphdr_parse(mopt.args[0]);
          if(!hr)
          {
            xprintf(0, gettext("Invalid additional HTTP header - \"%s\"\n"),
              mopt.args[0]);
            usage_short();
          }
          *((dllist **) cpar->val_adr) =
            dllist_append(*((dllist **) cpar->val_adr), (dllist_t) hr);
        }
        break;
      case PARAM_DEBUGL:
#ifdef DEBUG
        {
          int dl = debug_level_parse(mopt.args[0]);
          if(dl == -1)
          {
            usage_short();
          }
          *((int *) cpar->val_adr) = dl;
        }
#endif
        break;
      case PARAM_REQUEST:
        {
          url_info *ui = url_info_parse(mopt.args[0]);
          if(!ui)
          {
            xprintf(0, gettext("Invalid request specification - \"%s\"\n"),
              mopt.args[0]);
            usage_short();
          }
          *((dllist **) cpar->val_adr) =
            dllist_append(*((dllist **) cpar->val_adr), (dllist_t) ui);
        }
        break;