config.c

网络爬虫程序

                gettext("Invalid js_transform specification - \"%s\"\n"),
                lns);
              rv = 1;
            }
            else
              cfg.js_transform = dllist_append(cfg.js_transform, (dllist_t) jt);
            _free(ps1);
            _free(ps2);
            _free(ps3);
            _free(ps4);
          }
#endif
          break;
        case PARAM_NUMLIST:
          cfg_set_to_default(&params[i]);
          if(params[i].mval_adr)
            *((bool_t *) params[i].mval_adr) =
              (bool_t) (long) params[i].mdefault_val;
          if(*lns)
            *((dllist **) params[i].val_adr) = tl_numlist_split(lns, ",");
          if(*lns && !*((dllist **) params[i].val_adr))
          {
            xprintf(0,
              gettext("Invalid number list specification - \"%s\"\n"), lns);
            rv = 1;
          }
          break;
        case PARAM_FTPHS:
          {
            char *ps1 = tl_strdup(get_1qstr(lns));
            char *ps2 = tl_strdup(get_1qstr(NULL));
            ftp_handshake_info *fhi = NULL;

            if(ps1 && ps2)
              fhi = ftp_handshake_info_parse(ps1, ps2);

            if(!fhi)
            {
              xprintf(0,
                gettext("Invalid FTP login handshake string \"%s\".\n"), lns);
              rv = 1;
            }
            *((dllist **) params[i].val_adr) =
              dllist_append(*((dllist **) params[i].val_adr), (dllist_t) fhi);
            _free(ps1);
            _free(ps2);
          }
          break;
        case PARAM_TAGPAT:
          {
            char *ps1 = tl_strdup(get_1qstr(lns));
            char *ps2 = tl_strdup(get_1qstr(NULL));
            char *ps3 = tl_strdup(get_1qstr(NULL));
            tag_pattern_t *tp = NULL;

            if(ps1 && ps2 && ps3)
	    {
              tp = tag_pattern_new((long) params[i].mdefault_val,
	      ps1, ps2, ps3);
            }
            if(!tp)
              rv = 1;
            *((dllist **) params[i].val_adr) =
              dllist_append(*((dllist **) params[i].val_adr), (dllist_t) tp);
            _free(ps1);
            _free(ps2);
            _free(ps3);
          }
          break;
        }
      }
    }
    if(!found && !strncasecmp(lns, "URL:", 4))
    {
      url_info *ui;

      lns += 4;
      for(; *lns && tl_ascii_isspace(*lns); lns++);
      for(p = lns + strlen(lns); *p && tl_ascii_isspace(*p); p--)
        *p = '\0';
      if(!*lns)
        continue;

      ui = url_info_new(lns);
      cfg.request = dllist_append(cfg.request, (dllist_t) ui);
      cfg.total_cnt++;
    }
    else if(!found)
    {
      xprintf(0, gettext("Unable to parse \"%s\"\n"), lns);
      rv = 1;
    }
  }

  return rv;
}

int cfg_load(const char *filename)
{
  int rv;
  bufio *fd;

  if(!(fd = bufio_open(filename, O_BINARY | O_RDONLY)))
  {
    xperror(filename);
    return -1;
  }

  rv = cfg_load_fd(fd);

  bufio_close(fd);

  return rv;
}

static int cfg_load_scenario(const char *filename)
{
  char *fn;
  int rv;

  _free(cfg.scenario);

  if(strchr(filename, '/') || !cfg.scndir)
    fn = tl_strdup(filename);
  else
    fn = tl_str_concat(tl_strdup(cfg.scndir), "/", filename, NULL);

  if((rv = cfg_load(fn)))
  {
    xprintf(0, gettext("ERROR: Scenario loading failed (%s)\n"), fn);
    exit(PAVUK_EXIT_CFG_ERR);
  }

  cfg.scenario = fn;

  return rv;
}

void cfg_load_setup(void)
{
  char pom[PATH_MAX];
  char *p;

#ifdef DEFAULTRC
  if(!access(DEFAULTRC, R_OK))
    cfg_load(DEFAULTRC);
#endif
  p = getenv("PAVUKRC_FILE");
  if(!p)
  {
    snprintf(pom, sizeof(pom), "%s/%s", cfg.path_to_home, ".pavukrc");
    p = pom;
  }
  if(!access(p, R_OK))
    cfg_load(p);
}

static int cfg_dump_fd(int fd)
{
  int i, j;
  char pom[8192];
  char pom2[20];
  char **pl;

  if(cfg.request)
  {
    dllist *dptr;

    for(dptr = cfg.request; dptr; dptr = dptr->next)
    {
      url_info *ui = (url_info *) dptr->data;

      if(ui->type == URLI_NORMAL && !ui->localname)
      {
        write(fd, "URL: ", 5);
        write(fd, ui->urlstr, strlen(ui->urlstr));
        write(fd, "\n", 1);
      }
    }
  }

  for(i = 0; i < NUM_ELEM(params); i++)
  {
    if(params[i].type & PARAM_UNSUPPORTED)
      continue;
    if(params[i].type & PARAM_FOREIGN)
      continue;
    if(!params[i].par_entry)
      continue;

    switch (params[i].type)
    {
    case PARAM_NUM:
      snprintf(pom, sizeof(pom), "%s %ld\n", params[i].par_entry,
        *((long *) params[i].val_adr));
      write(fd, pom, strlen(pom));
      break;
    case PARAM_NBOOL:
    case PARAM_PBOOL:
      if(*((bool_t *) params[i].val_adr))
        snprintf(pom, sizeof(pom), "%s true\n", params[i].par_entry);
      else
        snprintf(pom, sizeof(pom), "%s false\n", params[i].par_entry);
      write(fd, pom, strlen(pom));
      break;
    case PARAM_PORT_RANGE:
      if(*((long *) params[i].val_adr) >= 0)
      {
        snprintf(pom, sizeof(pom), "%s %ld:%ld\n", params[i].par_entry,
          *((long *) params[i].val_adr), *((long *) params[i].mval_adr));
        write(fd, pom, strlen(pom));
      }
      break;
    case PARAM_PATH:
    case PARAM_STR:
    case PARAM_PASS:
      if(*((char **) params[i].val_adr))
      {
        snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry,
          *((char **) params[i].val_adr));
        write(fd, pom, strlen(pom));
      }
      break;
    case PARAM_STRLIST:
      if(!params[i].mval_adr || (params[i].mval_adr &&
          (*((bool_t *) params[i].mval_adr) ==
            (bool_t) (long) params[i].mdefault_val)))
      {
        pl = *((char ***) params[i].val_adr);
        if(pl && pl[0])
        {
          snprintf(pom, sizeof(pom), "%s %s", params[i].par_entry, pl[0]);
          write(fd, pom, strlen(pom));

          j = 1;
          while(pl[j])
          {
            write(fd, ",", 1);
            write(fd, pl[j], strlen(pl[j]));
            j++;
          }
          write(fd, "\n", 1);
        }
      }
      break;
    case PARAM_CONN:
      if(*((char **) params[i].val_adr))
      {
        snprintf(pom, sizeof(pom), "%s %s:%d\n", params[i].par_entry,
          *((char **) params[i].val_adr), *((int *) params[i].mval_adr));
        write(fd, pom, strlen(pom));
      }
      break;
    case PARAM_AUTHSCH:
      snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry,
        http_auths[*((long *) params[i].val_adr)].name);
      write(fd, pom, strlen(pom));
      break;
    case PARAM_MODE:
      snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry,
        mode_get_str(cfg.mode));
      write(fd, pom, strlen(pom));
      break;
    case PARAM_TIME:
      if(*((time_t *) params[i].val_adr))
      {
        LOCK_TIME;
        strftime(pom2, sizeof(pom2), "%Y.%m.%d.%H:%M",
          localtime((time_t *) params[i].val_adr));
        UNLOCK_TIME;
        snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry, pom2);
        write(fd, pom, strlen(pom));
      }
      break;
    case PARAM_HTMLTAG:
      {
        int x, y;
        bool_t first = TRUE;

        snprintf(pom, sizeof(pom), "%s ", params[i].par_entry);
        for(x = 0; x < html_link_tags_num(); x++)
        {
          for(y = 0; html_link_tags[x].attribs[y].attrib; y++)
          {
            if(!(html_link_tags[x].attribs[y].stat & LINK_DISABLED) ==
              !(params[i].default_val))
            {
              if(!first)
              {
                strncat(pom, ";", sizeof(pom) - strlen(pom));
                pom[sizeof(pom) - 1] = '\0';
              }
              strncat(pom, html_link_tags[x].tag, sizeof(pom) - strlen(pom));
              pom[sizeof(pom) - 1] = '\0';
              strncat(pom, ",", sizeof(pom) - strlen(pom));
              pom[sizeof(pom) - 1] = '\0';
              strncat(pom, html_link_tags[x].attribs[y].attrib,
                sizeof(pom) - strlen(pom));
              pom[sizeof(pom) - 1] = '\0';
              first = FALSE;
            }
          }
        }
        strncat(pom, "\n", sizeof(pom) - strlen(pom));
        pom[sizeof(pom) - 1] = '\0';
        if(!first)
          write(fd, pom, strlen(pom));
      }
      break;
    case PARAM_TWO_QSTR:
      if(*((char **) params[i].val_adr))
      {
        char *p1, *p2;
        p1 = escape_str(*((char **) params[i].val_adr), "\\\"");
        p2 = escape_str(*((char **) params[i].mval_adr), "\\\"");
        snprintf(pom, sizeof(pom), "%s \"%s\" \"%s\"\n", params[i].par_entry,
          p1, p2);
        _free(p1);
        _free(p2);
        write(fd, pom, strlen(pom));
      }
      break;
    case PARAM_DOUBLE:
      snprintf(pom, sizeof(pom), "%s %.3f\n", params[i].par_entry,
        *((double *) params[i].val_adr));
      write(fd, pom, strlen(pom));
      break;
    case PARAM_LFNAME:
      {
        dllist *pdl = cfg.lfnames;
        while(pdl)
        {
          lfname *lfnm = (lfname *) pdl->data;
          char *p1, *p2;

          p1 = escape_str(lfnm->matchstr, "\\\"");
          p2 = escape_str(lfnm->transstr, "\\\"");

          snprintf(pom, sizeof(pom), "%s \"%s\" \"%s\" \"%s\"\n",
            params[i].par_entry, (lfnm->type == LFNAME_FNMATCH) ? "F" : "R",
            p1, p2);
          _free(p1);
          _free(p2);
          write(fd, pom, strlen(pom));

          pdl = pdl->next;
        }
      }
      break;
    case PARAM_RE:
#ifdef HAVE_REGEX
      {
        dllist *ptr = *((dllist **) params[i].val_adr);
        while(ptr)
        {
          re_entry *ree = (re_entry *) ptr->data;

          snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry,
            ree->pattern);
          write(fd, pom, strlen(pom));

          ptr = ptr->next;
        }
      }
#endif
      break;
    case PARAM_USTRAT:
      snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry,
        get_strategie_str(*(strategie *) params[i].val_adr));
      write(fd, pom, strlen(pom));
      break;
    case PARAM_SSLVER:
      snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry,
        get_ssl_version_str(*(int *) params[i].val_adr));
      write(fd, pom, strlen(pom));
      break;
    case PARAM_HTTPHDR:
      {
        dllist *ptr = *((dllist **) params[i].val_adr);
        while(ptr)
        {
          httphdr *hr = (httphdr *) ptr->data;

          snprintf(pom, sizeof(pom), "%s %s%s %s\n", params[i].par_entry,
            hr->all ? "+" : "", hr->name, hr->val);
          write(fd, pom, strlen(pom));

          ptr = ptr->next;
        }
      }
      break;
    case PARAM_DEBUGL:
#ifdef DEBUG
      {
        char strbuf[1024];
        debug_level_construct(*((int *) params[i].val_adr), strbuf);
        snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry, strbuf);
        write(fd, pom, strlen(pom));
      }
#endif
      break;
    case PARAM_REQUEST:
      {
        dllist *ptr = *((dllist **) params[i].val_adr);
        while(ptr)
        {
          url_info *ui = (url_info *) ptr->data;

          if(ui->type != URLI_NORMAL || ui->localname)
          {
            char *p = url_info_dump(ui);
            snprintf(pom, sizeof(pom), "%s %s\n", params[i].par_entry, p);
            _free(p);
            write(fd, pom, strlen(pom));
          }

          ptr = ptr->next;
        }
      }
      break;
    case PARAM_TRANSPARENT:
      {
        http_proxy *pr = *((http_proxy **) params[i].val_adr);
        if(pr)
        {
          snprintf(pom, sizeof(pom), "%s %s:%d\n", params[i].par_entry,
            pr->addr, pr->port);
          write(fd, pom, strlen(pom));
        }
      }
      break;
    case PARAM_PROXY:
      {
        dllist *ptr = *((dllist **) params[i].val_adr);
        while(ptr)
        {
          http_proxy *pr = (http_proxy *) ptr->data;
          snprintf(pom, sizeof(pom), "%s %s:%hu\n", params[i].par_entry,
            pr->addr, pr->port);
          write(fd, pom, strlen(pom));

          ptr = ptr->next;
        }
      }
      break;
    case PARAM_FUNC:
      break;
    case PARAM_JSTRANS:
#ifdef HAVE_REGEX
      {
        dllist *ptr;
        for(ptr = cfg.js_transform; ptr; ptr = ptr->next)
        {
          js_transform_t *jt = (js_transform_t *) ptr->data;
          char *p[4];

          if(jt->type != (long) params[i].mdefault_val)
            continue;

          p[0] = escape_str(jt->re->pattern, "\\\"");
          p[1] = escape_str(jt->transform, "\\\"");
          p[2] = escape_str(jt->tag, "\\\"");
          p[3] = escape_str(jt->attrib, "\\\"");

          snprintf(pom, sizeof(pom), "%s \"%s\" \"%s\" \"%s\" \"%s\"\n",
            params[i].par_entry, p[0], p[1], p[2], p[3]);
          _free(p[0]);
          _free(p[1]);
          _free(p[2]);
          _free(p[3]);
          write(fd, pom, strlen(pom));
        }
      }
#endif
      break;
    case PARAM_NUMLIST:
      if(!params[i].mval_adr || (params[i].mval_adr &&
          (*((bool_t *) params[i].mval_adr) ==
            (bool_t) (long) params[i].mdefault_val)))
      {
        dllist *ptr = *((dllist **) params[i].val_adr);
        if(ptr)
        {
          snprintf(pom, sizeof(pom), "%s %ld", params[i].par_entry,
            (long) ptr->data);
          write(fd, pom, strlen(pom));

          j = 1;
          for(; ptr; ptr = ptr->next)
          {
            snprintf(pom, sizeof(pom), ",%ld", (long) ptr->data);
            write(fd, pom, strlen(pom));
          }
          write(fd, "\n", 1);
        }
      }
      break;
    case PARAM_FTPHS:
      {
        dllist *ptr;
        for(ptr = *((dllist **) params[i].val_adr); ptr; ptr = ptr->next)