htmlparser.c

来自「网络爬虫程序」· C语言 代码 · 共 1,536 行 · 第 1/3 页

    }
  }
}

void html_parser_url_to_local(html_parser_t * hpinfo, char *stack,
  html_rewrite_info_t * rinfo)
{
  url *urlp, *before_url;
  char *anchor, *fn;
  int is_local;

  if(!hpinfo->rewrite || rinfo->all_to_remote)
    return;

  urlp = url_parse(hpinfo->tag_attrib);
  assert(urlp->type != URLT_FROMPARENT);

  if(urlp->type == URLT_FILE || !prottable[urlp->type].supported)
  {
    free_deep_url(urlp);
    _free(urlp);
    return;
  }

  anchor = url_get_anchor_name(urlp);

  /*******************************************/
  /* for better performance with info files  */
  /* we should rather use filename generated */
  /* for previous occurence of this URL      */
  /*******************************************/
  before_url = url_was_befor(urlp);

  if(!before_url)
  {
    dllist *ptr;

    ptr = dllist_find2(rinfo->einfo->urls, (dllist_t) urlp,
    dllist_url_compare);
    if(ptr)
      before_url = (url *) ptr->data;
  }

  if(before_url)
    fn = url_to_filename(before_url, TRUE);
  else
    fn = url_to_filename(urlp, FALSE);

  is_local = !access(fn, R_OK);

  if(is_local || rinfo->all_to_local ||
    (rinfo->selected_to_local && before_url) ||
    url_compare(urlp, hpinfo->doc_url))
  {
    char *actname, *relname;
    struct stat estat;

    if(is_local && !stat(fn, &estat) && S_ISDIR(estat.st_mode))
      fn = tl_str_concat(NULL, fn, "/", priv_cfg.index_name, NULL);
    else
      fn = tl_strdup(fn);

    actname = url_to_filename(hpinfo->doc_url, FALSE);

    /* it seems that lynx and netscape behave different on   */
    /* empty HREFs, so use it only in case when is specified */
    /* partname of document (#xxx)         */
    /* this is URL of current document -> "" */
    if(anchor && !strcmp(actname, fn))
      relname = tl_strdup("");
    else
      relname = get_relative_path(actname, fn);

    _free(fn);

    /* workaround for -sel_to_local && -nostore_index */
    if(rinfo->selected_to_local && !rinfo->store_index)
    {
      char *slp = strrchr(relname, '/');

      if(!slp)
        slp = relname;
      else
        slp++;

      if(!strcmp(slp, priv_cfg.index_name))
        *slp = '\0';
    }

    if(anchor)
      relname = tl_str_concat(relname, "#", anchor, NULL);

    DEBUG_HTML("Rewriting URL (to loc) - %s -> %s\n", hpinfo->tag_attrib,
      relname);

    _free(hpinfo->tag_attrib);
    hpinfo->tag_attrib = relname;
  }
  free_deep_url(urlp);
  _free(urlp);
}

void html_parser_remove_advertisement(html_parser_t * hpinfo, char *stack,
  void *data)
{
#ifdef HAVE_REGEX
  int is_adver = FALSE;

  if(hpinfo->current_tag->type != HTML_TAG_IMG ||
    hpinfo->current_attrib->type != HTML_ATTRIB_SRC)
    return;

  if(cfg.remove_adv && priv_cfg.advert_res)
  {
    dllist *ptr = priv_cfg.advert_res;

    while(ptr)
    {
      if(re_pmatch((re_entry *) ptr->data, hpinfo->tag_attrib))
      {
        DEBUG_HTML("Removing advert URL - %s\n", hpinfo->tag_attrib);
        is_adver = TRUE;
        break;
      }
      ptr = ptr->next;
    }
  }

  if(is_adver)
  {
    int lp = strlen(ADVERT_PREFIX);
    int ls = strlen(ADVERT_SUFFIX);

    html_parser_SEND(hpinfo);
    html_parser_SEXPAND(hpinfo, (lp + ls));

    memmove(hpinfo->stack + lp, hpinfo->stack, strlen(hpinfo->stack) + 1);
    memcpy(hpinfo->stack, ADVERT_PREFIX, lp);
    strcat(hpinfo->stack, ADVERT_SUFFIX);
  }
#endif
}

void html_parser_change_url(html_parser_t * hpinfo, char *stack,
  html_change_info_t * chinfo)
{
  url *urlp;

  urlp = url_parse(hpinfo->tag_attrib);
  assert(urlp->type != URLT_FROMPARENT);

  if(urlp->type == URLT_FILE || !prottable[urlp->type].supported)
  {
    free_deep_url(urlp);
    _free(urlp);
    return;
  }

  if(url_compare(urlp, chinfo->url_old))
  {
    DEBUG_HTML("Rewriting URL (change) - %s -> %s\n",
      chinfo->url_new, chinfo->url_new);

    _free(hpinfo->tag_attrib);
    hpinfo->tag_attrib = tl_strdup(chinfo->url_new);
  }

  free_deep_url(urlp);
  _free(urlp);
}

/********************************************************/
/* functions for processing CSS parts of HTML documents */
/********************************************************/
void html_parser_style_to_absolute_urls(html_parser_t * hpinfo, char *stack,
  void *data)
{
  char *alttag;

  if(!hpinfo->rewrite)
    return;

  if(hpinfo->tag_attrib)
  {
    alttag = css_to_absolute_links(hpinfo->doc_url,
      hpinfo->tag_attrib, hpinfo->base, hpinfo->baset);
    _free(hpinfo->tag_attrib);
    hpinfo->tag_attrib = alttag;
  }
  else
  {
    int l;

    alttag = css_to_absolute_links(hpinfo->doc_url,
      hpinfo->stack, hpinfo->base, hpinfo->baset);

    l = strlen(alttag);
    if(l > hpinfo->stack_offset)
    {
      hpinfo->stack_offset = 0;
      html_parser_SEXPAND(hpinfo, l);
    }
    memcpy(hpinfo->stack, alttag, l + 1);
    hpinfo->stack_offset = l;
    _free(alttag);
  }
}

void html_parser_get_style_urls(html_parser_t * hpinfo, char *stack,
  html_extract_info_t * einfo)
{
  dllist *pv;
  if(!cfg.read_css)
  {                             /* don't fetch from css if not wanted */
    return;
  }
  if(hpinfo->tag_attrib)
    pv = css_get_all_links(hpinfo->doc_url, hpinfo->tag_attrib,
      hpinfo->base, hpinfo->baset, einfo->no_limits);
  else
    pv = css_get_all_links(hpinfo->doc_url, hpinfo->stack,
      hpinfo->base, hpinfo->baset, einfo->no_limits);

  einfo->urls = dllist_concat(einfo->urls, pv);
}

void html_parser_style_to_local_urls(html_parser_t * hpinfo, char *stack,
  html_rewrite_info_t * rinfo)
{
  char *alttag;

  if(!hpinfo->rewrite || rinfo->all_to_remote)
    return;

  if(hpinfo->tag_attrib)
  {
    alttag = css_remote_to_local_links(hpinfo->doc_url,
      hpinfo->tag_attrib, rinfo->all_to_local,
      rinfo->selected_to_local, hpinfo->base, hpinfo->baset);
    _free(hpinfo->tag_attrib);
    hpinfo->tag_attrib = alttag;
  }
  else
  {
    int l;

    alttag = css_remote_to_local_links(hpinfo->doc_url,
      hpinfo->stack, rinfo->all_to_local,
      rinfo->selected_to_local, hpinfo->base, hpinfo->baset);

    l = strlen(alttag);
    if(l > hpinfo->stack_offset)
    {
      hpinfo->stack_offset = 0;
      html_parser_SEXPAND(hpinfo, l);
    }
    memcpy(hpinfo->stack, alttag, l + 1);
    hpinfo->stack_offset = l;
    _free(alttag);
  }
}

void html_parser_style_change_url(html_parser_t * hpinfo, char *stack,
  html_change_info_t * chinfo)
{
  char *alttag;

  if(!hpinfo->rewrite)
    return;

  if(hpinfo->tag_attrib)
  {
    alttag = css_change_url(hpinfo->doc_url, hpinfo->tag_attrib,
      chinfo->url_old, chinfo->url_new);
    _free(hpinfo->tag_attrib);
    hpinfo->tag_attrib = alttag;
  }
  else
  {
    int l;

    alttag = css_change_url(hpinfo->doc_url, hpinfo->stack,
      chinfo->url_old, chinfo->url_new);

    l = strlen(alttag);
    if(l > hpinfo->stack_offset)
    {
      hpinfo->stack_offset = 0;
      html_parser_SEXPAND(hpinfo, l);
    }
    memcpy(hpinfo->stack, alttag, l + 1);
    hpinfo->stack_offset = l;
    _free(alttag);
  }
}

/***********************************************************/
/* functions for processing SCRIPTs part of HTML documents */
/***********************************************************/
void html_parser_parse_jspatterns(html_parser_t * hpinfo, char *stack,
  void *data)
{
#ifdef HAVE_REGEX
  dllist *ptr;
  int found = FALSE;
  int start, end;

  start = -1;
  end = -1;

  if(!hpinfo->tag_attrib)
    return;

  for(ptr = priv_cfg.js_patterns; ptr; ptr = ptr->next)
  {
    if(re_pmatch_sub((re_entry *) ptr->data, hpinfo->tag_attrib,
        1, &start, &end))
    {
      found = TRUE;
      break;
    }
  }

  if(found && (start >= 0))
  {
    char *saved_attrib, *new_attrib;

    saved_attrib = hpinfo->tag_attrib;

    hpinfo->tag_attrib = tl_strndup(hpinfo->tag_attrib + start, end - start);
    html_parser_call_funcs(hpinfo, hpinfo->attrib_funcs);

    new_attrib = _malloc(start + strlen(saved_attrib + end) +
      strlen(hpinfo->tag_attrib) + 1);

    strncpy(new_attrib, saved_attrib, start);
    strcpy(new_attrib + start, hpinfo->tag_attrib);
    strcat(new_attrib, saved_attrib + end);

    _free(hpinfo->tag_attrib);
    _free(saved_attrib);

    hpinfo->tag_attrib = new_attrib;
  }
#endif
}

void html_parser_parse_body_jspatterns(html_parser_t * hpinfo, char *stack,
  void *data)
{
#ifdef HAVE_REGEX
  char *stackc = NULL;
  char *p;
  int ilen;

  html_tag_t t = { HTML_TAG_HACK, "HACK",
    {{HTML_ATTRIB_HACK, "HACK", LINK_INLINE | LINK_DOWNLD},
      {HTML_ATTRIB_NULL, NULL, 0}}
  };

  if(hpinfo->tag_attrib)
    return;

  /*****************************************/
  /* quite dirty hack to make happy attrib */
  /* parsing funcs which require valid     */
  /* current_tag & current_attrib          */
  /*****************************************/
  hpinfo->current_tag = &t;
  hpinfo->current_attrib = &(t.attribs[0]);

  p = hpinfo->stack;

  while(*p)
  {
    ilen = strcspn(p, "\r\n");

    hpinfo->tag_attrib = tl_strndup(p, ilen);

    html_parser_parse_jspatterns(hpinfo, stack, data);

    if(hpinfo->rewrite)
      stackc = tl_str_concat(stackc, hpinfo->tag_attrib, "\n", NULL);
    _free(hpinfo->tag_attrib);

    p += ilen;
    p += strspn(p, "\n\r");
  }

  if(hpinfo->rewrite)
  {
    ilen = strlen(stackc);
    hpinfo->stack_offset = 0;
    html_parser_SEXPAND(hpinfo, ilen);
    memcpy(hpinfo->stack, stackc, ilen + 1);
    hpinfo->stack_offset = ilen;
    _free(stackc);
  }

  /* :-) unhack */
  hpinfo->current_tag = NULL;
  hpinfo->current_attrib = NULL;

  hpinfo->tag_attrib = NULL;
#endif
}

void html_parser_parse_body_jstransform(html_parser_t * hpinfo, char *stack,
  void *data)
{
#ifdef HAVE_REGEX
  char *p;
  int ilen;
  char *stackc = NULL;

  html_tag_t t = { HTML_TAG_HACK, "HACK",
    {{HTML_ATTRIB_HACK, "HACK", LINK_INLINE | LINK_DOWNLD},
      {HTML_ATTRIB_NULL, NULL, 0}}
  };

  if(hpinfo->tag_attrib)
    return;

  /*****************************************/
  /* quite dirty hack to make happy attrib */
  /* parsing funcs which require valid     */
  /* current_tag & current_attrib          */
  /*****************************************/
  hpinfo->current_tag = &t;
  hpinfo->current_attrib = &(t.attribs[0]);

  p = hpinfo->stack;

  while(*p)
  {
    dllist *ptr;
    char *ln;

    ilen = strcspn(p, "\r\n");

    ln = tl_strndup(p, ilen);

    for(ptr = priv_cfg.js_transform; ptr; ptr = ptr->next)
    {
      int nsub, *subs;
      js_transform_t *jt = (js_transform_t *) ptr->data;

      if(jt->tag[0])
        continue;

      if(!re_pmatch_subs(jt->re, ln, &nsub, &subs))
      {
        continue;
      }

      hpinfo->tag_attrib = js_transform_apply(jt, ln, nsub, subs);

      /*****************************************/
      /* quite dirty hack to make happy attrib */
      /* parsing funcs which require valid     */
      /* current_tag & current_attrib          */
      /*****************************************/
      hpinfo->current_tag = &t;
      hpinfo->current_attrib = &(t.attribs[0]);

      if(hpinfo->tag_attrib)
        html_parser_call_funcs(hpinfo, hpinfo->attrib_funcs);

      if(hpinfo->rewrite && jt->type == 1 && nsub)
      {
        int l = strlen(hpinfo->tag_attrib);

        ln = _realloc(ln, strlen(ln) + l + 1);
        memmove(ln + l + subs[2], ln + subs[3], strlen(ln + subs[3]) + 1);
        memcpy(ln + subs[2], hpinfo->tag_attrib, l);
      }

      _free(subs);

      /* :-) unhack */
      hpinfo->current_tag = NULL;
      hpinfo->current_attrib = NULL;

      _free(hpinfo->tag_attrib);
    }

    if(hpinfo->rewrite)
      stackc = tl_str_concat(stackc, ln, "\n", NULL);

    _free(ln);
    p += ilen;
    p += strspn(p, "\n\r");
  }

  if(hpinfo->rewrite)
  {
    ilen = strlen(stackc);
    hpinfo->stack_offset = 0;
    html_parser_SEXPAND(hpinfo, ilen);
    memcpy(hpinfo->stack, stackc, ilen + 1);
    hpinfo->stack_offset = ilen;
    _free(stackc);
  }

  /* :-) unhack */
  hpinfo->current_tag = NULL;
  hpinfo->current_attrib = NULL;

  hpinfo->tag_attrib = NULL;
#endif
}