html-parse.c

Wget很好的处理了http和ftp的下载,很值得学习的经典代码

/* HTML parser for Wget.
   Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
   2007, 2008 Free Software Foundation, Inc.

This file is part of GNU Wget.

GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License, or (at
your option) any later version.

GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Wget.  If not, see <http://www.gnu.org/licenses/>.

Additional permission under GNU GPL version 3 section 7

If you modify this program, or any covered work, by linking or
combining it with the OpenSSL project's OpenSSL library (or a
modified version of that library), containing parts covered by the
terms of the OpenSSL or SSLeay licenses, the Free Software Foundation
grants you additional permission to convey the resulting work.
Corresponding Source for a non-source form of such a combination
shall include the source code for the parts of OpenSSL used as well
as that of the covered work.  */

/* The only entry point to this module is map_html_tags(), which see.  */

/* TODO:

   - Allow hooks for callers to process contents outside tags.  This
     is needed to implement handling <style> and <script>.  The
     taginfo structure already carries the information about where the
     tags are, but this is not enough, because one would also want to
     skip the comments.  (The funny thing is that for <style> and
     <script> you *don't* want to skip comments!)

   - Create a test suite for regression testing. */

/* HISTORY:

   This is the third HTML parser written for Wget.  The first one was
   written some time during the Geturl 1.0 beta cycle, and was very
   inefficient and buggy.  It also contained some very complex code to
   remember a list of parser states, because it was supposed to be
   reentrant.

   The second HTML parser was written for Wget 1.4 (the first version
   by the name `Wget'), and was a complete rewrite.  Although the new
   parser behaved much better and made no claims of reentrancy, it
   still shared many of the fundamental flaws of the old version -- it
   only regarded HTML in terms tag-attribute pairs, where the
   attribute's value was a URL to be returned.  Any other property of
   HTML, such as <base href=...>, or strange way to specify a URL,
   such as <meta http-equiv=Refresh content="0; URL=..."> had to be
   crudely hacked in -- and the caller had to be aware of these hacks.
   Like its predecessor, this parser did not support HTML comments.

   After Wget 1.5.1 was released, I set out to write a third HTML
   parser.  The objectives of the new parser were to: (1) provide a
   clean way to analyze HTML lexically, (2) separate interpretation of
   the markup from the parsing process, (3) be as correct as possible,
   e.g. correctly skipping comments and other SGML declarations, (4)
   understand the most common errors in markup and skip them or be
   relaxed towrds them, and (5) be reasonably efficient (no regexps,
   minimum copying and minimum or no heap allocation).

   I believe this parser meets all of the above goals.  It is
   reasonably well structured, and could be relatively easily
   separated from Wget and used elsewhere.  While some of its
   intrinsic properties limit its value as a general-purpose HTML
   parser, I believe that, with minimum modifications, it could serve
   as a backend for one.

   Due to time and other constraints, this parser was not integrated
   into Wget until the version 1.7. */

/* DESCRIPTION:

   The single entry point of this parser is map_html_tags(), which
   works by calling a function you specify for each tag.  The function
   gets called with the pointer to a structure describing the tag and
   its attributes.  */

/* To test as standalone, compile with `-DSTANDALONE -I.'.  You'll
   still need Wget headers to compile.  */

#include <config.h>

#ifdef STANDALONE
# define I_REALLY_WANT_CTYPE_MACROS
#endif

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>

#include "wget.h"
#include "html-parse.h"

#ifdef STANDALONE
# undef xmalloc
# undef xrealloc
# undef xfree
# define xmalloc malloc
# define xrealloc realloc
# define xfree free

# undef ISSPACE
# undef ISDIGIT
# undef ISXDIGIT
# undef ISALPHA
# undef ISALNUM
# undef TOLOWER
# undef TOUPPER

# define ISSPACE(x) isspace (x)
# define ISDIGIT(x) isdigit (x)
# define ISXDIGIT(x) isxdigit (x)
# define ISALPHA(x) isalpha (x)
# define ISALNUM(x) isalnum (x)
# define TOLOWER(x) tolower (x)
# define TOUPPER(x) toupper (x)

struct hash_table {
  int dummy;
};
static void *
hash_table_get (const struct hash_table *ht, void *ptr)
{
  return ptr;
}
#else  /* not STANDALONE */
# include "hash.h"
#endif

/* Pool support.  A pool is a resizable chunk of memory.  It is first
   allocated on the stack, and moved to the heap if it needs to be
   larger than originally expected.  map_html_tags() uses it to store
   the zero-terminated names and values of tags and attributes.

   Thus taginfo->name, and attr->name and attr->value for each
   attribute, do not point into separately allocated areas, but into
   different parts of the pool, separated only by terminating zeros.
   This ensures minimum amount of allocation and, for most tags, no
   allocation because the entire pool is kept on the stack.  */

struct pool {
  char *contents;               /* pointer to the contents. */
  int size;                     /* size of the pool. */
  int tail;                     /* next available position index. */
  bool resized;                 /* whether the pool has been resized
                                   using malloc. */

  char *orig_contents;          /* original pool contents, usually
                                   stack-allocated.  used by POOL_FREE
                                   to restore the pool to the initial
                                   state. */
  int orig_size;
};

/* Initialize the pool to hold INITIAL_SIZE bytes of storage. */

#define POOL_INIT(p, initial_storage, initial_size) do {        \
  struct pool *P = (p);                                         \
  P->contents = (initial_storage);                              \
  P->size = (initial_size);                                     \
  P->tail = 0;                                                  \
  P->resized = false;                                           \
  P->orig_contents = P->contents;                               \
  P->orig_size = P->size;                                       \
} while (0)

/* Grow the pool to accomodate at least SIZE new bytes.  If the pool
   already has room to accomodate SIZE bytes of data, this is a no-op.  */

#define POOL_GROW(p, increase)                                  \
  GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase), \
              (p)->resized, char)

/* Append text in the range [beg, end) to POOL.  No zero-termination
   is done.  */

#define POOL_APPEND(p, beg, end) do {                   \
  const char *PA_beg = (beg);                           \
  int PA_size = (end) - PA_beg;                         \
  POOL_GROW (p, PA_size);                               \
  memcpy ((p)->contents + (p)->tail, PA_beg, PA_size);  \
  (p)->tail += PA_size;                                 \
} while (0)

/* Append one character to the pool.  Can be used to zero-terminate
   pool strings.  */

#define POOL_APPEND_CHR(p, ch) do {             \
  char PAC_char = (ch);                         \
  POOL_GROW (p, 1);                             \
  (p)->contents[(p)->tail++] = PAC_char;        \
} while (0)

/* Forget old pool contents.  The allocated memory is not freed. */
#define POOL_REWIND(p) (p)->tail = 0

/* Free heap-allocated memory for contents of POOL.  This calls
   xfree() if the memory was allocated through malloc.  It also
   restores `contents' and `size' to their original, pre-malloc
   values.  That way after POOL_FREE, the pool is fully usable, just
   as if it were freshly initialized with POOL_INIT.  */

#define POOL_FREE(p) do {                       \
  struct pool *P = p;                           \
  if (P->resized)                               \
    xfree (P->contents);                        \
  P->contents = P->orig_contents;               \
  P->size = P->orig_size;                       \
  P->tail = 0;                                  \
  P->resized = false;                           \
} while (0)

/* Used for small stack-allocated memory chunks that might grow.  Like
   DO_REALLOC, this macro grows BASEVAR as necessary to take
   NEEDED_SIZE items of TYPE.

   The difference is that on the first resize, it will use
   malloc+memcpy rather than realloc.  That way you can stack-allocate
   the initial chunk, and only resort to heap allocation if you
   stumble upon large data.

   After the first resize, subsequent ones are performed with realloc,
   just like DO_REALLOC.  */

#define GROW_ARRAY(basevar, sizevar, needed_size, resized, type) do {           \
  long ga_needed_size = (needed_size);                                          \
  long ga_newsize = (sizevar);                                                  \
  while (ga_newsize < ga_needed_size)                                           \
    ga_newsize <<= 1;                                                           \
  if (ga_newsize != (sizevar))                                                  \
    {                                                                           \
      if (resized)                                                              \
        basevar = xrealloc (basevar, ga_newsize * sizeof (type));               \
      else                                                                      \
        {                                                                       \
          void *ga_new = xmalloc (ga_newsize * sizeof (type));                  \
          memcpy (ga_new, basevar, (sizevar) * sizeof (type));                  \
          (basevar) = ga_new;                                                   \
          resized = true;                                                       \
        }                                                                       \
      (sizevar) = ga_newsize;                                                   \
    }                                                                           \
} while (0)

/* Test whether n+1-sized entity name fits in P.  We don't support
   IE-style non-terminated entities, e.g. "&ltfoo" -> "<foo".
   However, "&lt;foo" will work, as will "&lt!foo", "&lt", etc.  In
   other words an entity needs to be terminated by either a
   non-alphanumeric or the end of string.  */
#define FITS(p, n) (p + n == end || (p + n < end && !ISALNUM (p[n])))

/* Macros that test entity names by returning true if P is followed by
   the specified characters.  */
#define ENT1(p, c0) (FITS (p, 1) && p[0] == c0)
#define ENT2(p, c0, c1) (FITS (p, 2) && p[0] == c0 && p[1] == c1)
#define ENT3(p, c0, c1, c2) (FITS (p, 3) && p[0]==c0 && p[1]==c1 && p[2]==c2)

/* Increment P by INC chars.  If P lands at a semicolon, increment it
   past the semicolon.  This ensures that e.g. "&lt;foo" is converted
   to "<foo", but "&lt,foo" to "<,foo".  */
#define SKIP_SEMI(p, inc) (p += inc, p < end && *p == ';' ? ++p : p)

/* Decode the HTML character entity at *PTR, considering END to be end
   of buffer.  It is assumed that the "&" character that marks the
   beginning of the entity has been seen at *PTR-1.  If a recognized
   ASCII entity is seen, it is returned, and *PTR is moved to the end
   of the entity.  Otherwise, -1 is returned and *PTR left unmodified.

   The recognized entities are: &lt, &gt, &amp, &apos, and &quot.  */

static int
decode_entity (const char **ptr, const char *end)
{
  const char *p = *ptr;
  int value = -1;

  if (++p == end)
    return -1;

  switch (*p++)
    {
    case '#':
      /* Process numeric entities "&#DDD;" and "&#xHH;".  */
      {
        int digits = 0;
        value = 0;
        if (*p == 'x')
          for (++p; value < 256 && p < end && ISXDIGIT (*p); p++, digits++)
            value = (value << 4) + XDIGIT_TO_NUM (*p);
        else
          for (; value < 256 && p < end && ISDIGIT (*p); p++, digits++)
            value = (value * 10) + (*p - '0');
        if (!digits)
          return -1;
        /* Don't interpret 128+ codes and NUL because we cannot
           portably reinserted them into HTML.  */
        if (!value || (value & ~0x7f))
          return -1;
        *ptr = SKIP_SEMI (p, 0);
        return value;
      }
    /* Process named ASCII entities.  */
    case 'g':
      if (ENT1 (p, 't'))
        value = '>', *ptr = SKIP_SEMI (p, 1);
      break;
    case 'l':
      if (ENT1 (p, 't'))
        value = '<', *ptr = SKIP_SEMI (p, 1);
      break;
    case 'a':
      if (ENT2 (p, 'm', 'p'))
        value = '&', *ptr = SKIP_SEMI (p, 2);
      else if (ENT3 (p, 'p', 'o', 's'))
        /* handle &apos for the sake of the XML/XHTML crowd. */
        value = '\'', *ptr = SKIP_SEMI (p, 3);
      break;
    case 'q':
      if (ENT3 (p, 'u', 'o', 't'))
        value = '\"', *ptr = SKIP_SEMI (p, 3);
      break;
    }
  return value;
}
#undef ENT1
#undef ENT2
#undef ENT3
#undef FITS
#undef SKIP_SEMI

enum {
  AP_DOWNCASE           = 1,
  AP_DECODE_ENTITIES    = 2,
  AP_TRIM_BLANKS        = 4
};

/* Copy the text in the range [BEG, END) to POOL, optionally
   performing operations specified by FLAGS.  FLAGS may be any
   combination of AP_DOWNCASE, AP_DECODE_ENTITIES and AP_TRIM_BLANKS
   with the following meaning:

   * AP_DOWNCASE -- downcase all the letters;

   * AP_DECODE_ENTITIES -- decode the named and numeric entities in
     the ASCII range when copying the string.