convert.c

Wget很好的处理了http和ftp的下载,很值得学习的经典代码

   handle "." and ".." in links, so make sure they're not there
   (e.g. using path_simplify).  */

static char *
construct_relative (const char *basefile, const char *linkfile)
{
  char *link;
  int basedirs;
  const char *b, *l;
  int i, start;

  /* First, skip the initial directory components common to both
     files.  */
  start = 0;
  for (b = basefile, l = linkfile; *b == *l && *b != '\0'; ++b, ++l)
    {
      if (*b == '/')
        start = (b - basefile) + 1;
    }
  basefile += start;
  linkfile += start;

  /* With common directories out of the way, the situation we have is
     as follows:
         b - b1/b2/[...]/bfile
         l - l1/l2/[...]/lfile

     The link we're constructing needs to be:
       lnk - ../../l1/l2/[...]/lfile

     Where the number of ".."'s equals the number of bN directory
     components in B.  */

  /* Count the directory components in B. */
  basedirs = 0;
  for (b = basefile; *b; b++)
    {
      if (*b == '/')
        ++basedirs;
    }

  /* Construct LINK as explained above. */
  link = xmalloc (3 * basedirs + strlen (linkfile) + 1);
  for (i = 0; i < basedirs; i++)
    memcpy (link + 3 * i, "../", 3);
  strcpy (link + 3 * i, linkfile);
  return link;
}

/* Used by write_backup_file to remember which files have been
   written. */
static struct hash_table *converted_files;

static void
write_backup_file (const char *file, downloaded_file_t downloaded_file_return)
{
  /* Rather than just writing over the original .html file with the
     converted version, save the former to *.orig.  Note we only do
     this for files we've _successfully_ downloaded, so we don't
     clobber .orig files sitting around from previous invocations. */

  /* Construct the backup filename as the original name plus ".orig". */
  size_t         filename_len = strlen (file);
  char*          filename_plus_orig_suffix;

  if (downloaded_file_return == FILE_DOWNLOADED_AND_HTML_EXTENSION_ADDED)
    {
      /* Just write "orig" over "html".  We need to do it this way
         because when we're checking to see if we've downloaded the
         file before (to see if we can skip downloading it), we don't
         know if it's a text/html file.  Therefore we don't know yet
         at that stage that -E is going to cause us to tack on
         ".html", so we need to compare vs. the original URL plus
         ".orig", not the original URL plus ".html.orig". */
      filename_plus_orig_suffix = alloca (filename_len + 1);
      strcpy (filename_plus_orig_suffix, file);
      strcpy ((filename_plus_orig_suffix + filename_len) - 4, "orig");
    }
  else /* downloaded_file_return == FILE_DOWNLOADED_NORMALLY */
    {
      /* Append ".orig" to the name. */
      filename_plus_orig_suffix = alloca (filename_len + sizeof (".orig"));
      strcpy (filename_plus_orig_suffix, file);
      strcpy (filename_plus_orig_suffix + filename_len, ".orig");
    }

  if (!converted_files)
    converted_files = make_string_hash_table (0);

  /* We can get called twice on the same URL thanks to the
     convert_all_links() call in main().  If we write the .orig file
     each time in such a case, it'll end up containing the first-pass
     conversion, not the original file.  So, see if we've already been
     called on this file. */
  if (!string_set_contains (converted_files, file))
    {
      /* Rename <file> to <file>.orig before former gets written over. */
      if (rename (file, filename_plus_orig_suffix) != 0)
        logprintf (LOG_NOTQUIET, _("Cannot back up %s as %s: %s\n"),
                   file, filename_plus_orig_suffix, strerror (errno));

      /* Remember that we've already written a .orig backup for this file.
         Note that we never free this memory since we need it till the
         convert_all_links() call, which is one of the last things the
         program does before terminating.  BTW, I'm not sure if it would be
         safe to just set 'converted_file_ptr->string' to 'file' below,
         rather than making a copy of the string...  Another note is that I
         thought I could just add a field to the urlpos structure saying
         that we'd written a .orig file for this URL, but that didn't work,
         so I had to make this separate list.
         -- Dan Harkless <[e-mail]>

         This [adding a field to the urlpos structure] didn't work
         because convert_file() is called from convert_all_links at
         the end of the retrieval with a freshly built new urlpos
         list.
         -- Hrvoje Niksic <[e-mail]>
      */
      string_set_add (converted_files, file);
    }
}

static bool find_fragment (const char *, int, const char **, const char **);

/* Replace an attribute's original text with NEW_TEXT. */

static const char *
replace_attr (const char *p, int size, FILE *fp, const char *new_text)
{
  bool quote_flag = false;
  char quote_char = '\"';       /* use "..." for quoting, unless the
                                   original value is quoted, in which
                                   case reuse its quoting char. */
  const char *frag_beg, *frag_end;

  /* Structure of our string is:
       "...old-contents..."
       <---    size    --->  (with quotes)
     OR:
       ...old-contents...
       <---    size   -->    (no quotes)   */

  if (*p == '\"' || *p == '\'')
    {
      quote_char = *p;
      quote_flag = true;
      ++p;
      size -= 2;                /* disregard opening and closing quote */
    }
  putc (quote_char, fp);
  fputs (new_text, fp);

  /* Look for fragment identifier, if any. */
  if (find_fragment (p, size, &frag_beg, &frag_end))
    fwrite (frag_beg, 1, frag_end - frag_beg, fp);
  p += size;
  if (quote_flag)
    ++p;
  putc (quote_char, fp);

  return p;
}

/* The same as REPLACE_ATTR, but used when replacing
   <meta http-equiv=refresh content="new_text"> because we need to
   append "timeout_value; URL=" before the next_text.  */

static const char *
replace_attr_refresh_hack (const char *p, int size, FILE *fp,
                           const char *new_text, int timeout)
{
  /* "0; URL=..." */
  char *new_with_timeout = (char *)alloca (numdigit (timeout)
                                           + 6 /* "; URL=" */
                                           + strlen (new_text)
                                           + 1);
  sprintf (new_with_timeout, "%d; URL=%s", timeout, new_text);

  return replace_attr (p, size, fp, new_with_timeout);
}

/* Find the first occurrence of '#' in [BEG, BEG+SIZE) that is not
   preceded by '&'.  If the character is not found, return zero.  If
   the character is found, return true and set BP and EP to point to
   the beginning and end of the region.

   This is used for finding the fragment indentifiers in URLs.  */

static bool
find_fragment (const char *beg, int size, const char **bp, const char **ep)
{
  const char *end = beg + size;
  bool saw_amp = false;
  for (; beg < end; beg++)
    {
      switch (*beg)
        {
        case '&':
          saw_amp = true;
          break;
        case '#':
          if (!saw_amp)
            {
              *bp = beg;
              *ep = end;
              return true;
            }
          /* fallthrough */
        default:
          saw_amp = false;
        }
    }
  return false;
}

/* Quote FILE for use as local reference to an HTML file.

   We quote ? as %3F to avoid passing part of the file name as the
   parameter when browsing the converted file through HTTP.  However,
   it is safe to do this only when `--html-extension' is turned on.
   This is because converting "index.html?foo=bar" to
   "index.html%3Ffoo=bar" would break local browsing, as the latter
   isn't even recognized as an HTML file!  However, converting
   "index.html?foo=bar.html" to "index.html%3Ffoo=bar.html" should be
   safe for both local and HTTP-served browsing.

   We always quote "#" as "%23" and "%" as "%25" because those
   characters have special meanings in URLs.  */

static char *
local_quote_string (const char *file)
{
  const char *from;
  char *newname, *to;

  char *any = strpbrk (file, "?#%");
  if (!any)
    return html_quote_string (file);

  /* Allocate space assuming the worst-case scenario, each character
     having to be quoted.  */
  to = newname = (char *)alloca (3 * strlen (file) + 1);
  for (from = file; *from; from++)
    switch (*from)
      {
      case '%':
        *to++ = '%';
        *to++ = '2';
        *to++ = '5';
        break;
      case '#':
        *to++ = '%';
        *to++ = '2';
        *to++ = '3';
        break;
      case '?':
        if (opt.html_extension)
          {
            *to++ = '%';
            *to++ = '3';
            *to++ = 'F';
            break;
          }
        /* fallthrough */
      default:
        *to++ = *from;
      }
  *to = '\0';

  return html_quote_string (newname);
}

/* Book-keeping code for dl_file_url_map, dl_url_file_map,
   downloaded_html_list, and downloaded_html_set.  Other code calls
   these functions to let us know that a file has been downloaded.  */

#define ENSURE_TABLES_EXIST do {                        \
  if (!dl_file_url_map)                                 \
    dl_file_url_map = make_string_hash_table (0);       \
  if (!dl_url_file_map)                                 \
    dl_url_file_map = make_string_hash_table (0);       \
} while (0)

/* Return true if S1 and S2 are the same, except for "/index.html".
   The three cases in which it returns one are (substitute any
   substring for "foo"):

   m("foo/index.html", "foo/")  ==> 1
   m("foo/", "foo/index.html")  ==> 1
   m("foo", "foo/index.html")   ==> 1
   m("foo", "foo/"              ==> 1
   m("foo", "foo")              ==> 1  */

static bool
match_except_index (const char *s1, const char *s2)
{
  int i;
  const char *lng;

  /* Skip common substring. */
  for (i = 0; *s1 && *s2 && *s1 == *s2; s1++, s2++, i++)
    ;
  if (i == 0)
    /* Strings differ at the very beginning -- bail out.  We need to
       check this explicitly to avoid `lng - 1' reading outside the
       array.  */
    return false;

  if (!*s1 && !*s2)
    /* Both strings hit EOF -- strings are equal. */
    return true;
  else if (*s1 && *s2)
    /* Strings are randomly different, e.g. "/foo/bar" and "/foo/qux". */
    return false;
  else if (*s1)
    /* S1 is the longer one. */
    lng = s1;
  else
    /* S2 is the longer one. */
    lng = s2;

  /* foo            */            /* foo/           */
  /* foo/index.html */  /* or */  /* foo/index.html */
  /*    ^           */            /*     ^          */

  if (*lng != '/')
    /* The right-hand case. */
    --lng;

  if (*lng == '/' && *(lng + 1) == '\0')
    /* foo  */
    /* foo/ */
    return true;

  return 0 == strcmp (lng, "/index.html");
}

static int
dissociate_urls_from_file_mapper (void *key, void *value, void *arg)
{
  char *mapping_url = (char *)key;
  char *mapping_file = (char *)value;
  char *file = (char *)arg;

  if (0 == strcmp (mapping_file, file))