host.c

Wget很好的处理了http和ftp的下载,很值得学习的经典代码

            }
        }
      else if (ch == '.' && saw_digit)
        {
          if (octets == 4)
            return false;
          val = 0;
          saw_digit = false;
        }
      else
        return false;
    }
  if (octets < 4)
    return false;
  
  return true;
}

bool
is_valid_ipv6_address (const char *str, const char *end)
{
  /* Use lower-case for these to avoid clash with system headers.  */
  enum {
    ns_inaddrsz  = 4,
    ns_in6addrsz = 16,
    ns_int16sz   = 2
  };

  const char *curtok;
  int tp;
  const char *colonp;
  bool saw_xdigit;
  unsigned int val;

  tp = 0;
  colonp = NULL;

  if (str == end)
    return false;
  
  /* Leading :: requires some special handling. */
  if (*str == ':')
    {
      ++str;
      if (str == end || *str != ':')
        return false;
    }

  curtok = str;
  saw_xdigit = false;
  val = 0;

  while (str < end)
    {
      int ch = *str++;

      /* if ch is a number, add it to val. */
      if (ISXDIGIT (ch))
        {
          val <<= 4;
          val |= XDIGIT_TO_NUM (ch);
          if (val > 0xffff)
            return false;
          saw_xdigit = true;
          continue;
        }

      /* if ch is a colon ... */
      if (ch == ':')
        {
          curtok = str;
          if (!saw_xdigit)
            {
              if (colonp != NULL)
                return false;
              colonp = str + tp;
              continue;
            }
          else if (str == end)
            return false;
          if (tp > ns_in6addrsz - ns_int16sz)
            return false;
          tp += ns_int16sz;
          saw_xdigit = false;
          val = 0;
          continue;
        }

      /* if ch is a dot ... */
      if (ch == '.' && (tp <= ns_in6addrsz - ns_inaddrsz)
          && is_valid_ipv4_address (curtok, end) == 1)
        {
          tp += ns_inaddrsz;
          saw_xdigit = false;
          break;
        }
    
      return false;
    }

  if (saw_xdigit)
    {
      if (tp > ns_in6addrsz - ns_int16sz) 
        return false;
      tp += ns_int16sz;
    }

  if (colonp != NULL)
    {
      if (tp == ns_in6addrsz) 
        return false;
      tp = ns_in6addrsz;
    }

  if (tp != ns_in6addrsz)
    return false;

  return true;
}

/* Simple host cache, used by lookup_host to speed up resolving.  The
   cache doesn't handle TTL because Wget is a fairly short-lived
   application.  Refreshing is attempted when connect fails, though --
   see connect_to_host.  */

/* Mapping between known hosts and to lists of their addresses. */
static struct hash_table *host_name_addresses_map;


/* Return the host's resolved addresses from the cache, if
   available.  */

static struct address_list *
cache_query (const char *host)
{
  struct address_list *al;
  if (!host_name_addresses_map)
    return NULL;
  al = hash_table_get (host_name_addresses_map, host);
  if (al)
    {
      DEBUGP (("Found %s in host_name_addresses_map (%p)\n", host, al));
      ++al->refcount;
      return al;
    }
  return NULL;
}

/* Cache the DNS lookup of HOST.  Subsequent invocations of
   lookup_host will return the cached value.  */

static void
cache_store (const char *host, struct address_list *al)
{
  if (!host_name_addresses_map)
    host_name_addresses_map = make_nocase_string_hash_table (0);

  ++al->refcount;
  hash_table_put (host_name_addresses_map, xstrdup_lower (host), al);

  IF_DEBUG
    {
      int i;
      debug_logprintf ("Caching %s =>", host);
      for (i = 0; i < al->count; i++)
        debug_logprintf (" %s", print_address (al->addresses + i));
      debug_logprintf ("\n");
    }
}

/* Remove HOST from the DNS cache.  Does nothing is HOST is not in
   the cache.  */

static void
cache_remove (const char *host)
{
  struct address_list *al;
  if (!host_name_addresses_map)
    return;
  al = hash_table_get (host_name_addresses_map, host);
  if (al)
    {
      address_list_release (al);
      hash_table_remove (host_name_addresses_map, host);
    }
}

/* Look up HOST in DNS and return a list of IP addresses.

   This function caches its result so that, if the same host is passed
   the second time, the addresses are returned without DNS lookup.
   (Use LH_REFRESH to force lookup, or set opt.dns_cache to 0 to
   globally disable caching.)

   The order of the returned addresses is affected by the setting of
   opt.prefer_family: if it is set to prefer_ipv4, IPv4 addresses are
   placed at the beginning; if it is prefer_ipv6, IPv6 ones are placed
   at the beginning; otherwise, the order is left intact.  The
   relative order of addresses with the same family is left
   undisturbed in either case.

   FLAGS can be a combination of:
     LH_SILENT  - don't print the "resolving ... done" messages.
     LH_BIND    - resolve addresses for use with bind, which under
                  IPv6 means to use AI_PASSIVE flag to getaddrinfo.
                  Passive lookups are not cached under IPv6.
     LH_REFRESH - if HOST is cached, remove the entry from the cache
                  and resolve it anew.  */

struct address_list *
lookup_host (const char *host, int flags)
{
  struct address_list *al;
  bool silent = !!(flags & LH_SILENT);
  bool use_cache;
  bool numeric_address = false;
  double timeout = opt.dns_timeout;

#ifndef ENABLE_IPV6
  /* If we're not using getaddrinfo, first check if HOST specifies a
     numeric IPv4 address.  Some implementations of gethostbyname
     (e.g. the Ultrix one and possibly Winsock) don't accept
     dotted-decimal IPv4 addresses.  */
  {
    uint32_t addr_ipv4 = (uint32_t)inet_addr (host);
    if (addr_ipv4 != (uint32_t) -1)
      {
        /* No need to cache host->addr relation, just return the
           address.  */
        char *vec[2];
        vec[0] = (char *)&addr_ipv4;
        vec[1] = NULL;
        return address_list_from_ipv4_addresses (vec);
      }
  }
#else  /* ENABLE_IPV6 */
  /* If we're using getaddrinfo, at least check whether the address is
     already numeric, in which case there is no need to print the
     "Resolving..." output.  (This comes at no additional cost since
     the is_valid_ipv*_address are already required for
     url_parse.)  */
  {
    const char *end = host + strlen (host);
    if (is_valid_ipv4_address (host, end) || is_valid_ipv6_address (host, end))
      numeric_address = true;
  }
#endif

  /* Cache is normally on, but can be turned off with --no-dns-cache.
     Don't cache passive lookups under IPv6.  */
  use_cache = opt.dns_cache;
#ifdef ENABLE_IPV6
  if ((flags & LH_BIND) || numeric_address)
    use_cache = false;
#endif

  /* Try to find the host in the cache so we don't need to talk to the
     resolver.  If LH_REFRESH is requested, remove HOST from the cache
     instead.  */
  if (use_cache)
    {
      if (!(flags & LH_REFRESH))
        {
          al = cache_query (host);
          if (al)
            return al;
        }
      else
        cache_remove (host);
    }

  /* No luck with the cache; resolve HOST. */

  if (!silent && !numeric_address)
    logprintf (LOG_VERBOSE, _("Resolving %s... "), escnonprint (host));

#ifdef ENABLE_IPV6
  {
    int err;
    struct addrinfo hints, *res;

    xzero (hints);
    hints.ai_socktype = SOCK_STREAM;
    if (opt.ipv4_only)
      hints.ai_family = AF_INET;
    else if (opt.ipv6_only)
      hints.ai_family = AF_INET6;
    else
      /* We tried using AI_ADDRCONFIG, but removed it because: it
         misinterprets IPv6 loopbacks, it is broken on AIX 5.1, and
         it's unneeded since we sort the addresses anyway.  */
        hints.ai_family = AF_UNSPEC;

    if (flags & LH_BIND)
      hints.ai_flags |= AI_PASSIVE;

#ifdef AI_NUMERICHOST
    if (numeric_address)
      {
        /* Where available, the AI_NUMERICHOST hint can prevent costly
           access to DNS servers.  */
        hints.ai_flags |= AI_NUMERICHOST;
        timeout = 0;            /* no timeout needed when "resolving"
                                   numeric hosts -- avoid setting up
                                   signal handlers and such. */
      }
#endif

    err = getaddrinfo_with_timeout (host, NULL, &hints, &res, timeout);
    if (err != 0 || res == NULL)
      {
        if (!silent)
          logprintf (LOG_VERBOSE, _("failed: %s.\n"),
                     err != EAI_SYSTEM ? gai_strerror (err) : strerror (errno));
        return NULL;
      }
    al = address_list_from_addrinfo (res);
    freeaddrinfo (res);
    if (!al)
      {
        logprintf (LOG_VERBOSE,
                   _("failed: No IPv4/IPv6 addresses for host.\n"));
        return NULL;
      }

    /* Reorder addresses so that IPv4 ones (or IPv6 ones, as per
       --prefer-family) come first.  Sorting is stable so the order of
       the addresses with the same family is undisturbed.  */
    if (al->count > 1 && opt.prefer_family != prefer_none)
      stable_sort (al->addresses, al->count, sizeof (ip_address),
                   opt.prefer_family == prefer_ipv4
                   ? cmp_prefer_ipv4 : cmp_prefer_ipv6);
  }
#else  /* not ENABLE_IPV6 */
  {
    struct hostent *hptr = gethostbyname_with_timeout (host, timeout);
    if (!hptr)
      {
        if (!silent)
          {
            if (errno != ETIMEDOUT)
              logprintf (LOG_VERBOSE, _("failed: %s.\n"),
                         host_errstr (h_errno));
            else
              logputs (LOG_VERBOSE, _("failed: timed out.\n"));
          }
        return NULL;
      }
    /* Do older systems have h_addr_list?  */
    al = address_list_from_ipv4_addresses (hptr->h_addr_list);
  }
#endif /* not ENABLE_IPV6 */

  /* Print the addresses determined by DNS lookup, but no more than
     three.  */
  if (!silent && !numeric_address)
    {
      int i;
      int printmax = al->count <= 3 ? al->count : 3;
      for (i = 0; i < printmax; i++)
        {
          logputs (LOG_VERBOSE, print_address (al->addresses + i));
          if (i < printmax - 1)
            logputs (LOG_VERBOSE, ", ");
        }
      if (printmax != al->count)
        logputs (LOG_VERBOSE, ", ...");
      logputs (LOG_VERBOSE, "\n");
    }

  /* Cache the lookup information. */
  if (use_cache)
    cache_store (host, al);

  return al;
}

/* Determine whether a URL is acceptable to be followed, according to
   a list of domains to accept.  */
bool
accept_domain (struct url *u)
{
  assert (u->host != NULL);
  if (opt.domains)
    {
      if (!sufmatch ((const char **)opt.domains, u->host))
        return false;
    }
  if (opt.exclude_domains)
    {
      if (sufmatch ((const char **)opt.exclude_domains, u->host))
        return false;
    }
  return true;
}

/* Check whether WHAT is matched in LIST, each element of LIST being a
   pattern to match WHAT against, using backward matching (see
   match_backwards() in utils.c).

   If an element of LIST matched, 1 is returned, 0 otherwise.  */
bool
sufmatch (const char **list, const char *what)
{
  int i, j, k, lw;

  lw = strlen (what);
  for (i = 0; list[i]; i++)
    {
      for (j = strlen (list[i]), k = lw; j >= 0 && k >= 0; j--, k--)
        if (TOLOWER (list[i][j]) != TOLOWER (what[k]))
          break;
      /* The domain must be first to reach to beginning.  */
      if (j == -1)
        return true;
    }
  return false;
}

void
host_cleanup (void)
{
  if (host_name_addresses_map)
    {
      hash_table_iterator iter;
      for (hash_table_iterate (host_name_addresses_map, &iter);
           hash_table_iter_next (&iter);
           )
        {
          char *host = iter.key;
          struct address_list *al = iter.value;
          xfree (host);
          assert (al->refcount == 1);
          address_list_delete (al);
        }
      hash_table_destroy (host_name_addresses_map);
      host_name_addresses_map = NULL;
    }
}