ftp.c

wget (command line browser) source code

/* File Transfer Protocol support.
   Copyright (C) 1995, 1996, 1997, 1998, 2000, 2001
   Free Software Foundation, Inc.

This file is part of GNU Wget.

GNU Wget is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

GNU Wget is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Wget; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

In addition, as a special exception, the Free Software Foundation
gives permission to link the code of its release of Wget with the
OpenSSL project's "OpenSSL" library (or with modified versions of it
that use the same license as the "OpenSSL" library), and distribute
the linked executables.  You must obey the GNU General Public License
in all respects for all of the code used other than "OpenSSL".  If you
modify this file, you may extend this exception to your version of the
file, but you are not obligated to do so.  If you do not wish to do
so, delete this exception statement from your version.  */

#include <config.h>

#include <stdio.h>
#include <stdlib.h>
#ifdef HAVE_STRING_H
# include <string.h>
#else
# include <strings.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#include <sys/types.h>
#include <assert.h>
#include <errno.h>

#include "wget.h"
#include "utils.h"
#include "url.h"
#include "rbuf.h"
#include "retr.h"
#include "ftp.h"
#include "connect.h"
#include "host.h"
#include "netrc.h"
#include "convert.h"		/* for downloaded_file */

#ifndef errno
extern int errno;
#endif

extern LARGE_INT total_downloaded_bytes;

/* File where the "ls -al" listing will be saved.  */
#define LIST_FILENAME ".listing"

extern char ftp_last_respline[];

typedef struct
{
  int st;			/* connection status */
  int cmd;			/* command code */
  struct rbuf rbuf;		/* control connection buffer */
  double dltime;		/* time of the download in msecs */
  enum stype rs;		/* remote system reported by ftp server */ 
  char *id;			/* initial directory */
  char *target;			/* target file name */
  struct url *proxy;		/* FTWK-style proxy */
} ccon;


/* Look for regexp "( *[0-9]+ *byte" (literal parenthesis) anywhere in
   the string S, and return the number converted to long, if found, 0
   otherwise.  */
static long
ftp_expected_bytes (const char *s)
{
  long res;

  while (1)
    {
      while (*s && *s != '(')
	++s;
      if (!*s)
	return 0;
      for (++s; *s && ISSPACE (*s); s++);
      if (!*s)
	return 0;
      if (!ISDIGIT (*s))
	continue;
      res = 0;
      do
	{
	  res = (*s - '0') + 10 * res;
	  ++s;
	}
      while (*s && ISDIGIT (*s));
      if (!*s)
	return 0;
      while (*s && ISSPACE (*s))
	++s;
      if (!*s)
	return 0;
      if (TOLOWER (*s) != 'b')
	continue;
      if (strncasecmp (s, "byte", 4))
	continue;
      else
	break;
    }
  return res;
}

/* Retrieves a file with denoted parameters through opening an FTP
   connection to the server.  It always closes the data connection,
   and closes the control connection in case of error.  */
static uerr_t
getftp (struct url *u, long *len, long restval, ccon *con)
{
  int csock, dtsock, res;
  uerr_t err;
  FILE *fp;
  char *user, *passwd, *respline;
  char *tms, *tmrate;
  int cmd = con->cmd;
  int pasv_mode_open = 0;
  long expected_bytes = 0L;

  assert (con != NULL);
  assert (con->target != NULL);

  /* Debug-check of the sanity of the request by making sure that LIST
     and RETR are never both requested (since we can handle only one
     at a time.  */
  assert (!((cmd & DO_LIST) && (cmd & DO_RETR)));
  /* Make sure that at least *something* is requested.  */
  assert ((cmd & (DO_LIST | DO_CWD | DO_RETR | DO_LOGIN)) != 0);

  user = u->user;
  passwd = u->passwd;
  search_netrc (u->host, (const char **)&user, (const char **)&passwd, 1);
  user = user ? user : opt.ftp_acc;
  passwd = passwd ? passwd : opt.ftp_pass;
  assert (user && passwd);

  dtsock = -1;
  con->dltime = 0;

  if (!(cmd & DO_LOGIN))
    csock = RBUF_FD (&con->rbuf);
  else				/* cmd & DO_LOGIN */
    {
      char type_char;
      struct address_list *al;

      char    *host = con->proxy ? con->proxy->host : u->host;
      int      port = con->proxy ? con->proxy->port : u->port;
      char *logname = user;

      if (con->proxy)
	{
	  /* If proxy is in use, log in as username@target-site. */
	  logname = xmalloc (strlen (user) + 1 + strlen (u->host) + 1);
	  sprintf (logname, "%s@%s", user, u->host);
	}

      /* Login to the server: */

      /* First: Establish the control connection.  */

      al = lookup_host (host, 0);
      if (!al)
	return HOSTERR;
      set_connection_host_name (host);
      csock = connect_to_many (al, port, 0);
      set_connection_host_name (NULL);
      address_list_release (al);

      if (csock < 0)
	return CONNECT_ERROR (errno);

      if (cmd & LEAVE_PENDING)
	rbuf_initialize (&con->rbuf, csock);
      else
	rbuf_uninitialize (&con->rbuf);

      /* Since this is a new connection, we may safely discard
	 anything left in the buffer.  */
      rbuf_discard (&con->rbuf);

      /* Second: Login with proper USER/PASS sequence.  */
      logprintf (LOG_VERBOSE, _("Logging in as %s ... "), user);
      if (opt.server_response)
	logputs (LOG_ALWAYS, "\n");
      err = ftp_login (&con->rbuf, logname, passwd);

      if (con->proxy)
	xfree (logname);

      /* FTPRERR, FTPSRVERR, WRITEFAILED, FTPLOGREFUSED, FTPLOGINC */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	  break;
	case FTPSRVERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("Error in server greeting.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	  break;
	case WRITEFAILED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Write failed, closing control connection.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	  break;
	case FTPLOGREFUSED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("The server refuses login.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return FTPLOGREFUSED;
	  break;
	case FTPLOGINC:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("Login incorrect.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return FTPLOGINC;
	  break;
	case FTPOK:
	  if (!opt.server_response)
	    logputs (LOG_VERBOSE, _("Logged in!\n"));
	  break;
	default:
	  abort ();
	  exit (1);
	  break;
	}
      /* Third: Get the system type */
      if (!opt.server_response)
	logprintf (LOG_VERBOSE, "==> SYST ... ");
      err = ftp_syst (&con->rbuf, &con->rs);
      /* FTPRERR */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	  break;
	case FTPSRVERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Server error, can't determine system type.\n"));
	  break;
	case FTPOK:
	  /* Everything is OK.  */
	  break;
	default:
	  abort ();
	  break;
	}
      if (!opt.server_response && err != FTPSRVERR)
	logputs (LOG_VERBOSE, _("done.    "));

      /* Fourth: Find the initial ftp directory */

      if (!opt.server_response)
	logprintf (LOG_VERBOSE, "==> PWD ... ");
      err = ftp_pwd(&con->rbuf, &con->id);
      /* FTPRERR */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	  break;
	case FTPSRVERR :
	  /* PWD unsupported -- assume "/". */
	  FREE_MAYBE (con->id);
	  con->id = xstrdup ("/");
	  break;
	case FTPOK:
	  /* Everything is OK.  */
	  break;
	default:
	  abort ();
	  break;
	}
      /* VMS will report something like "PUB$DEVICE:[INITIAL.FOLDER]".
         Convert it to "/INITIAL/FOLDER" */ 
      if (con->rs == ST_VMS)
        {
          char *path = strchr (con->id, '[');
	  char *pathend = path ? strchr (path + 1, ']') : NULL;
	  if (!path || !pathend)
	    DEBUGP (("Initial VMS directory not in the form [...]!\n"));
	  else
	    {
	      char *idir = con->id;
	      DEBUGP (("Preprocessing the initial VMS directory\n"));
	      DEBUGP (("  old = '%s'\n", con->id));
	      /* We do the conversion in-place by copying the stuff
		 between [ and ] to the beginning, and changing dots
		 to slashes at the same time.  */
	      *idir++ = '/';
	      for (++path; path < pathend; path++, idir++)
		*idir = *path == '.' ? '/' : *path;
	      *idir = '\0';
	      DEBUGP (("  new = '%s'\n\n", con->id));
	    }
	}
      if (!opt.server_response)
	logputs (LOG_VERBOSE, _("done.\n"));

      /* Fifth: Set the FTP type.  */
      type_char = ftp_process_type (u->params);
      if (!opt.server_response)
	logprintf (LOG_VERBOSE, "==> TYPE %c ... ", type_char);
      err = ftp_type (&con->rbuf, type_char);
      /* FTPRERR, WRITEFAILED, FTPUNKNOWNTYPE */
      switch (err)
	{
	case FTPRERR:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET, _("\
Error in server response, closing control connection.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	  break;
	case WRITEFAILED:
	  logputs (LOG_VERBOSE, "\n");
	  logputs (LOG_NOTQUIET,
		   _("Write failed, closing control connection.\n"));
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	  break;
	case FTPUNKNOWNTYPE:
	  logputs (LOG_VERBOSE, "\n");
	  logprintf (LOG_NOTQUIET,
		     _("Unknown type `%c', closing control connection.\n"),
		     type_char);
	  CLOSE (csock);
	  rbuf_uninitialize (&con->rbuf);
	  return err;
	case FTPOK:
	  /* Everything is OK.  */
	  break;
	default:
	  abort ();
	  break;
	}
      if (!opt.server_response)
	logputs (LOG_VERBOSE, _("done.  "));
    } /* do login */

  if (cmd & DO_CWD)
    {
      if (!*u->dir)
	logputs (LOG_VERBOSE, _("==> CWD not needed.\n"));
      else
	{
	  char *target = u->dir;

	  DEBUGP (("changing working directory\n"));

	  /* Change working directory.  To change to a non-absolute
	     Unix directory, we need to prepend initial directory
	     (con->id) to it.  Absolute directories "just work".

	     A relative directory is one that does not begin with '/'
	     and, on non-Unix OS'es, one that doesn't begin with
	     "[a-z]:".

	     This is not done for OS400, which doesn't use
	     "/"-delimited directories, nor does it support directory
	     hierarchies.  "CWD foo" followed by "CWD bar" leaves us
	     in "bar", not in "foo/bar", as would be customary
	     elsewhere.  */

	  if (target[0] != '/'
	      && !(con->rs != ST_UNIX
		   && ISALPHA (target[0])
		   && target[1] == ':')
	      && con->rs != ST_OS400)
	    {
	      int idlen = strlen (con->id);
	      char *ntarget, *p;

	      /* Strip trailing slash(es) from con->id. */
	      while (idlen > 0 && con->id[idlen - 1] == '/')
		--idlen;
	      p = ntarget = (char *)alloca (idlen + 1 + strlen (u->dir) + 1);
	      memcpy (p, con->id, idlen);
	      p += idlen;
	      *p++ = '/';
	      strcpy (p, target);

              DEBUGP (("Prepended initial PWD to relative path:\n"));
              DEBUGP (("   pwd: '%s'\n   old: '%s'\n  new: '%s'\n",
		       con->id, target, ntarget));
	      target = ntarget;
	    }

	  /* If the FTP host runs VMS, we will have to convert the absolute
             directory path in UNIX notation to absolute directory path in
             VMS notation as VMS FTP servers do not like UNIX notation of
             absolute paths.  "VMS notation" is [dir.subdir.subsubdir]. */

	  if (con->rs == ST_VMS)
	    {
	      char *tmpp;
	      char *ntarget = (char *)alloca (strlen (target) + 2);
	      /* We use a converted initial dir, so directories in
                 TARGET will be separated with slashes, something like
                 "/INITIAL/FOLDER/DIR/SUBDIR".  Convert that to
                 "[INITIAL.FOLDER.DIR.SUBDIR]".  */
	      strcpy (ntarget, target);
	      assert (*ntarget == '/');
	      *ntarget = '[';
	      for (tmpp = ntarget + 1; *tmpp; tmpp++)
		if (*tmpp == '/')
		  *tmpp = '.';
	      *tmpp++ = ']';
	      *tmpp = '\0';
              DEBUGP (("Changed file name to VMS syntax:\n"));
              DEBUGP (("  Unix: '%s'\n  VMS: '%s'\n", target, ntarget));
	      target = ntarget;
	    }

	  if (!opt.server_response)