ftp.c

wget (command line browser) source code

  {
    /* Close or flush the file.  We have to be careful to check for
       error here.  Checking the result of fwrite() is not enough --
       errors could go unnoticed!  */
    int flush_res;
    if (!opt.dfp || con->cmd & DO_LIST)
      flush_res = fclose (fp);
    else
      flush_res = fflush (fp);
    if (flush_res == EOF)
      res = -2;
  }
  /* If get_contents couldn't write to fp, bail out.  */
  if (res == -2)
    {
      logprintf (LOG_NOTQUIET, _("%s: %s, closing control connection.\n"),
		 con->target, strerror (errno));
      CLOSE (csock);
      rbuf_uninitialize (&con->rbuf);
      return FWRITEERR;
    }
  else if (res == -1)
    {
      logprintf (LOG_NOTQUIET, _("%s (%s) - Data connection: %s; "),
		 tms, tmrate, strerror (errno));
      if (opt.server_response)
	logputs (LOG_ALWAYS, "\n");
    }

  /* Get the server to tell us if everything is retrieved.  */
  err = ftp_response (&con->rbuf, &respline);
  /* ...and empty the buffer.  */
  rbuf_discard (&con->rbuf);
  if (err != FTPOK)
    {
      xfree (respline);
      /* The control connection is decidedly closed.  Print the time
	 only if it hasn't already been printed.  */
      if (res != -1)
	logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
      logputs (LOG_NOTQUIET, _("Control connection closed.\n"));
      /* If there is an error on the control connection, close it, but
	 return FTPRETRINT, since there is a possibility that the
	 whole file was retrieved nevertheless (but that is for
	 ftp_loop_internal to decide).  */
      CLOSE (csock);
      rbuf_uninitialize (&con->rbuf);
      return FTPRETRINT;
    } /* err != FTPOK */
  /* If retrieval failed for any reason, return FTPRETRINT, but do not
     close socket, since the control connection is still alive.  If
     there is something wrong with the control connection, it will
     become apparent later.  */
  if (*respline != '2')
    {
      xfree (respline);
      if (res != -1)
	logprintf (LOG_NOTQUIET, "%s (%s) - ", tms, tmrate);
      logputs (LOG_NOTQUIET, _("Data transfer aborted.\n"));
      return FTPRETRINT;
    }
  xfree (respline);

  if (res == -1)
    {
      /* What now?  The data connection was erroneous, whereas the
	 response says everything is OK.  We shall play it safe.  */
      return FTPRETRINT;
    }

  if (!(cmd & LEAVE_PENDING))
    {
      /* I should probably send 'QUIT' and check for a reply, but this
	 is faster.  #### Is it OK, though?  */
      CLOSE (csock);
      rbuf_uninitialize (&con->rbuf);
    }
  /* If it was a listing, and opt.server_response is true,
     print it out.  */
  if (opt.server_response && (con->cmd & DO_LIST))
    {
      mkalldirs (con->target);
      fp = fopen (con->target, "r");
      if (!fp)
	logprintf (LOG_ALWAYS, "%s: %s\n", con->target, strerror (errno));
      else
	{
	  char *line;
	  /* The lines are being read with read_whole_line because of
	     no-buffering on opt.lfile.  */
	  while ((line = read_whole_line (fp)))
	    {
	      logprintf (LOG_ALWAYS, "%s\n", line);
	      xfree (line);
	    }
	  fclose (fp);
	}
    } /* con->cmd & DO_LIST && server_response */

  return RETRFINISHED;
}

/* A one-file FTP loop.  This is the part where FTP retrieval is
   retried, and retried, and retried, and...

   This loop either gets commands from con, or (if ON_YOUR_OWN is
   set), makes them up to retrieve the file given by the URL.  */
static uerr_t
ftp_loop_internal (struct url *u, struct fileinfo *f, ccon *con)
{
  int count, orig_lp;
  long restval, len;
  char *tms, *locf;
  char *tmrate = NULL;
  uerr_t err;
  struct stat st;

  if (!con->target)
    con->target = url_file_name (u);

  if (opt.noclobber && file_exists_p (con->target))
    {
      logprintf (LOG_VERBOSE,
		 _("File `%s' already there, not retrieving.\n"), con->target);
      /* If the file is there, we suppose it's retrieved OK.  */
      return RETROK;
    }

  /* Remove it if it's a link.  */
  remove_link (con->target);
  if (!opt.output_document)
    locf = con->target;
  else
    locf = opt.output_document;

  count = 0;

  if (con->st & ON_YOUR_OWN)
    con->st = ON_YOUR_OWN;

  orig_lp = con->cmd & LEAVE_PENDING ? 1 : 0;

  /* THE loop.  */
  do
    {
      /* Increment the pass counter.  */
      ++count;
      sleep_between_retrievals (count);
      if (con->st & ON_YOUR_OWN)
	{
	  con->cmd = 0;
	  con->cmd |= (DO_RETR | LEAVE_PENDING);
	  if (rbuf_initialized_p (&con->rbuf))
	    con->cmd &= ~ (DO_LOGIN | DO_CWD);
	  else
	    con->cmd |= (DO_LOGIN | DO_CWD);
	}
      else /* not on your own */
	{
	  if (rbuf_initialized_p (&con->rbuf))
	    con->cmd &= ~DO_LOGIN;
	  else
	    con->cmd |= DO_LOGIN;
	  if (con->st & DONE_CWD)
	    con->cmd &= ~DO_CWD;
	  else
	    con->cmd |= DO_CWD;
	}

      /* Assume no restarting.  */
      restval = 0L;
      if ((count > 1 || opt.always_rest)
	  && !(con->cmd & DO_LIST)
	  && file_exists_p (locf))
	if (stat (locf, &st) == 0 && S_ISREG (st.st_mode))
	  restval = st.st_size;

      /* In `-c' is used, check whether the file we're writing to
	 exists and is of non-zero length.  If so, we'll refuse to
	 truncate it if the server doesn't support continued
	 downloads.  */
      if (opt.always_rest && restval > 0)
	con->cmd |= NO_TRUNCATE;

      /* Get the current time string.  */
      tms = time_str (NULL);
      /* Print fetch message, if opt.verbose.  */
      if (opt.verbose)
	{
	  char *hurl = url_string (u, 1);
	  char tmp[15];
	  strcpy (tmp, "        ");
	  if (count > 1)
	    sprintf (tmp, _("(try:%2d)"), count);
	  logprintf (LOG_VERBOSE, "--%s--  %s\n  %s => `%s'\n",
		     tms, hurl, tmp, locf);
#ifdef WINDOWS
	  ws_changetitle (hurl, 1);
#endif
	  xfree (hurl);
	}
      /* Send getftp the proper length, if fileinfo was provided.  */
      if (f)
	len = f->size;
      else
	len = 0;
      err = getftp (u, &len, restval, con);

      if (!rbuf_initialized_p (&con->rbuf))
	con->st &= ~DONE_CWD;
      else
	con->st |= DONE_CWD;

      switch (err)
	{
	case HOSTERR: case CONREFUSED: case FWRITEERR: case FOPENERR:
	case FTPNSFOD: case FTPLOGINC: case FTPNOPASV: case CONTNOTSUPPORTED:
	  /* Fatal errors, give up.  */
	  return err;
	  break;
	case CONSOCKERR: case CONERROR: case FTPSRVERR: case FTPRERR:
	case WRITEFAILED: case FTPUNKNOWNTYPE: case CONPORTERR:
	case BINDERR: case LISTENERR: case ACCEPTERR:
	case FTPPORTERR: case FTPLOGREFUSED: case FTPINVPASV:
	  printwhat (count, opt.ntry);
	  /* non-fatal errors */
	  continue;
	  break;
	case FTPRETRINT:
	  /* If the control connection was closed, the retrieval
	     will be considered OK if f->size == len.  */
	  if (!f || len != f->size)
	    {
	      printwhat (count, opt.ntry);
	      continue;
	    }
	  break;
	case RETRFINISHED:
	  /* Great!  */
	  break;
	default:
	  /* Not as great.  */
	  abort ();
	}
      /* Time?  */
      tms = time_str (NULL);
      if (!opt.spider)
        tmrate = retr_rate (len - restval, con->dltime, 0);

      /* If we get out of the switch above without continue'ing, we've
	 successfully downloaded a file.  Remember this fact. */
      downloaded_file (FILE_DOWNLOADED_NORMALLY, locf);

      if (con->st & ON_YOUR_OWN)
	{
	  CLOSE (RBUF_FD (&con->rbuf));
	  rbuf_uninitialize (&con->rbuf);
	}
      if (!opt.spider)
        logprintf (LOG_VERBOSE, _("%s (%s) - `%s' saved [%ld]\n\n"),
		   tms, tmrate, locf, len);
      if (!opt.verbose && !opt.quiet)
	{
	  /* Need to hide the password from the URL.  The `if' is here
             so that we don't do the needless allocation every
             time. */
	  char *hurl = url_string (u, 1);
	  logprintf (LOG_NONVERBOSE, "%s URL: %s [%ld] -> \"%s\" [%d]\n",
		     tms, hurl, len, locf, count);
	  xfree (hurl);
	}

      if ((con->cmd & DO_LIST))
	/* This is a directory listing file. */
	{
	  if (!opt.remove_listing)
	    /* --dont-remove-listing was specified, so do count this towards the
	       number of bytes and files downloaded. */
	    {
	      total_downloaded_bytes += len;
	      opt.numurls++;
	    }

	  /* Deletion of listing files is not controlled by --delete-after, but
	     by the more specific option --dont-remove-listing, and the code
	     to do this deletion is in another function. */
	}
      else if (!opt.spider)
	/* This is not a directory listing file. */
	{
	  /* Unlike directory listing files, don't pretend normal files weren't
	     downloaded if they're going to be deleted.  People seeding proxies,
	     for instance, may want to know how many bytes and files they've
	     downloaded through it. */
	  total_downloaded_bytes += len;
	  opt.numurls++;

	  if (opt.delete_after)
	    {
	      DEBUGP (("Removing file due to --delete-after in"
		       " ftp_loop_internal():\n"));
	      logprintf (LOG_VERBOSE, _("Removing %s.\n"), locf);
	      if (unlink (locf))
		logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
	    }
	}
      
      /* Restore the original leave-pendingness.  */
      if (orig_lp)
	con->cmd |= LEAVE_PENDING;
      else
	con->cmd &= ~LEAVE_PENDING;
      return RETROK;
    } while (!opt.ntry || (count < opt.ntry));

  if (rbuf_initialized_p (&con->rbuf) && (con->st & ON_YOUR_OWN))
    {
      CLOSE (RBUF_FD (&con->rbuf));
      rbuf_uninitialize (&con->rbuf);
    }
  return TRYLIMEXC;
}

/* Return the directory listing in a reusable format.  The directory
   is specifed in u->dir.  */
uerr_t
ftp_get_listing (struct url *u, ccon *con, struct fileinfo **f)
{
  uerr_t err;
  char *uf;			/* url file name */
  char *lf;			/* list file name */
  char *old_target = con->target;

  con->st &= ~ON_YOUR_OWN;
  con->cmd |= (DO_LIST | LEAVE_PENDING);
  con->cmd &= ~DO_RETR;

  /* Find the listing file name.  We do it by taking the file name of
     the URL and replacing the last component with the listing file
     name.  */
  uf = url_file_name (u);
  lf = file_merge (uf, LIST_FILENAME);
  xfree (uf);
  DEBUGP ((_("Using `%s' as listing tmp file.\n"), lf));

  con->target = lf;
  err = ftp_loop_internal (u, NULL, con);
  con->target = old_target;

  if (err == RETROK)
    *f = ftp_parse_ls (lf, con->rs);
  else
    *f = NULL;
  if (opt.remove_listing)
    {
      if (unlink (lf))
	logprintf (LOG_NOTQUIET, "unlink: %s\n", strerror (errno));
      else
	logprintf (LOG_VERBOSE, _("Removed `%s'.\n"), lf);
    }
  xfree (lf);
  con->cmd &= ~DO_LIST;
  return err;
}

static uerr_t ftp_retrieve_dirs PARAMS ((struct url *, struct fileinfo *,
					 ccon *));
static uerr_t ftp_retrieve_glob PARAMS ((struct url *, ccon *, int));
static struct fileinfo *delelement PARAMS ((struct fileinfo *,
					    struct fileinfo **));
static void freefileinfo PARAMS ((struct fileinfo *f));

/* Retrieve a list of files given in struct fileinfo linked list.  If
   a file is a symbolic link, do not retrieve it, but rather try to
   set up a similar link on the local disk, if the symlinks are
   supported.

   If opt.recursive is set, after all files have been retrieved,
   ftp_retrieve_dirs will be called to retrieve the directories.  */
static uerr_t
ftp_retrieve_list (struct url *u, struct fileinfo *f, ccon *con)
{
  static int depth = 0;
  uerr_t err;
  struct fileinfo *orig;
  long local_size;
  time_t tml;
  int dlthis;

  /* Increase the depth.  */
  ++depth;
  if (opt.reclevel != INFINITE_RECURSION && depth > opt.reclevel)
    {
      DEBUGP ((_("Recursion depth %d exceeded max. depth %d.\n"),
	       depth, opt.reclevel));
      --depth;
      return RECLEVELEXC;
    }

  assert (f != NULL);
  orig = f;

  con->st &= ~ON_YOUR_OWN;
  if (!(con->st & DONE_CWD))
    con->cmd |= DO_CWD;
  else
    con->cmd &= ~DO_CWD;
  con->cmd |= (DO_RETR | LEAVE_PENDING);

  if (!rbuf_initialized_p (&con->rbuf))
    con->cmd |= DO_LOGIN;
  else
    con->cmd &= ~DO_LOGIN;

  err = RETROK;			/* in case it's not used */

  while (f)
    {
      char *old_target, *ofile;

      if (opt.quota && total_downloaded_bytes > opt.quota)
	{
	  --depth;
	  return QUOTEXC;
	}
      old_target = con->target;

      ofile = xstrdup (u->file);
      url_set_file (u, f->name);

      con->target = url_file_name (u);
      err = RETROK;

      dlthis = 1;
      if (opt.timestamping && f->type == FT_PLAINFILE)
        {
	  struct stat st;
	  /* If conversion of HTML files retrieved via FTP is ever implemented,
	     we'll need to stat() <file>.orig here when -K has been specified.
	     I'm not implementing it now since files on an FTP server are much
	     more likely than files on an HTTP server to legitimately have a
	     .orig suffix. */
	  if (!stat (con->target, &st))
	    {
              int eq_size;
              int cor_val;
	      /* Else, get it from the file.  */
	      local_size = st.st_size;
	      tml = st.st_mtime;
#ifdef WINDOWS
	      /* Modification time granularity is 2 seconds for Windows, so
		 increase local time by 1 second for later comparison. */
	      tml++;
#endif
              /* Compare file sizes only for servers that tell us correct
                 values. Assumme sizes being equal for servers that lie
                 about file size.  */
              cor_val = (con->rs == ST_UNIX || con->rs == ST_WINNT);
              eq_size = cor_val ? (local_size == f->size) : 1 ;
	      if (f->tstamp <= tml && eq_size)