ftp.c

harvest是一个下载html网页得机器人

            from, to, totalsize);
      ftp->dont_check = TRUE; /* dont check for successful transfer */
    }

    if((data->set.ftp_list_only) || !ftp->file) {
      /* The specified path ends with a slash, and therefore we think this
         is a directory that is requested, use LIST. But before that we
         need to set ASCII transfer mode. */
      dirlist = TRUE;

      /* Set type to ASCII */
      result = ftp_transfertype(conn, TRUE /* ASCII enforced */);
      if(result)
        return result;

      /* if this output is to be machine-parsed, the NLST command will be
         better used since the LIST command output is not specified or
         standard in any way */

      FTPSENDF(conn, "%s",
            data->set.customrequest?data->set.customrequest:
            (data->set.ftp_list_only?"NLST":"LIST"));
    }
    else {
      ssize_t foundsize;

      /* Set type to binary (unless specified ASCII) */
      result = ftp_transfertype(conn, data->set.ftp_ascii);
      if(result)
        return result;

      /* Send any PREQUOTE strings after transfer type is set? */
      if(data->set.prequote) {
        if ((result = ftp_sendquote(conn, data->set.prequote)) != CURLE_OK)
          return result;
      }

      /* Attempt to get the size, it'll be useful in some cases: for resumed
         downloads and when talking to servers that don't give away the size
         in the RETR response line. */
      result = ftp_getsize(conn, ftp->file, &foundsize);
      if(CURLE_OK == result) {
        if (data->set.max_filesize && foundsize > data->set.max_filesize) {
          failf(data, "Maximum file size exceeded");
          return CURLE_FILESIZE_EXCEEDED;
        }
        downloadsize = foundsize;
      }

      if(conn->resume_from) {

        /* Daniel: (August 4, 1999)
         *
         * We start with trying to use the SIZE command to figure out the size
         * of the file we're gonna get. If we can get the size, this is by far
         * the best way to know if we're trying to resume beyond the EOF.
         *
         * Daniel, November 28, 2001. We *always* get the size on downloads
         * now, so it is done before this even when not doing resumes. I saved
         * the comment above for nostalgical reasons! ;-)
         */
        if(CURLE_OK != result) {
          infof(data, "ftp server doesn't support SIZE\n");
          /* We couldn't get the size and therefore we can't know if there
             really is a part of the file left to get, although the server
             will just close the connection when we start the connection so it
             won't cause us any harm, just not make us exit as nicely. */
        }
        else {
          /* We got a file size report, so we check that there actually is a
             part of the file left to get, or else we go home.  */
          if(conn->resume_from< 0) {
            /* We're supposed to download the last abs(from) bytes */
            if(foundsize < -conn->resume_from) {
              failf(data, "Offset (%d) was beyond file size (%d)",
                    conn->resume_from, foundsize);
              return CURLE_FTP_BAD_DOWNLOAD_RESUME;
            }
            /* convert to size to download */
            downloadsize = -conn->resume_from;
            /* download from where? */
            conn->resume_from = foundsize - downloadsize;
          }
          else {
            if(foundsize < conn->resume_from) {
              failf(data, "Offset (%d) was beyond file size (%d)",
                    conn->resume_from, foundsize);
              return CURLE_FTP_BAD_DOWNLOAD_RESUME;
            }
            /* Now store the number of bytes we are expected to download */
            downloadsize = foundsize-conn->resume_from;
          }
        }

        if (downloadsize == 0) {
          /* no data to transfer */
          result=Curl_Transfer(conn, -1, -1, FALSE, NULL, -1, NULL);
          infof(data, "File already completely downloaded\n");

          /* Set no_transfer so that we won't get any error in Curl_ftp_done()
           * because we didn't transfer the any file */
          ftp->no_transfer = TRUE;
          return CURLE_OK;
        }
	
        /* Set resume file transfer offset */
        infof(data, "Instructs server to resume from offset %d\n",
              conn->resume_from);

        FTPSENDF(conn, "REST %d", conn->resume_from);

        result = Curl_GetFTPResponse(&nread, conn, &ftpcode);
        if(result)
          return result;

        if(ftpcode != 350) {
          failf(data, "Couldn't use REST: %s", buf+4);
          return CURLE_FTP_COULDNT_USE_REST;
        }
      }

      FTPSENDF(conn, "RETR %s", ftp->file);
    }

    result = Curl_GetFTPResponse(&nread, conn, &ftpcode);
    if(result)
      return result;

    if((ftpcode == 150) || (ftpcode == 125)) {

      /*
        A;
        150 Opening BINARY mode data connection for /etc/passwd (2241
        bytes).  (ok, the file is being transfered)
	
        B:
        150 Opening ASCII mode data connection for /bin/ls 

        C:
        150 ASCII data connection for /bin/ls (137.167.104.91,37445) (0 bytes).

        D:
        150 Opening ASCII mode data connection for /linux/fisk/kpanelrc (0.0.0.0,0) (545 bytes).
          
        E:
        125 Data connection already open; Transfer starting. */

      int size=-1; /* default unknown size */


      /*
       * It appears that there are FTP-servers that return size 0 for files
       * when SIZE is used on the file while being in BINARY mode. To work
       * around that (stupid) behavior, we attempt to parse the RETR response
       * even if the SIZE returned size zero.
       *
       * Debugging help from Salvatore Sorrentino on February 26, 2003.
       */

      if(!dirlist &&
         !data->set.ftp_ascii &&
         (downloadsize < 1)) {
        /*
         * It seems directory listings either don't show the size or very
         * often uses size 0 anyway. ASCII transfers may very well turn out
         * that the transfered amount of data is not the same as this line
         * tells, why using this number in those cases only confuses us.
         *
         * Example D above makes this parsing a little tricky */
        char *bytes;
        bytes=strstr(buf, " bytes");
        if(bytes--) {
          int in=bytes-buf;
          /* this is a hint there is size information in there! ;-) */
          while(--in) {
            /* scan for the parenthesis and break there */
            if('(' == *bytes)
              break;
            /* if only skip digits, or else we're in deep trouble */
            if(!isdigit((int)*bytes)) {
              bytes=NULL;
              break;
            }
            /* one more estep backwards */
            bytes--;
          }
          /* only if we have nothing but digits: */
          if(bytes++) {
            /* get the number! */
            size = atoi(bytes);
          }
            
        }
      }
      else if(downloadsize > -1)
        size = downloadsize;

      if(data->set.ftp_use_port) {
        result = AllowServerConnect(data, conn, conn->secondarysocket);
        if( result )
          return result;
      }

      infof(data, "Getting file with size: %d\n", size);

      /* FTP download: */
      result=Curl_Transfer(conn, conn->secondarysocket, size, FALSE,
                           bytecountp,
                           -1, NULL); /* no upload here */
      if(result)
        return result;
    }
    else {
      if(dirlist && (ftpcode == 450)) {
        /* simply no matching files */
        ftp->no_transfer = TRUE; /* don't think we should download anything */
      }
      else {
        failf(data, "%s", buf+4);
        return CURLE_FTP_COULDNT_RETR_FILE;
      }
    }
	
  }
  /* end of transfer */

  return CURLE_OK;
}

/***********************************************************************
 *
 * ftp_perform()
 *
 * This is the actual DO function for FTP. Get a file/directory according to
 * the options previously setup.
 */

static
CURLcode ftp_perform(struct connectdata *conn,
                     bool *connected)  /* for the TCP connect status after
                                          PASV / PORT */
{
  /* this is FTP and no proxy */
  CURLcode result=CURLE_OK;
  struct SessionHandle *data=conn->data;
  char *buf = data->state.buffer; /* this is our buffer */

  /* the ftp struct is already inited in Curl_ftp_connect() */
  struct FTP *ftp = conn->proto.ftp;

  /* Send any QUOTE strings? */
  if(data->set.quote) {
    if ((result = ftp_sendquote(conn, data->set.quote)) != CURLE_OK)
      return result;
  }

  /* This is a re-used connection. Since we change directory to where the
     transfer is taking place, we must now get back to the original dir
     where we ended up after login: */
  if (conn->bits.reuse && ftp->entrypath) {
    if ((result = ftp_cwd_and_mkd(conn, ftp->entrypath)) != CURLE_OK)
      return result;
  }

  {
    int i; /* counter for loop */
    for (i=0; ftp->dirs[i]; i++) {
      /* RFC 1738 says empty components should be respected too, but
         that is plain stupid since CWD can't be used with an empty argument */
      if ((result = ftp_cwd_and_mkd(conn, ftp->dirs[i])) != CURLE_OK)
        return result;
    }
  }

  /* Requested time of file or time-depended transfer? */
  if((data->set.get_filetime || data->set.timecondition) &&
     ftp->file) {
    result = ftp_getfiletime(conn, ftp->file);
    switch( result )
      {
      case CURLE_FTP_COULDNT_RETR_FILE:
      case CURLE_OK:
        if(data->set.timecondition) {
          if((data->info.filetime > 0) && (data->set.timevalue > 0)) {
            switch(data->set.timecondition) {
            case TIMECOND_IFMODSINCE:
            default:
              if(data->info.filetime < data->set.timevalue) {
                infof(data, "The requested document is not new enough\n");
                ftp->no_transfer = TRUE; /* mark this to not transfer data */
                return CURLE_OK;
              }
              break;
            case TIMECOND_IFUNMODSINCE:
              if(data->info.filetime > data->set.timevalue) {
                infof(data, "The requested document is not old enough\n");
                ftp->no_transfer = TRUE; /* mark this to not transfer data */
                return CURLE_OK;
              }
              break;
            } /* switch */
          }
          else {
            infof(data, "Skipping time comparison\n");
          }
        }
        break;
      default:
        return result;
      } /* switch */
  }

  /* If we have selected NOBODY and HEADER, it means that we only want file
     information. Which in FTP can't be much more than the file size and
     date. */
  if(data->set.no_body && data->set.include_header && ftp->file) {
    /* The SIZE command is _not_ RFC 959 specified, and therefor many servers
       may not support it! It is however the only way we have to get a file's
       size! */
    ssize_t filesize;
    ssize_t nread;
    int ftpcode;

    ftp->no_transfer = TRUE; /* this means no actual transfer is made */
    
    /* Some servers return different sizes for different modes, and thus we
       must set the proper type before we check the size */
    result = ftp_transfertype(conn, data->set.ftp_ascii);
    if(result)
      return result;

    /* failing to get size is not a serious error */
    result = ftp_getsize(conn, ftp->file, &filesize);

    if(CURLE_OK == result) {
      sprintf(buf, "Content-Length: %d\r\n", filesize);
      result = Curl_client_write(data, CLIENTWRITE_BOTH, buf, 0);
      if(result)
        return result;
    }

    /* Determine if server can respond to REST command and therefore
       whether it can do a range */
    FTPSENDF(conn, "REST 0", NULL);
    result = Curl_GetFTPResponse(&nread, conn, &ftpcode);

    if ((CURLE_OK == result) && (ftpcode == 350)) {
      result = Curl_client_write(data, CLIENTWRITE_BOTH,
                                 (char *)"Accept-ranges: bytes\r\n", 0);
      if(result)
        return result;
    }

    /* If we asked for a time of the file and we actually got one as
       well, we "emulate" a HTTP-style header in our output. */

#ifdef HAVE_STRFTIME
    if(data->set.get_filetime && (data->info.filetime>=0) ) {
      struct tm *tm;
#ifdef HAVE_GMTIME_R
      struct tm buffer;
      tm = (struct tm *)gmtime_r((time_t *)&data->info.filetime, &buffer);
#else
      tm = gmtime((time_t *)&data->info.filetime);
#endif
      /* format: "Tue, 15 Nov 1994 12:45:26" */
      strftime(buf, BUFSIZE-1, "Last-Modified: %a, %d %b %Y %H:%M:%S GMT\r\n",
               tm);
      result = Curl_client_write(data, CLIENTWRITE_BOTH, buf, 0);
      if(result)
        return result;
    }
#endif

    return CURLE_OK;
  }

  if(data->set.no_body)
    /* doesn't really transfer any data */
    ftp->no_transfer = TRUE;
  /* Get us a second connection up and connected */
  else if(data->set.ftp_use_port) {
    /* We have chosen to use the PORT command */
    result = ftp_use_port(conn);
    if(CURLE_OK == result) {
      /* we have the data connection ready */
      infof(data, "Ordered connect of the data stream with PORT!\n");
      *connected = TRUE; /* mark us "still connected" */
    }
  }
  else {
    /* We have chosen (this is default) to use the PASV command */
    result = ftp_use_pasv(conn, connected);
    if(!result && *connected)
      infof(data, "Connected the data stream with PASV!\n");
  }
  
  return result;
}

/***********************************************************************
 *
 * Curl_ftp()
 *
 * This function is registered as 'curl_do' function. It decodes the path
 * parts etc as a wrapper to the actual DO function (ftp_perform).
 *
 * The input argument is already checked for validity.
 */
CURLcode Curl_ftp(struct connectdata *conn)
{
  CURLcode retcode=CURLE_OK;
  bool connected=0;

  struct SessionHandle *data = conn->data;
  struct FTP *ftp;

  char *slash_pos;  /* position of the first '/' char in curpos */
  char *cur_pos=conn->ppath; /* current position in ppath. point at the begin
                                of next path component */
  int path_part=0;/* current path component */

  /* the ftp struct is already inited in ftp_connect() */
  ftp = conn->proto.ftp;
  conn->size = -1; /* make sure this is unknown at this point */

  Curl_pgrsSetUploadCounter(data, 0);
  Curl_pgrsSetDownloadCounter(data, 0);
  Curl_pgrsSetUploadSize(data, 0);
  Curl_pgrsSetDownloadSize(data, 0);

  /*  fixed : initialize ftp->dirs[xxx] to NULL !
      is done in Curl_ftp_connect() */

  /* parse the URL path into separate path components