ftp.c

来自「网络爬虫程序」· C语言 代码 · 共 2,200 行 · 第 1/4 页

/***************************************************************************/
/*    This code is part of WWW grabber called pavuk                        */
/*    Copyright (c) 1997 - 2001 Stefan Ondrejicka                          */
/*    Distributed under GPL 2 or later                                     */
/***************************************************************************/

#include "config.h"

#include <ctype.h>
#include <errno.h>
#include <errno.h>
#include <fcntl.h>
#include <netdb.h>
#include <netinet/in.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>
#include <unistd.h>
#include <unistd.h>
#ifdef HAVE_ARPA_INET_H
#include <arpa/inet.h>
#endif

#include "absio.h"
#include "abstract.h"
#include "dns.h"
#include "doc.h"
#include "errcode.h"
#include "ftp.h"
#include "http.h"
#include "mode.h"
#include "myssl.h"
#include "net.h"
#include "times.h"
#include "tools.h"
#include "uexit.h"
#include "url.h"

static ftp_handshake_info *ftp_handshake_info_data_find(char *, int);
static int ftp_do_login_handshake_cust(ftp_handshake_info *, doc *, char *,
  char *);

static int ftp_control_write(doc * docp, char *buf, size_t bufsize)
{
  return bufio_write(docp->ftp_control, buf, bufsize);
}

static int ftp_control_readln(doc * docp, char *buf, int bufsize)
{
  return bufio_readln(docp->ftp_control, buf, bufsize);
}

/********************************************************/
/* zistenie navratoveho kodu z FTP servera              */
/* FIXME: Translate me!                                 */
/********************************************************/
int ftp_get_response(doc * docp, char **mbuf, int setrc)
{
  char buf[1024];
  bool_t end = FALSE;
  int alen, tlen = 0;
  char *sresp = NULL;

  while(!end)
  {
    if((alen = ftp_control_readln(docp, buf, sizeof(buf))) > 0)
    {
      DEBUG_PROTOS("%s", buf);
      if(mbuf)
      {
        tlen += alen;
        sresp = _realloc(sresp, tlen + 1);
        memcpy(sresp + tlen - alen, buf, alen + 1);
      }

      if(strlen(buf) > 3)
      {
        if(tl_ascii_isdigit(*(buf)) &&
          tl_ascii_isdigit(*(buf + 1)) &&
          tl_ascii_isdigit(*(buf + 2)) && tl_ascii_isspace(*(buf + 3)))
          end = TRUE;
      }
    }
    else
    {
      if(alen)
        xperror("ftp_get_response");
      else
        xprintf(1,
          gettext
          ("ftp_get_response: ftp control connection closed before any response\n"));
      if(setrc)
        docp->ftp_respc = 600;
      return 600;
    }
  }

  if(mbuf)
    *mbuf = sresp;

  if(setrc)
    docp->ftp_respc = _atoi(buf);

  return _atoi(buf);
}

static bufio *ftp_open_data_finish(doc * docp)
{
  if(docp->ftp_data_con_finished)
    return docp->datasock;

  if(!cfg.ftp_activec || (priv_cfg.ftp_proxy && cfg.ftp_dirtyp))
  {
    if(priv_cfg.ftp_proxy && cfg.ftp_dirtyp)
    {
      if(!(docp->datasock =
          bufio_sock_fdopen(net_connect(priv_cfg.ftp_proxy,
              cfg.ftp_proxy_port, docp))))
      {
        if(_h_errno_ != 0)
          xherror(priv_cfg.ftp_proxy);
        else
          xperror("net_connect");

        docp->errcode = ERR_PROXY_CONNECT;
        return NULL;
      }
      if(http_dumy_proxy_connect(docp, docp->ftp_pasv_host,
          docp->ftp_pasv_port, priv_cfg.ftp_proxy, cfg.ftp_proxy_port))
      {
        docp->errcode = ERR_PROXY_CONNECT;
        bufio_close(docp->datasock);
        docp->datasock = NULL;
        return NULL;
      }
    }
    else
    {
      if(!(docp->datasock =
          bufio_sock_fdopen(net_connect(docp->ftp_pasv_host,
              docp->ftp_pasv_port, docp))))
      {
        if(_h_errno_ != 0)
          xherror(priv_cfg.ftp_proxy);
        else
          xperror("net_connect");

        docp->errcode = ERR_FTP_DATACON;
        return NULL;
      }
    }
  }
  else
  {
    bufio *dsock = bufio_sock_fdopen(net_accept(bufio_getfd(docp->datasock)));
    bufio_close(docp->datasock);
    docp->datasock = dsock;
  }

#ifdef IP_TOS
#ifdef IPTOS_THROUGHPUT
  if(docp->datasock)
  {
    int v = IPTOS_THROUGHPUT;

    if(setsockopt(bufio_getfd(docp->datasock),
        IPPROTO_IP, IP_TOS, (char *) &v, sizeof(v)))
    {
      xperror(gettext("ftp: setsockopt TOS - THROUGHPUT failed"));
    }
  }
#endif
#endif

#ifdef USE_SSL
  if(docp->doc_url->type == URLT_FTPS)
  {
    bufio *ssl_sock;

    ssl_sock = my_ssl_do_connect(docp, docp->datasock,
      bufio_get_ssl_hook_data(docp->ftp_control));

    if(!ssl_sock)
    {
      if(!docp->errcode)
        docp->errcode = ERR_FTPS_DATASSLCONNECT;
      bufio_close(docp->datasock);
      docp->datasock = NULL;
    }
    else
      docp->datasock = ssl_sock;
  }
#endif

  if(docp->datasock)
    docp->ftp_data_con_finished = TRUE;

  return docp->datasock;
}

static int ftp_open_data_init(doc * docp)
{
  char buf[256];
  char *mbuf;
  struct sockaddr_storage scaddr;
  struct sockaddr *caddr;
  int reply;
  socklen_t n;

  caddr = (struct sockaddr *) &scaddr;

  n = sizeof(scaddr);
  if(getsockname(bufio_getfd(docp->ftp_control), caddr, &n))
  {
    xperror("FTP control getsockname");
    docp->datasock = NULL;
    return -1;
  }


  if(!cfg.ftp_activec || (priv_cfg.ftp_proxy && cfg.ftp_dirtyp))
  {
#ifdef HAVE_INET6
    if(caddr->sa_family == AF_INET6)
    {
      sprintf(buf, "EPSV\r\n");
      DEBUG_PROTOC("%s", buf);

      ftp_control_write(docp, buf, strlen(buf));

      mbuf = NULL;

      if((reply = ftp_get_response(docp, &mbuf, TRUE)) >= 400)
      {
        int h[16], p[2], hal, af, pal;
        abs_addr haddr;

        _free(mbuf) if(reply == 600)
        {
          docp->ftp_fatal_err = TRUE;
          return -1;
        }
        sprintf(buf, "LPSV\r\n");
        DEBUG_PROTOC("%s", buf);

        ftp_control_write(docp, buf, strlen(buf));

        mbuf = NULL;

        if((reply = ftp_get_response(docp, &mbuf, TRUE)) >= 400)
        {
          docp->ftp_fatal_err = (reply == 600);
          _free(mbuf);
          return -1;
        }
        n = sscanf(mbuf,
          "%d %*[^0-9]%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d",
          &reply, &af, &hal, &h[0], &h[1], &h[2], &h[3],
          &h[4], &h[5], &h[6], &h[7], &h[8], &h[9],
          &h[10], &h[11], &h[12], &h[13], &h[14], &h[15], &pal, &p[0], &p[1]);
        if(n != 22 || reply != 228 || af != 6 || hal != 16 || pal != 2)
        {
          xprintf(1,
            gettext("Error parsing FTP response to %s command - %s\n"),
            "LPSV", mbuf);
          _free(mbuf);
          return -1;
        }
        _free(mbuf);
        docp->ftp_pasv_port = (p[0] << 8) + p[1];
        haddr.family = AF_INET6;
        for(n = 0; n < 16; n++)
          haddr.addr[n] = (unsigned char) h[n];
        docp->ftp_pasv_host = dns_get_abs_addr_ip(&haddr);
      }
      else
      {
        int port;
        char d[4];

        n = sscanf(mbuf, "%d %*[^0-9(]\(%c%c%c%d%c", &reply,
          &d[0], &d[1], &d[2], &port, &d[3]);
        if(n != 6 || reply != 229 ||
          d[0] != d[1] || d[0] != d[2] || d[0] != d[3])
        {
          xprintf(1,
            gettext("Error parsing FTP response to %s command - %s\n"),
            "EPSV", mbuf);
          _free(mbuf);
          return -1;
        }
        _free(mbuf);
        docp->ftp_pasv_port = port;
        docp->ftp_pasv_host = dns_get_sockaddr_ip(caddr);
      }
    }
    else
#endif
    {
      int h0, h1, h2, h3, p0, p1;

      sprintf(buf, "PASV\r\n");
      DEBUG_PROTOC("%s", buf);

      ftp_control_write(docp, buf, strlen(buf));

      mbuf = NULL;

      if((reply = ftp_get_response(docp, &mbuf, TRUE)) >= 400)
      {
        docp->ftp_fatal_err = (reply == 600);
        _free(mbuf);
        return -1;
      }

      if(!mbuf)
      {
        return -1;
      }

      n = sscanf(mbuf, "%d %*[^0-9]%d,%d,%d,%d,%d,%d", &reply,
        &h0, &h1, &h2, &h3, &p0, &p1);


      if(n != 7 || reply != 227)
      {
        xprintf(1, gettext("Error parsing FTP response to %s command - %s\n"),
          "PASV", mbuf);
        _free(mbuf);
        return -1;
      }
      _free(mbuf);

      docp->ftp_pasv_port = (p0 << 8) + p1;

      sprintf(buf, "%d.%d.%d.%d", h0, h1, h2, h3);
      docp->ftp_pasv_host = tl_strdup(buf);
    }
  }
  else
  {
    struct sockaddr_storage saddr;
    struct sockaddr *addr;
    char *p;
    int port;

    addr = (struct sockaddr *) &saddr;

    if(!docp->datasock)
    {
      int bfd;
      abs_addr laddr;
      memset(laddr.addr, '\0', sizeof(laddr.addr));
      laddr.family = caddr->sa_family;

      if(priv_cfg.local_ip)
      {
        memcpy(laddr.addr, cfg.local_ip_addr.addr, sizeof(laddr.addr));
        laddr.family = cfg.local_ip_addr.family;
      }

      bfd = net_bindport(&laddr, cfg.active_ftp_min_port,
        cfg.active_ftp_max_port);

      if(!(docp->datasock = bufio_sock_fdopen(bfd)))
      {
        xperror("bind");
        return -1;
      }
    }

    n = sizeof(saddr);
    if(getsockname(bufio_getfd(docp->datasock), addr, &n))
    {
      xperror("FTP data getsockname");
      bufio_close(docp->datasock);
      docp->datasock = NULL;
      return -1;
    }

    port = dns_get_sockaddr_port(addr);
    mbuf = dns_get_sockaddr_ip(caddr);
#ifdef HAVE_INET6
    if(caddr->sa_family == AF_INET6)
    {
      snprintf(buf, sizeof(buf), "EPRT |2|%s|%d|\r\n", mbuf, port);
      _free(mbuf);

      ftp_control_write(docp, buf, strlen(buf));
      DEBUG_PROTOC("%s", buf);

      if((reply = ftp_get_response(docp, NULL, TRUE)) >= 400)
      {
        unsigned char *h;

        if(reply == 600)
        {
          bufio_close(docp->datasock);
          docp->datasock = NULL;
          docp->ftp_fatal_err = TRUE;
          return -1;
        }
        h = ((struct sockaddr_in6 *) caddr)->sin6_addr.s6_addr;

        sprintf(buf,
          "LPRT 6,16,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,2,%d,%d\r\n",
          h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], h[8], h[9], h[10],
          h[11], h[12], h[13], h[14], h[15], port / 256, port % 256);

        ftp_control_write(docp, buf, strlen(buf));
        DEBUG_PROTOC("%s", buf);

        if((reply = ftp_get_response(docp, NULL, TRUE)) >= 400)
        {
          bufio_close(docp->datasock);
          docp->datasock = NULL;
          docp->ftp_fatal_err = (reply == 600);
          return -1;
        }
      }
    }
    else
#endif
    {
      for(p = mbuf; (p = strchr(p, '.')); *p = ',');
      snprintf(buf, sizeof(buf), "PORT %s,%d,%d\r\n", mbuf, port / 256, port % 256);
      _free(mbuf);

      ftp_control_write(docp, buf, strlen(buf));
      DEBUG_PROTOC("%s", buf);

      if((reply = ftp_get_response(docp, NULL, TRUE)) >= 400)
      {
        bufio_close(docp->datasock);
        docp->datasock = NULL;
        docp->ftp_fatal_err = (reply == 600);
        return -1;
      }
    }
  }

  return 0;
}

static int ftp_do_login_handshake(doc * docp, char *ruser, char *user,
  char *password)
{
  ftp_handshake_info *fhi;

  fhi = ftp_handshake_info_data_find(docp->doc_url->p.ftp.host,
    docp->doc_url->p.ftp.port);

  if(!fhi)
    fhi = ftp_handshake_info_data_find("", DEFAULT_FTP_PORT);

  if(fhi)
  {
    if(ftp_do_login_handshake_cust(fhi, docp, ruser, password))
    {
      docp->ftp_fatal_err = TRUE;
      docp->errcode = ERR_FTP_LOGIN_HANDSHAKE;
      return -1;
    }
  }
  else
  {
    char buf[2048];
    int reply;

    if(priv_cfg.ftp_proxy_user && priv_cfg.ftp_proxy &&
      !cfg.ftp_via_http && !cfg.ftp_dirtyp)
    {
      snprintf(buf, sizeof(buf), "USER %s\r\n", priv_cfg.ftp_proxy_user);
      ftp_control_write(docp, buf, strlen(buf));
      DEBUG_PROTOC("%s", buf);

      if((reply = ftp_get_response(docp, NULL, TRUE)) >= 400)
      {
        docp->ftp_fatal_err = TRUE;
        docp->errcode = ERR_FTP_BPROXYUSER;
        return -1;
      }
    }
    else
      reply = 0;

    if(priv_cfg.ftp_proxy_pass && priv_cfg.ftp_proxy &&
      !cfg.ftp_via_http && !cfg.ftp_dirtyp)
    {

      if(reply == 331)  /*** we need password ***/
      {
        snprintf(buf, sizeof(buf), "PASS %s\r\n", priv_cfg.ftp_proxy_pass);
        ftp_control_write(docp, buf, strlen(buf));
        DEBUG_PROTOC("%s", buf);

        if(ftp_get_response(docp, NULL, TRUE) >= 400)
        {
          docp->ftp_fatal_err = TRUE;
          docp->errcode = ERR_FTP_BPROXYPASS;
          return -1;
        }
      }
    }

    snprintf(buf, sizeof(buf), "USER %s\r\n", user);
    ftp_control_write(docp, buf, strlen(buf));
    DEBUG_PROTOC("%s", buf);

    if((reply = ftp_get_response(docp, NULL, TRUE)) >= 400)
    {
      docp->ftp_fatal_err = TRUE;
      docp->errcode = ERR_FTP_BUSER;
      return -1;
    }

    if(reply == 331)  /*** we need password ***/
    {
      snprintf(buf, sizeof(buf), "PASS %s\r\n", password);
      ftp_control_write(docp, buf, strlen(buf));
      DEBUG_PROTOC("%s", buf);

      if(ftp_get_response(docp, NULL, TRUE) >= 400)
      {
        docp->ftp_fatal_err = TRUE;
        docp->errcode = ERR_FTP_BPASS;
        return -1;
      }
    }
  }

  return 0;
}

#define CLOSE_CONTROL \
  bufio_close(docp->ftp_control);\
  docp->ftp_control = NULL;

static bufio *ftp_open_control_connection(doc * docp, char *host, int port,
  char *ruser, char *user, char *password)
{

  docp->errcode = ERR_NOERROR;
  docp->ftp_fatal_err = FALSE;
  docp->ftp_respc = 0;